{ "schema_version": "1.3", "index_version": "0.4", "snapshot_date": "2026-09-13", "published_at": "2026-09-13T15:39:32Z", "supersedes": "2026-09-11", "source_index": "Artificial Analysis Intelligence Index v4.1.1 component evaluations", "retrieved": { "default": "2026-09-01", "overrides": [ { "model": "GPT-6 Astra", "date": "2026-09-04" } ] }, "rule": { "efficacy_bar": 0.85, "honesty_band_se_multiple": 2, "honesty_standard_error": 1.291, "grade_bounds_pct": { "A": 50.0, "B": 75.0 }, "accuracy_evals": [ "GPQA Diamond", "HLE", "CritPt", "SciCode", "Terminal-Bench v2.1", "AA-LCR" ], "gate_eval": "AA-Omniscience", "ranking": "APPROVED rows ordered by honesty grade (A, B, C), then cost_per_solve_usd" }, "cost_unit": "USD per attempted task on that evaluation, at first-party rates", "score_unit": "percent solved, except AA-Omniscience which is a net index from -100 to 100 (correct +1, wrong -1, abstain 0)", "scope": "bounded work only. GDPval-AA, tau3-Banking and Vending-Bench are excluded as long-horizon. Terminal-Bench v2.1 is kept as short agentic coding (tasks complete in minutes) and can be toggled off.", "counts": { "qualifies": 21, "fails_honesty": 20, "below_efficacy": 20, "honesty_inconclusive": 6, "cost_undefined": 4, "total": 71 }, "dispositions": { "approved": 21, "deny": 24, "review": 26 }, "grades": { "A": 5, "B": 12, "C": 4 }, "grade_sensitivity": [ { "b_bound_pct": 60.0, "counts": { "A": 5, "B": 4, "C": 12 }, "c_models": [ "Claude Fable 5.1", "Claude Opus 5", "GPT-5.6 Sol" ], "c_labs": [ "Anthropic", "OpenAI" ] }, { "b_bound_pct": 66.0, "counts": { "A": 5, "B": 8, "C": 8 }, "c_models": [ "Claude Fable 5.1", "GPT-5.6 Sol" ], "c_labs": [ "Anthropic", "OpenAI" ] }, { "b_bound_pct": 75.0, "counts": { "A": 5, "B": 12, "C": 4 }, "c_models": [ "GPT-5.6 Sol" ], "c_labs": [ "OpenAI" ] }, { "b_bound_pct": 80.0, "counts": { "A": 5, "B": 12, "C": 4 }, "c_models": [ "GPT-5.6 Sol" ], "c_labs": [ "OpenAI" ] }, { "b_bound_pct": 90.0, "counts": { "A": 5, "B": 12, "C": 4 }, "c_models": [ "GPT-5.6 Sol" ], "c_labs": [ "OpenAI" ] } ], "provenance": { "generator": "export_public_snapshot.py", "commit": "e212af8cbde728c59d7f16f94622351b18ed5fa6", "tree_dirty": true }, "notes": [ "Benchmark evidence, not a production measurement. Latency, tool access, context limits, retry behaviour and escalation policy are untested here.", "Effort labels are vendor-specific and are not comparable across models.", "answers_when_unsure_pct is measured on a pure-knowledge test with no retrieval. Within a family it is nearly constant across effort settings while accuracy moves, and it is uncorrelated with capability, so it is read as a trained trait and GRADED. The base rate of being wrong is task-specific and is NOT carried over from this test.", "Grade A declines at least as often as it answers when unsure. Grade C answers more than three times in four. Grade B is between. Ranking is grade first, then cost, so rank 1 is the default.", "v0.3 (2026-09-11) gated on the trait and approved four configurations from one lab; v0.2 (2026-09-10) priced review time from this test's error rate. Both are superseded and stay published." ], "configurations": [ { "id": "openai/gpt-6-astra@low", "aliases": [], "rank": 1, "rank_by_cost": 2, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "OpenAI", "model": "GPT-6 Astra", "effort": "low", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.1722, "efficacy_mean": 0.8787, "efficacy_min": 0.814, "weakest_eval": "CritPt", "honesty": { "net_per_100": 40.55, "accuracy_pct": 59.5333, "answers_when_unsure_pct": 46.911, "grade": "A", "confident_errors_per_100": 18.9833, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0172 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 93.1313, "share_of_best": 0.967, "cost_per_attempt": 0.015512, "cost_per_solved": 0.0167 }, "HLE": { "score": 49.2122, "share_of_best": 0.833, "cost_per_attempt": 0.038865, "cost_per_solved": 0.079 }, "CritPt": { "score": 26.2857, "share_of_best": 0.814, "cost_per_attempt": 0.14486, "cost_per_solved": 0.5511 }, "SciCode": { "score": 51.0417, "share_of_best": 0.823, "cost_per_attempt": 0.042507, "cost_per_solved": 0.0833 }, "Terminal-Bench v2.1": { "score": 88.015, "share_of_best": 0.963, "cost_per_attempt": 0.290309, "cost_per_solved": 0.3298 }, "AA-LCR": { "score": 72.6667, "share_of_best": 0.872, "cost_per_attempt": 0.950098, "cost_per_solved": 1.3075 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "meta/muse-spark-1.3@xhigh", "aliases": [], "rank": 2, "rank_by_cost": 4, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "Meta", "model": "Muse Spark 1.3", "effort": "xhigh", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.2135, "efficacy_mean": 0.9028, "efficacy_min": 0.803, "weakest_eval": "HLE", "honesty": { "net_per_100": 23.1167, "accuracy_pct": 41.5167, "answers_when_unsure_pct": 31.462, "grade": "A", "confident_errors_per_100": 18.4, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0675 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 94.1414, "share_of_best": 0.978, "cost_per_attempt": 0.029776, "cost_per_solved": 0.0316 }, "HLE": { "score": 47.4513, "share_of_best": 0.803, "cost_per_attempt": 0.104107, "cost_per_solved": 0.2194 }, "CritPt": { "score": 26.0, "share_of_best": 0.805, "cost_per_attempt": 0.308009, "cost_per_solved": 1.1846 }, "SciCode": { "score": 58.5648, "share_of_best": 0.945, "cost_per_attempt": 0.03707, "cost_per_solved": 0.0633 }, "Terminal-Bench v2.1": { "score": 85.3933, "share_of_best": 0.934, "cost_per_attempt": 0.93802, "cost_per_solved": 1.0985 }, "AA-LCR": { "score": 79.3333, "share_of_best": 0.952, "cost_per_attempt": 0.131481, "cost_per_solved": 0.1657 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "openai/gpt-6-astra@medium", "aliases": [], "rank": 3, "rank_by_cost": 6, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "OpenAI", "model": "GPT-6 Astra", "effort": "medium", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.2592, "efficacy_mean": 0.9148, "efficacy_min": 0.823, "weakest_eval": "SciCode", "honesty": { "net_per_100": 42.2167, "accuracy_pct": 60.5667, "answers_when_unsure_pct": 46.5342, "grade": "A", "confident_errors_per_100": 18.35, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0356 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 93.9394, "share_of_best": 0.976, "cost_per_attempt": 0.02436, "cost_per_solved": 0.0259 }, "HLE": { "score": 52.734, "share_of_best": 0.892, "cost_per_attempt": 0.100529, "cost_per_solved": 0.1906 }, "CritPt": { "score": 29.1429, "share_of_best": 0.902, "cost_per_attempt": 0.28512, "cost_per_solved": 0.9784 }, "SciCode": { "score": 51.0417, "share_of_best": 0.823, "cost_per_attempt": 0.050875, "cost_per_solved": 0.0997 }, "Terminal-Bench v2.1": { "score": 89.5131, "share_of_best": 0.979, "cost_per_attempt": 0.450612, "cost_per_solved": 0.5034 }, "AA-LCR": { "score": 76.3333, "share_of_best": 0.916, "cost_per_attempt": 0.95325, "cost_per_solved": 1.2488 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "openai/gpt-6-astra@high", "aliases": [], "rank": 4, "rank_by_cost": 8, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "OpenAI", "model": "GPT-6 Astra", "effort": "high", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.3602, "efficacy_mean": 0.9176, "efficacy_min": 0.833, "weakest_eval": "SciCode", "honesty": { "net_per_100": 43.7333, "accuracy_pct": 61.1333, "answers_when_unsure_pct": 44.7684, "grade": "A", "confident_errors_per_100": 17.4, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0622 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 94.9495, "share_of_best": 0.986, "cost_per_attempt": 0.037071, "cost_per_solved": 0.039 }, "HLE": { "score": 53.0584, "share_of_best": 0.898, "cost_per_attempt": 0.161848, "cost_per_solved": 0.305 }, "CritPt": { "score": 28.8571, "share_of_best": 0.893, "cost_per_attempt": 0.433525, "cost_per_solved": 1.5023 }, "SciCode": { "score": 51.6204, "share_of_best": 0.833, "cost_per_attempt": 0.069578, "cost_per_solved": 0.1348 }, "Terminal-Bench v2.1": { "score": 89.8876, "share_of_best": 0.983, "cost_per_attempt": 0.644953, "cost_per_solved": 0.7175 }, "AA-LCR": { "score": 76.0, "share_of_best": 0.912, "cost_per_attempt": 0.959718, "cost_per_solved": 1.2628 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "openai/gpt-6-astra@xhigh", "aliases": [], "rank": 5, "rank_by_cost": 15, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "OpenAI", "model": "GPT-6 Astra", "effort": "xhigh", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.5507, "efficacy_mean": 0.9265, "efficacy_min": 0.799, "weakest_eval": "SciCode", "honesty": { "net_per_100": 43.4167, "accuracy_pct": 61.85, "answers_when_unsure_pct": 48.318, "grade": "A", "confident_errors_per_100": 18.4333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.1156 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 96.2626, "share_of_best": 1.0, "cost_per_attempt": 0.072562, "cost_per_solved": 0.0754 }, "HLE": { "score": 54.5876, "share_of_best": 0.924, "cost_per_attempt": 0.253223, "cost_per_solved": 0.4639 }, "CritPt": { "score": 31.4286, "share_of_best": 0.973, "cost_per_attempt": 0.798015, "cost_per_solved": 2.5391 }, "SciCode": { "score": 49.537, "share_of_best": 0.799, "cost_per_attempt": 0.121633, "cost_per_solved": 0.2455 }, "Terminal-Bench v2.1": { "score": 89.1386, "share_of_best": 0.975, "cost_per_attempt": 0.870396, "cost_per_solved": 0.9765 }, "AA-LCR": { "score": 74.0, "share_of_best": 0.888, "cost_per_attempt": 0.96991, "cost_per_solved": 1.3107 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "google/gemini-3.8-flash@high", "aliases": [], "rank": 6, "rank_by_cost": 5, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "Google", "model": "Gemini 3.8 Flash", "effort": "high", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.2184, "efficacy_mean": 0.86, "efficacy_min": 0.566, "weakest_eval": "CritPt", "honesty": { "net_per_100": 29.55, "accuracy_pct": 54.6, "answers_when_unsure_pct": 55.1762, "grade": "B", "confident_errors_per_100": 25.05, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0446 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 95.25, "share_of_best": 0.989, "cost_per_attempt": 0.0245, "cost_per_solved": 0.0257 }, "HLE": { "score": 47.82, "share_of_best": 0.809, "cost_per_attempt": 0.092, "cost_per_solved": 0.1924 }, "CritPt": { "score": 18.29, "share_of_best": 0.566, "cost_per_attempt": 0.28, "cost_per_solved": 1.5309 }, "SciCode": { "score": 53.59, "share_of_best": 0.864, "cost_per_attempt": 0.076875, "cost_per_solved": 0.1435 }, "Terminal-Bench v2.1": { "score": 87.64, "share_of_best": 0.959, "cost_per_attempt": 0.717812, "cost_per_solved": 0.819 }, "AA-LCR": { "score": 81.0, "share_of_best": 0.972, "cost_per_attempt": 0.098833, "cost_per_solved": 0.122 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "anthropic/claude-opus-5@medium", "aliases": [], "rank": 7, "rank_by_cost": 9, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "Anthropic", "model": "Claude Opus 5", "effort": "medium", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.3792, "efficacy_mean": 0.8933, "efficacy_min": 0.818, "weakest_eval": "SciCode", "honesty": { "net_per_100": 31.02, "accuracy_pct": 57.0667, "answers_when_unsure_pct": 60.6755, "grade": "B", "confident_errors_per_100": 26.05, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.02 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 91.9, "share_of_best": 0.955, "cost_per_attempt": 0.0241, "cost_per_solved": 0.0262 }, "HLE": { "score": 51.3, "share_of_best": 0.868, "cost_per_attempt": 0.187, "cost_per_solved": 0.3645 }, "CritPt": { "score": 26.9, "share_of_best": 0.833, "cost_per_attempt": 1.4033, "cost_per_solved": 5.2167 }, "SciCode": { "score": 50.7, "share_of_best": 0.818, "cost_per_attempt": 0.0369, "cost_per_solved": 0.0728 }, "Terminal-Bench v2.1": { "score": 86.1, "share_of_best": 0.942, "cost_per_attempt": 0.7522, "cost_per_solved": 0.8736 }, "AA-LCR": { "score": 78.7, "share_of_best": 0.944, "cost_per_attempt": 0.7378, "cost_per_solved": 0.9375 } }, "match": { "api_ids": { "anthropic": [ "claude-opus-5" ], "bedrock": [ "us.anthropic.claude-opus-5" ] }, "confidence": "confirmed" } }, { "id": "anthropic/claude-fable-5.1@low", "aliases": [], "rank": 8, "rank_by_cost": 10, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "Anthropic", "model": "Claude Fable 5.1", "effort": "low", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.3974, "efficacy_mean": 0.8967, "efficacy_min": 0.827, "weakest_eval": "HLE", "honesty": { "net_per_100": 34.13, "accuracy_pct": 60.2333, "answers_when_unsure_pct": 65.6329, "grade": "B", "confident_errors_per_100": 26.1, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0252 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 88.1, "share_of_best": 0.915, "cost_per_attempt": 0.0164, "cost_per_solved": 0.0186 }, "HLE": { "score": 48.9, "share_of_best": 0.827, "cost_per_attempt": 0.1238, "cost_per_solved": 0.2532 }, "CritPt": { "score": 27.7, "share_of_best": 0.858, "cost_per_attempt": 1.2559, "cost_per_solved": 4.5339 }, "SciCode": { "score": 55.7, "share_of_best": 0.898, "cost_per_attempt": 0.069, "cost_per_solved": 0.1239 }, "Terminal-Bench v2.1": { "score": 85.0, "share_of_best": 0.93, "cost_per_attempt": 0.6832, "cost_per_solved": 0.8038 }, "AA-LCR": { "score": 79.3, "share_of_best": 0.952, "cost_per_attempt": 1.469, "cost_per_solved": 1.8525 } }, "match": { "api_ids": { "anthropic": [ "claude-fable-5-1" ] }, "confidence": "confirmed" } }, { "id": "moonshot/kimi-k3@max", "aliases": [], "rank": 9, "rank_by_cost": 12, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "Moonshot", "model": "Kimi K3", "effort": "max", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": 0.4953, "efficacy_mean": 0.8931, "efficacy_min": 0.724, "weakest_eval": "CritPt", "honesty": { "net_per_100": 19.7, "accuracy_pct": 47.5833, "answers_when_unsure_pct": 53.1955, "grade": "B", "confident_errors_per_100": 27.8833, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.6741 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 93.5, "share_of_best": 0.971, "cost_per_attempt": 0.1512, "cost_per_solved": 0.1617 }, "HLE": { "score": 46.9, "share_of_best": 0.794, "cost_per_attempt": 0.3936, "cost_per_solved": 0.8392 }, "CritPt": { "score": 23.4, "share_of_best": 0.724, "cost_per_attempt": 0.9019, "cost_per_solved": 3.8543 }, "SciCode": { "score": 58.7, "share_of_best": 0.947, "cost_per_attempt": 0.0602, "cost_per_solved": 0.1026 }, "Terminal-Bench v2.1": { "score": 85.0, "share_of_best": 0.93, "cost_per_attempt": 0.6271, "cost_per_solved": 0.7378 }, "AA-LCR": { "score": 82.7, "share_of_best": 0.992, "cost_per_attempt": 0.3086, "cost_per_solved": 0.3732 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "anthropic/claude-fable-5.1@medium", "aliases": [], "rank": 10, "rank_by_cost": 13, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "Anthropic", "model": "Claude Fable 5.1", "effort": "medium", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.5084, "efficacy_mean": 0.9218, "efficacy_min": 0.892, "weakest_eval": "SciCode", "honesty": { "net_per_100": 37.6, "accuracy_pct": 63.1, "answers_when_unsure_pct": 69.1057, "grade": "B", "confident_errors_per_100": 25.5, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0407 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 88.6, "share_of_best": 0.92, "cost_per_attempt": 0.0263, "cost_per_solved": 0.0297 }, "HLE": { "score": 53.8, "share_of_best": 0.91, "cost_per_attempt": 0.2254, "cost_per_solved": 0.419 }, "CritPt": { "score": 29.1, "share_of_best": 0.901, "cost_per_attempt": 1.8039, "cost_per_solved": 6.199 }, "SciCode": { "score": 55.3, "share_of_best": 0.892, "cost_per_attempt": 0.0756, "cost_per_solved": 0.1367 }, "Terminal-Bench v2.1": { "score": 88.0, "share_of_best": 0.963, "cost_per_attempt": 0.7691, "cost_per_solved": 0.874 }, "AA-LCR": { "score": 78.7, "share_of_best": 0.944, "cost_per_attempt": 1.4752, "cost_per_solved": 1.8745 } }, "match": { "api_ids": { "anthropic": [ "claude-fable-5-1" ] }, "confidence": "confirmed" } }, { "id": "anthropic/claude-opus-5@high", "aliases": [], "rank": 11, "rank_by_cost": 14, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "Anthropic", "model": "Claude Opus 5", "effort": "high", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.5493, "efficacy_mean": 0.9155, "efficacy_min": 0.876, "weakest_eval": "SciCode", "honesty": { "net_per_100": 33.72, "accuracy_pct": 58.8833, "answers_when_unsure_pct": 61.2079, "grade": "B", "confident_errors_per_100": 25.1667, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0279 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 93.7, "share_of_best": 0.973, "cost_per_attempt": 0.0452, "cost_per_solved": 0.0482 }, "HLE": { "score": 52.8, "share_of_best": 0.893, "cost_per_attempt": 0.3521, "cost_per_solved": 0.6669 }, "CritPt": { "score": 28.3, "share_of_best": 0.876, "cost_per_attempt": 1.9945, "cost_per_solved": 7.0477 }, "SciCode": { "score": 54.3, "share_of_best": 0.876, "cost_per_attempt": 0.0481, "cost_per_solved": 0.0886 }, "Terminal-Bench v2.1": { "score": 87.6, "share_of_best": 0.958, "cost_per_attempt": 1.2258, "cost_per_solved": 1.3993 }, "AA-LCR": { "score": 76.3, "share_of_best": 0.916, "cost_per_attempt": 0.7456, "cost_per_solved": 0.9772 } }, "match": { "api_ids": { "anthropic": [ "claude-opus-5" ], "bedrock": [ "us.anthropic.claude-opus-5" ] }, "confidence": "confirmed" } }, { "id": "anthropic/claude-fable-5.1@high", "aliases": [], "rank": 12, "rank_by_cost": 16, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "Anthropic", "model": "Claude Fable 5.1", "effort": "high", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.7325, "efficacy_mean": 0.9436, "efficacy_min": 0.924, "weakest_eval": "AA-LCR", "honesty": { "net_per_100": 40.8, "accuracy_pct": 64.9333, "answers_when_unsure_pct": 68.8213, "grade": "B", "confident_errors_per_100": 24.1333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0507 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 90.6, "share_of_best": 0.941, "cost_per_attempt": 0.0553, "cost_per_solved": 0.061 }, "HLE": { "score": 55.9, "share_of_best": 0.946, "cost_per_attempt": 0.397, "cost_per_solved": 0.7102 }, "CritPt": { "score": 30.3, "share_of_best": 0.938, "cost_per_attempt": 2.7477, "cost_per_solved": 9.0683 }, "SciCode": { "score": 57.6, "share_of_best": 0.929, "cost_per_attempt": 0.0891, "cost_per_solved": 0.1547 }, "Terminal-Bench v2.1": { "score": 89.9, "share_of_best": 0.984, "cost_per_attempt": 1.1876, "cost_per_solved": 1.321 }, "AA-LCR": { "score": 77.0, "share_of_best": 0.924, "cost_per_attempt": 1.481, "cost_per_solved": 1.9234 } }, "match": { "api_ids": { "anthropic": [ "claude-fable-5-1" ] }, "confidence": "confirmed" } }, { "id": "anthropic/claude-opus-5@xhigh", "aliases": [], "rank": 13, "rank_by_cost": 17, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "Anthropic", "model": "Claude Opus 5", "effort": "xhigh", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.7348, "efficacy_mean": 0.9195, "efficacy_min": 0.858, "weakest_eval": "CritPt", "honesty": { "net_per_100": 35.38, "accuracy_pct": 59.5, "answers_when_unsure_pct": 59.5473, "grade": "B", "confident_errors_per_100": 24.1167, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0404 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 93.7, "share_of_best": 0.973, "cost_per_attempt": 0.0635, "cost_per_solved": 0.0678 }, "HLE": { "score": 54.4, "share_of_best": 0.92, "cost_per_attempt": 0.5171, "cost_per_solved": 0.9506 }, "CritPt": { "score": 27.7, "share_of_best": 0.858, "cost_per_attempt": 2.4581, "cost_per_solved": 8.874 }, "SciCode": { "score": 55.0, "share_of_best": 0.887, "cost_per_attempt": 0.0723, "cost_per_solved": 0.1315 }, "Terminal-Bench v2.1": { "score": 88.0, "share_of_best": 0.963, "cost_per_attempt": 1.8677, "cost_per_solved": 2.1224 }, "AA-LCR": { "score": 76.3, "share_of_best": 0.916, "cost_per_attempt": 0.7529, "cost_per_solved": 0.9868 } }, "match": { "api_ids": { "anthropic": [ "claude-opus-5" ], "bedrock": [ "us.anthropic.claude-opus-5" ] }, "confidence": "confirmed" } }, { "id": "openai/gpt-6-astra@max", "aliases": [], "rank": 14, "rank_by_cost": 18, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "OpenAI", "model": "GPT-6 Astra", "effort": "max", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.7591, "efficacy_mean": 0.9393, "efficacy_min": 0.872, "weakest_eval": "SciCode", "honesty": { "net_per_100": 43.4, "accuracy_pct": 62.6, "answers_when_unsure_pct": 51.3369, "grade": "B", "confident_errors_per_100": 19.2, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.2235 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 96.0606, "share_of_best": 0.998, "cost_per_attempt": 0.092359, "cost_per_solved": 0.0961 }, "HLE": { "score": 54.6803, "share_of_best": 0.925, "cost_per_attempt": 0.378077, "cost_per_solved": 0.6914 }, "CritPt": { "score": 31.7143, "share_of_best": 0.982, "cost_per_attempt": 1.166162, "cost_per_solved": 3.6771 }, "SciCode": { "score": 54.0509, "share_of_best": 0.872, "cost_per_attempt": 0.232259, "cost_per_solved": 0.4297 }, "Terminal-Bench v2.1": { "score": 88.3895, "share_of_best": 0.967, "cost_per_attempt": 1.199745, "cost_per_solved": 1.3573 }, "AA-LCR": { "score": 74.3333, "share_of_best": 0.892, "cost_per_attempt": 0.997306, "cost_per_solved": 1.3417 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "anthropic/claude-opus-5@max", "aliases": [], "rank": 15, "rank_by_cost": 19, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "Anthropic", "model": "Claude Opus 5", "effort": "max", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.9409, "efficacy_mean": 0.9299, "efficacy_min": 0.898, "weakest_eval": "SciCode", "honesty": { "net_per_100": 37.07, "accuracy_pct": 60.8667, "answers_when_unsure_pct": 60.8177, "grade": "B", "confident_errors_per_100": 23.8, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0699 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 93.2, "share_of_best": 0.968, "cost_per_attempt": 0.0921, "cost_per_solved": 0.0988 }, "HLE": { "score": 54.9, "share_of_best": 0.929, "cost_per_attempt": 0.6736, "cost_per_solved": 1.227 }, "CritPt": { "score": 29.1, "share_of_best": 0.901, "cost_per_attempt": 2.8845, "cost_per_solved": 9.9124 }, "SciCode": { "score": 55.7, "share_of_best": 0.898, "cost_per_attempt": 0.1177, "cost_per_solved": 0.2113 }, "Terminal-Bench v2.1": { "score": 89.1, "share_of_best": 0.975, "cost_per_attempt": 2.4213, "cost_per_solved": 2.7175 }, "AA-LCR": { "score": 75.7, "share_of_best": 0.908, "cost_per_attempt": 0.7611, "cost_per_solved": 1.0054 } }, "match": { "api_ids": { "anthropic": [ "claude-opus-5" ], "bedrock": [ "us.anthropic.claude-opus-5" ] }, "confidence": "confirmed" } }, { "id": "anthropic/claude-fable-5.1@xhigh", "aliases": [], "rank": 16, "rank_by_cost": 20, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "Anthropic", "model": "Claude Fable 5.1", "effort": "xhigh", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 1.3242, "efficacy_mean": 0.9712, "efficacy_min": 0.936, "weakest_eval": "AA-LCR", "honesty": { "net_per_100": 42.38, "accuracy_pct": 66.2167, "answers_when_unsure_pct": 70.5476, "grade": "B", "confident_errors_per_100": 23.8333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.1824 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 93.4, "share_of_best": 0.97, "cost_per_attempt": 0.0939, "cost_per_solved": 0.1005 }, "HLE": { "score": 58.7, "share_of_best": 0.993, "cost_per_attempt": 1.0254, "cost_per_solved": 1.7468 }, "CritPt": { "score": 31.1, "share_of_best": 0.963, "cost_per_attempt": 4.5338, "cost_per_solved": 14.5781 }, "SciCode": { "score": 60.1, "share_of_best": 0.969, "cost_per_attempt": 0.241, "cost_per_solved": 0.401 }, "Terminal-Bench v2.1": { "score": 91.0, "share_of_best": 0.996, "cost_per_attempt": 2.4726, "cost_per_solved": 2.7171 }, "AA-LCR": { "score": 78.0, "share_of_best": 0.936, "cost_per_attempt": 1.5078, "cost_per_solved": 1.9331 } }, "match": { "api_ids": { "anthropic": [ "claude-fable-5-1" ] }, "confidence": "confirmed" } }, { "id": "anthropic/claude-fable-5.1@max", "aliases": [], "rank": 17, "rank_by_cost": 21, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "Anthropic", "model": "Claude Fable 5.1", "effort": "max", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 2.0422, "efficacy_mean": 0.9755, "efficacy_min": 0.92, "weakest_eval": "CritPt", "honesty": { "net_per_100": 43.45, "accuracy_pct": 67.2333, "answers_when_unsure_pct": 72.5839, "grade": "B", "confident_errors_per_100": 23.7833, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.6177 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 93.7, "share_of_best": 0.973, "cost_per_attempt": 0.1378, "cost_per_solved": 0.1471 }, "HLE": { "score": 59.1, "share_of_best": 1.0, "cost_per_attempt": 1.5859, "cost_per_solved": 2.6834 }, "CritPt": { "score": 29.7, "share_of_best": 0.92, "cost_per_attempt": 5.7138, "cost_per_solved": 19.2384 }, "SciCode": { "score": 62.0, "share_of_best": 1.0, "cost_per_attempt": 0.6585, "cost_per_solved": 1.0621 }, "Terminal-Bench v2.1": { "score": 91.4, "share_of_best": 1.0, "cost_per_attempt": 4.1512, "cost_per_solved": 4.5418 }, "AA-LCR": { "score": 80.0, "share_of_best": 0.96, "cost_per_attempt": 1.5845, "cost_per_solved": 1.9806 } }, "match": { "api_ids": { "anthropic": [ "claude-fable-5-1" ] }, "confidence": "confirmed" } }, { "id": "openai/gpt-5.6-sol@medium", "aliases": [], "rank": 18, "rank_by_cost": 1, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "OpenAI", "model": "GPT-5.6 Sol", "effort": "medium", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.137, "efficacy_mean": 0.855, "efficacy_min": 0.709, "weakest_eval": "CritPt", "honesty": { "net_per_100": 19.4, "accuracy_pct": 57.75, "answers_when_unsure_pct": 90.7692, "grade": "C", "confident_errors_per_100": 38.35, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0526 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 92.6, "share_of_best": 0.962, "cost_per_attempt": 0.0157, "cost_per_solved": 0.017 }, "HLE": { "score": 42.2, "share_of_best": 0.714, "cost_per_attempt": 0.043, "cost_per_solved": 0.1019 }, "CritPt": { "score": 22.9, "share_of_best": 0.709, "cost_per_attempt": 0.1259, "cost_per_solved": 0.5498 }, "SciCode": { "score": 56.5, "share_of_best": 0.911, "cost_per_attempt": 0.0285, "cost_per_solved": 0.0504 }, "Terminal-Bench v2.1": { "score": 86.1, "share_of_best": 0.942, "cost_per_attempt": 0.2302, "cost_per_solved": 0.2674 }, "AA-LCR": { "score": 74.3, "share_of_best": 0.892, "cost_per_attempt": 0.383, "cost_per_solved": 0.5155 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-sol" ], "zaun_gateway": [ "openai.gpt-5.6-sol" ] }, "confidence": "confirmed" } }, { "id": "openai/gpt-5.6-sol@high", "aliases": [], "rank": 19, "rank_by_cost": 3, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "OpenAI", "model": "GPT-5.6 Sol", "effort": "high", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.1891, "efficacy_mean": 0.8858, "efficacy_min": 0.778, "weakest_eval": "HLE", "honesty": { "net_per_100": 20.37, "accuracy_pct": 58.35, "answers_when_unsure_pct": 91.1965, "grade": "C", "confident_errors_per_100": 37.9833, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0884 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 92.8, "share_of_best": 0.964, "cost_per_attempt": 0.0243, "cost_per_solved": 0.0262 }, "HLE": { "score": 46.0, "share_of_best": 0.778, "cost_per_attempt": 0.0828, "cost_per_solved": 0.18 }, "CritPt": { "score": 25.7, "share_of_best": 0.796, "cost_per_attempt": 0.2453, "cost_per_solved": 0.9545 }, "SciCode": { "score": 56.9, "share_of_best": 0.918, "cost_per_attempt": 0.0353, "cost_per_solved": 0.062 }, "Terminal-Bench v2.1": { "score": 87.3, "share_of_best": 0.955, "cost_per_attempt": 0.2794, "cost_per_solved": 0.32 }, "AA-LCR": { "score": 75.3, "share_of_best": 0.904, "cost_per_attempt": 0.3859, "cost_per_solved": 0.5125 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-sol" ], "zaun_gateway": [ "openai.gpt-5.6-sol" ] }, "confidence": "confirmed" } }, { "id": "openai/gpt-5.6-sol@xhigh", "aliases": [], "rank": 20, "rank_by_cost": 7, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "OpenAI", "model": "GPT-5.6 Sol", "effort": "xhigh", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.2611, "efficacy_mean": 0.9085, "efficacy_min": 0.8, "weakest_eval": "HLE", "honesty": { "net_per_100": 20.98, "accuracy_pct": 58.8167, "answers_when_unsure_pct": 91.8656, "grade": "C", "confident_errors_per_100": 37.8333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.1578 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 93.1, "share_of_best": 0.967, "cost_per_attempt": 0.037, "cost_per_solved": 0.0397 }, "HLE": { "score": 47.3, "share_of_best": 0.8, "cost_per_attempt": 0.1421, "cost_per_solved": 0.3004 }, "CritPt": { "score": 28.6, "share_of_best": 0.885, "cost_per_attempt": 0.4367, "cost_per_solved": 1.5269 }, "SciCode": { "score": 56.0, "share_of_best": 0.903, "cost_per_attempt": 0.0449, "cost_per_solved": 0.0802 }, "Terminal-Bench v2.1": { "score": 89.5, "share_of_best": 0.979, "cost_per_attempt": 0.3821, "cost_per_solved": 0.4269 }, "AA-LCR": { "score": 76.3, "share_of_best": 0.916, "cost_per_attempt": 0.3876, "cost_per_solved": 0.508 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-sol" ], "zaun_gateway": [ "openai.gpt-5.6-sol" ] }, "confidence": "confirmed" } }, { "id": "openai/gpt-5.6-sol@max", "aliases": [], "rank": 21, "rank_by_cost": 11, "reason_code": "QUALIFIES", "disposition": "APPROVED", "vendor": "OpenAI", "model": "GPT-5.6 Sol", "effort": "max", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.4147, "efficacy_mean": 0.9359, "efficacy_min": 0.838, "weakest_eval": "HLE", "honesty": { "net_per_100": 21.97, "accuracy_pct": 59.4, "answers_when_unsure_pct": 92.2003, "grade": "C", "confident_errors_per_100": 37.4333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.3887 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 94.1, "share_of_best": 0.978, "cost_per_attempt": 0.0732, "cost_per_solved": 0.0778 }, "HLE": { "score": 49.5, "share_of_best": 0.838, "cost_per_attempt": 0.2684, "cost_per_solved": 0.5422 }, "CritPt": { "score": 32.3, "share_of_best": 1.0, "cost_per_attempt": 0.8591, "cost_per_solved": 2.6598 }, "SciCode": { "score": 56.1, "share_of_best": 0.905, "cost_per_attempt": 0.0797, "cost_per_solved": 0.1421 }, "Terminal-Bench v2.1": { "score": 88.0, "share_of_best": 0.963, "cost_per_attempt": 0.5435, "cost_per_solved": 0.6176 }, "AA-LCR": { "score": 77.7, "share_of_best": 0.932, "cost_per_attempt": 0.4016, "cost_per_solved": 0.5169 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-sol" ], "zaun_gateway": [ "openai.gpt-5.6-sol" ] }, "confidence": "confirmed" } }, { "id": "openai/gpt-5.6-luna@low", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "OpenAI", "model": "GPT-5.6 Luna", "effort": "low", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.0131, "efficacy_mean": 0.5461, "efficacy_min": 0.08, "weakest_eval": "CritPt", "honesty": { "net_per_100": -14.65, "accuracy_pct": 39.6167, "answers_when_unsure_pct": 89.8703, "grade": null, "confident_errors_per_100": 54.2667, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 83.5, "share_of_best": 0.867, "cost_per_attempt": 0.0008, "cost_per_solved": 0.001 }, "HLE": { "score": 19.8, "share_of_best": 0.335, "cost_per_attempt": 0.0014, "cost_per_solved": 0.0071 }, "CritPt": { "score": 2.6, "share_of_best": 0.08, "cost_per_attempt": 0.0037, "cost_per_solved": 0.1423 }, "SciCode": { "score": 45.6, "share_of_best": 0.735, "cost_per_attempt": 0.0015, "cost_per_solved": 0.0033 }, "Terminal-Bench v2.1": { "score": 43.4, "share_of_best": 0.475, "cost_per_attempt": 0.0239, "cost_per_solved": 0.0551 }, "AA-LCR": { "score": 65.3, "share_of_best": 0.784, "cost_per_attempt": 0.0191, "cost_per_solved": 0.0292 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-luna" ], "zaun_gateway": [ "openai.gpt-5.6-luna" ] }, "confidence": "confirmed" } }, { "id": "openai/gpt-5.6-luna@medium", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "OpenAI", "model": "GPT-5.6 Luna", "effort": "medium", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.0149, "efficacy_mean": 0.6109, "efficacy_min": 0.152, "weakest_eval": "CritPt", "honesty": { "net_per_100": -13.18, "accuracy_pct": 40.7, "answers_when_unsure_pct": 90.8657, "grade": null, "confident_errors_per_100": 53.8833, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 85.9, "share_of_best": 0.892, "cost_per_attempt": 0.0013, "cost_per_solved": 0.0015 }, "HLE": { "score": 25.8, "share_of_best": 0.437, "cost_per_attempt": 0.0029, "cost_per_solved": 0.0112 }, "CritPt": { "score": 4.9, "share_of_best": 0.152, "cost_per_attempt": 0.0066, "cost_per_solved": 0.1347 }, "SciCode": { "score": 45.8, "share_of_best": 0.739, "cost_per_attempt": 0.0018, "cost_per_solved": 0.0039 }, "Terminal-Bench v2.1": { "score": 53.2, "share_of_best": 0.582, "cost_per_attempt": 0.0243, "cost_per_solved": 0.0457 }, "AA-LCR": { "score": 72.0, "share_of_best": 0.864, "cost_per_attempt": 0.0193, "cost_per_solved": 0.0268 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-luna" ], "zaun_gateway": [ "openai.gpt-5.6-luna" ] }, "confidence": "confirmed" } }, { "id": "openai/gpt-5.6-luna@non-reasoning", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "OpenAI", "model": "GPT-5.6 Luna", "effort": "non-reasoning", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.0153, "efficacy_mean": 0.3893, "efficacy_min": 0.009, "weakest_eval": "CritPt", "honesty": { "net_per_100": -24.95, "accuracy_pct": 28.6, "answers_when_unsure_pct": 75.0, "grade": null, "confident_errors_per_100": 53.55, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 64.5, "share_of_best": 0.67, "cost_per_attempt": 0.0002, "cost_per_solved": 0.0003 }, "HLE": { "score": 7.2, "share_of_best": 0.122, "cost_per_attempt": 0.0003, "cost_per_solved": 0.0042 }, "CritPt": { "score": 0.3, "share_of_best": 0.009, "cost_per_attempt": 0.0023, "cost_per_solved": 0.7667 }, "SciCode": { "score": 39.9, "share_of_best": 0.644, "cost_per_attempt": 0.001, "cost_per_solved": 0.0025 }, "Terminal-Bench v2.1": { "score": 39.0, "share_of_best": 0.427, "cost_per_attempt": 0.0405, "cost_per_solved": 0.1038 }, "AA-LCR": { "score": 38.7, "share_of_best": 0.464, "cost_per_attempt": 0.019, "cost_per_solved": 0.0491 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-luna" ], "zaun_gateway": [ "openai.gpt-5.6-luna" ] }, "confidence": "confirmed" } }, { "id": "openai/gpt-5.6-luna@high", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "OpenAI", "model": "GPT-5.6 Luna", "effort": "high", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.021, "efficacy_mean": 0.7457, "efficacy_min": 0.514, "weakest_eval": "CritPt", "honesty": { "net_per_100": -12.0, "accuracy_pct": 41.7833, "answers_when_unsure_pct": 92.3848, "grade": null, "confident_errors_per_100": 53.7833, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 89.2, "share_of_best": 0.927, "cost_per_attempt": 0.0034, "cost_per_solved": 0.0038 }, "HLE": { "score": 33.4, "share_of_best": 0.565, "cost_per_attempt": 0.0095, "cost_per_solved": 0.0284 }, "CritPt": { "score": 16.6, "share_of_best": 0.514, "cost_per_attempt": 0.019, "cost_per_solved": 0.1145 }, "SciCode": { "score": 50.7, "share_of_best": 0.818, "cost_per_attempt": 0.0029, "cost_per_solved": 0.0057 }, "Terminal-Bench v2.1": { "score": 69.7, "share_of_best": 0.763, "cost_per_attempt": 0.0321, "cost_per_solved": 0.0461 }, "AA-LCR": { "score": 74.0, "share_of_best": 0.888, "cost_per_attempt": 0.0196, "cost_per_solved": 0.0265 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-luna" ], "zaun_gateway": [ "openai.gpt-5.6-luna" ] }, "confidence": "confirmed" } }, { "id": "openai/gpt-5.6-luna@xhigh", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "OpenAI", "model": "GPT-5.6 Luna", "effort": "xhigh", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.0296, "efficacy_mean": 0.7887, "efficacy_min": 0.626, "weakest_eval": "HLE", "honesty": { "net_per_100": -10.77, "accuracy_pct": 42.45, "answers_when_unsure_pct": 92.4703, "grade": null, "confident_errors_per_100": 53.2167, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 89.5, "share_of_best": 0.93, "cost_per_attempt": 0.0059, "cost_per_solved": 0.0066 }, "HLE": { "score": 37.0, "share_of_best": 0.626, "cost_per_attempt": 0.0167, "cost_per_solved": 0.0451 }, "CritPt": { "score": 20.6, "share_of_best": 0.638, "cost_per_attempt": 0.0395, "cost_per_solved": 0.1917 }, "SciCode": { "score": 50.0, "share_of_best": 0.806, "cost_per_attempt": 0.0045, "cost_per_solved": 0.009 }, "Terminal-Bench v2.1": { "score": 77.9, "share_of_best": 0.852, "cost_per_attempt": 0.0374, "cost_per_solved": 0.048 }, "AA-LCR": { "score": 73.3, "share_of_best": 0.88, "cost_per_attempt": 0.0199, "cost_per_solved": 0.0271 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-luna" ], "zaun_gateway": [ "openai.gpt-5.6-luna" ] }, "confidence": "confirmed" } }, { "id": "zai/glm-5.3-flash@default", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "Z.ai", "model": "GLM-5.3-Flash", "effort": "default", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": 0.0417, "efficacy_mean": 0.7834, "efficacy_min": 0.478, "weakest_eval": "CritPt", "honesty": { "net_per_100": 7.4667, "accuracy_pct": 27.5, "answers_when_unsure_pct": 27.6322, "grade": null, "confident_errors_per_100": 20.0333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.008 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 91.2121, "share_of_best": 0.948, "cost_per_attempt": 0.005501, "cost_per_solved": 0.006 }, "HLE": { "score": 39.8517, "share_of_best": 0.674, "cost_per_attempt": 0.021655, "cost_per_solved": 0.0543 }, "CritPt": { "score": 15.4286, "share_of_best": 0.478, "cost_per_attempt": 0.041999, "cost_per_solved": 0.2722 }, "SciCode": { "score": 46.0648, "share_of_best": 0.743, "cost_per_attempt": 0.013986, "cost_per_solved": 0.0304 }, "Terminal-Bench v2.1": { "score": 84.2697, "share_of_best": 0.922, "cost_per_attempt": 0.076216, "cost_per_solved": 0.0904 }, "AA-LCR": { "score": 78.0, "share_of_best": 0.936, "cost_per_attempt": 0.016855, "cost_per_solved": 0.0216 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "openai/gpt-5.6-luna@max", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "OpenAI", "model": "GPT-5.6 Luna", "effort": "max", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.0444, "efficacy_mean": 0.8207, "efficacy_min": 0.638, "weakest_eval": "CritPt", "honesty": { "net_per_100": -10.28, "accuracy_pct": 42.7333, "answers_when_unsure_pct": 92.5786, "grade": null, "confident_errors_per_100": 53.0167, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 91.1, "share_of_best": 0.946, "cost_per_attempt": 0.0099, "cost_per_solved": 0.0109 }, "HLE": { "score": 39.5, "share_of_best": 0.668, "cost_per_attempt": 0.0293, "cost_per_solved": 0.0742 }, "CritPt": { "score": 20.6, "share_of_best": 0.638, "cost_per_attempt": 0.0633, "cost_per_solved": 0.3073 }, "SciCode": { "score": 52.5, "share_of_best": 0.847, "cost_per_attempt": 0.0093, "cost_per_solved": 0.0177 }, "Terminal-Bench v2.1": { "score": 80.9, "share_of_best": 0.885, "cost_per_attempt": 0.0529, "cost_per_solved": 0.0654 }, "AA-LCR": { "score": 78.3, "share_of_best": 0.94, "cost_per_attempt": 0.0209, "cost_per_solved": 0.0267 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-luna" ], "zaun_gateway": [ "openai.gpt-5.6-luna" ] }, "confidence": "confirmed" } }, { "id": "google/gemini-3.7-flash@low", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "Google", "model": "Gemini 3.7 Flash", "effort": "low", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.0462, "efficacy_mean": 0.7306, "efficacy_min": 0.176, "weakest_eval": "CritPt", "honesty": { "net_per_100": 22.13, "accuracy_pct": 53.5667, "answers_when_unsure_pct": 67.6956, "grade": null, "confident_errors_per_100": 31.4333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.005 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 90.1, "share_of_best": 0.936, "cost_per_attempt": 0.0036, "cost_per_solved": 0.004 }, "HLE": { "score": 35.1, "share_of_best": 0.594, "cost_per_attempt": 0.0059, "cost_per_solved": 0.0168 }, "CritPt": { "score": 5.7, "share_of_best": 0.176, "cost_per_attempt": 0.023, "cost_per_solved": 0.4035 }, "SciCode": { "score": 53.6, "share_of_best": 0.865, "cost_per_attempt": 0.0045, "cost_per_solved": 0.0084 }, "Terminal-Bench v2.1": { "score": 79.8, "share_of_best": 0.873, "cost_per_attempt": 0.3184, "cost_per_solved": 0.399 }, "AA-LCR": { "score": 78.3, "share_of_best": 0.94, "cost_per_attempt": 0.0836, "cost_per_solved": 0.1068 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "alibaba/qwen3.8-flash-next@default", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "Alibaba", "model": "Qwen3.8-Flash-Next", "effort": "default", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": 0.0527, "efficacy_mean": 0.7617, "efficacy_min": 0.345, "weakest_eval": "CritPt", "honesty": { "net_per_100": -9.7167, "accuracy_pct": 24.5, "answers_when_unsure_pct": 45.3201, "grade": null, "confident_errors_per_100": 34.2167, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 92.3232, "share_of_best": 0.959, "cost_per_attempt": 0.00832, "cost_per_solved": 0.009 }, "HLE": { "score": 38.0445, "share_of_best": 0.644, "cost_per_attempt": 0.024013, "cost_per_solved": 0.0631 }, "CritPt": { "score": 11.1429, "share_of_best": 0.345, "cost_per_attempt": 0.055688, "cost_per_solved": 0.4998 }, "SciCode": { "score": 46.875, "share_of_best": 0.756, "cost_per_attempt": 0.012633, "cost_per_solved": 0.027 }, "Terminal-Bench v2.1": { "score": 86.1423, "share_of_best": 0.942, "cost_per_attempt": 0.105955, "cost_per_solved": 0.123 }, "AA-LCR": { "score": 77.0, "share_of_best": 0.924, "cost_per_attempt": 0.017531, "cost_per_solved": 0.0228 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "google/gemini-3.8-flash@low", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "Google", "model": "Gemini 3.8 Flash", "effort": "low", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.0576, "efficacy_mean": 0.7394, "efficacy_min": 0.124, "weakest_eval": "CritPt", "honesty": { "net_per_100": 21.35, "accuracy_pct": 52.2167, "answers_when_unsure_pct": 64.5971, "grade": null, "confident_errors_per_100": 30.8667, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0039 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 92.02, "share_of_best": 0.956, "cost_per_attempt": 0.004167, "cost_per_solved": 0.0045 }, "HLE": { "score": 37.07, "share_of_best": 0.627, "cost_per_attempt": 0.00825, "cost_per_solved": 0.0223 }, "CritPt": { "score": 4.0, "share_of_best": 0.124, "cost_per_attempt": 0.0295, "cost_per_solved": 0.7375 }, "SciCode": { "score": 54.28, "share_of_best": 0.875, "cost_per_attempt": 0.00425, "cost_per_solved": 0.0078 }, "Terminal-Bench v2.1": { "score": 83.15, "share_of_best": 0.91, "cost_per_attempt": 0.492688, "cost_per_solved": 0.5925 }, "AA-LCR": { "score": 78.67, "share_of_best": 0.944, "cost_per_attempt": 0.083667, "cost_per_solved": 0.1064 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "google/gemini-3.7-flash@medium", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "Google", "model": "Gemini 3.7 Flash", "effort": "medium", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.0611, "efficacy_mean": 0.7784, "efficacy_min": 0.291, "weakest_eval": "CritPt", "honesty": { "net_per_100": 23.7, "accuracy_pct": 54.0, "answers_when_unsure_pct": 65.8696, "grade": null, "confident_errors_per_100": 30.3, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0093 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 92.1, "share_of_best": 0.957, "cost_per_attempt": 0.0049, "cost_per_solved": 0.0053 }, "HLE": { "score": 39.0, "share_of_best": 0.66, "cost_per_attempt": 0.0097, "cost_per_solved": 0.0249 }, "CritPt": { "score": 9.4, "share_of_best": 0.291, "cost_per_attempt": 0.0392, "cost_per_solved": 0.417 }, "SciCode": { "score": 57.9, "share_of_best": 0.934, "cost_per_attempt": 0.01, "cost_per_solved": 0.0173 }, "Terminal-Bench v2.1": { "score": 78.3, "share_of_best": 0.857, "cost_per_attempt": 0.4048, "cost_per_solved": 0.517 }, "AA-LCR": { "score": 81.0, "share_of_best": 0.972, "cost_per_attempt": 0.0856, "cost_per_solved": 0.1057 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "deepseek/deepseek-v4-pro@high", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "DeepSeek", "model": "DeepSeek V4 Pro", "effort": "high", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": 0.0618, "efficacy_mean": 0.6845, "efficacy_min": 0.31, "weakest_eval": "CritPt", "honesty": { "net_per_100": -10.5667, "accuracy_pct": 41.4, "answers_when_unsure_pct": 88.6803, "grade": null, "confident_errors_per_100": 51.9667, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 90.5051, "share_of_best": 0.94, "cost_per_attempt": 0.009228, "cost_per_solved": 0.0102 }, "HLE": { "score": 35.2178, "share_of_best": 0.596, "cost_per_attempt": 0.028587, "cost_per_solved": 0.0812 }, "CritPt": { "score": 10.0, "share_of_best": 0.31, "cost_per_attempt": 0.06531, "cost_per_solved": 0.6531 }, "SciCode": { "score": 46.412, "share_of_best": 0.749, "cost_per_attempt": 0.004604, "cost_per_solved": 0.0099 }, "Terminal-Bench v2.1": { "score": 64.794, "share_of_best": 0.709, "cost_per_attempt": 0.105098, "cost_per_solved": 0.1622 }, "AA-LCR": { "score": 67.0, "share_of_best": 0.804, "cost_per_attempt": 0.043066, "cost_per_solved": 0.0643 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "openai/gpt-5.6-terra@non-reasoning", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "OpenAI", "model": "GPT-5.6 Terra", "effort": "non-reasoning", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.0837, "efficacy_mean": 0.504, "efficacy_min": 0.062, "weakest_eval": "CritPt", "honesty": { "net_per_100": -23.22, "accuracy_pct": 36.8167, "answers_when_unsure_pct": 95.0145, "grade": null, "confident_errors_per_100": 60.0333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 74.6, "share_of_best": 0.775, "cost_per_attempt": 0.0022, "cost_per_solved": 0.0029 }, "HLE": { "score": 11.4, "share_of_best": 0.193, "cost_per_attempt": 0.003, "cost_per_solved": 0.0263 }, "CritPt": { "score": 2.0, "share_of_best": 0.062, "cost_per_attempt": 0.0255, "cost_per_solved": 1.275 }, "SciCode": { "score": 44.6, "share_of_best": 0.719, "cost_per_attempt": 0.01, "cost_per_solved": 0.0224 }, "Terminal-Bench v2.1": { "score": 56.2, "share_of_best": 0.615, "cost_per_attempt": 0.2535, "cost_per_solved": 0.4511 }, "AA-LCR": { "score": 55.0, "share_of_best": 0.66, "cost_per_attempt": 0.1896, "cost_per_solved": 0.3447 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-terra" ], "zaun_gateway": [ "openai.gpt-5.6-terra" ] }, "confidence": "confirmed" } }, { "id": "openai/gpt-5.6-terra@low", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "OpenAI", "model": "GPT-5.6 Terra", "effort": "low", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.0839, "efficacy_mean": 0.6604, "efficacy_min": 0.291, "weakest_eval": "CritPt", "honesty": { "net_per_100": -6.83, "accuracy_pct": 43.7333, "answers_when_unsure_pct": 89.8697, "grade": null, "confident_errors_per_100": 50.5667, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 84.3, "share_of_best": 0.876, "cost_per_attempt": 0.006, "cost_per_solved": 0.0071 }, "HLE": { "score": 29.2, "share_of_best": 0.494, "cost_per_attempt": 0.0142, "cost_per_solved": 0.0486 }, "CritPt": { "score": 9.4, "share_of_best": 0.291, "cost_per_attempt": 0.0434, "cost_per_solved": 0.4617 }, "SciCode": { "score": 49.2, "share_of_best": 0.794, "cost_per_attempt": 0.0119, "cost_per_solved": 0.0242 }, "Terminal-Bench v2.1": { "score": 62.5, "share_of_best": 0.684, "cost_per_attempt": 0.2033, "cost_per_solved": 0.3253 }, "AA-LCR": { "score": 68.7, "share_of_best": 0.824, "cost_per_attempt": 0.1907, "cost_per_solved": 0.2776 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-terra" ], "zaun_gateway": [ "openai.gpt-5.6-terra" ] }, "confidence": "confirmed" } }, { "id": "xai/grok-4.6@low", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "xAI", "model": "Grok 4.6", "effort": "low", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.089, "efficacy_mean": 0.6842, "efficacy_min": 0.176, "weakest_eval": "CritPt", "honesty": { "net_per_100": 25.9, "accuracy_pct": 43.2833, "answers_when_unsure_pct": 30.6494, "grade": null, "confident_errors_per_100": 17.3833, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.01 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 87.9, "share_of_best": 0.913, "cost_per_attempt": 0.0075, "cost_per_solved": 0.0085 }, "HLE": { "score": 27.6, "share_of_best": 0.467, "cost_per_attempt": 0.0132, "cost_per_solved": 0.0478 }, "CritPt": { "score": 5.7, "share_of_best": 0.176, "cost_per_attempt": 0.0391, "cost_per_solved": 0.686 }, "SciCode": { "score": 48.4, "share_of_best": 0.781, "cost_per_attempt": 0.0098, "cost_per_solved": 0.0202 }, "Terminal-Bench v2.1": { "score": 75.3, "share_of_best": 0.824, "cost_per_attempt": 0.2647, "cost_per_solved": 0.3515 }, "AA-LCR": { "score": 78.7, "share_of_best": 0.944, "cost_per_attempt": 0.1962, "cost_per_solved": 0.2493 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "deepseek/deepseek-v4-pro@max", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "DeepSeek", "model": "DeepSeek V4 Pro", "effort": "max", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": 0.091, "efficacy_mean": 0.7171, "efficacy_min": 0.398, "weakest_eval": "CritPt", "honesty": { "net_per_100": -10.7333, "accuracy_pct": 42.95, "answers_when_unsure_pct": 94.0987, "grade": null, "confident_errors_per_100": 53.6833, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 88.7879, "share_of_best": 0.922, "cost_per_attempt": 0.025241, "cost_per_solved": 0.0284 }, "HLE": { "score": 37.5348, "share_of_best": 0.635, "cost_per_attempt": 0.046351, "cost_per_solved": 0.1235 }, "CritPt": { "score": 12.8571, "share_of_best": 0.398, "cost_per_attempt": 0.092888, "cost_per_solved": 0.7225 }, "SciCode": { "score": 50.0, "share_of_best": 0.806, "cost_per_attempt": 0.011361, "cost_per_solved": 0.0227 }, "Terminal-Bench v2.1": { "score": 64.0449, "share_of_best": 0.701, "cost_per_attempt": 0.09488, "cost_per_solved": 0.1481 }, "AA-LCR": { "score": 70.0, "share_of_best": 0.84, "cost_per_attempt": 0.046411, "cost_per_solved": 0.0663 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "openai/gpt-5.6-terra@medium", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "OpenAI", "model": "GPT-5.6 Terra", "effort": "medium", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.0937, "efficacy_mean": 0.7407, "efficacy_min": 0.539, "weakest_eval": "CritPt", "honesty": { "net_per_100": -5.13, "accuracy_pct": 44.6, "answers_when_unsure_pct": 89.7714, "grade": null, "confident_errors_per_100": 49.7333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 87.2, "share_of_best": 0.906, "cost_per_attempt": 0.0093, "cost_per_solved": 0.0107 }, "HLE": { "score": 33.3, "share_of_best": 0.563, "cost_per_attempt": 0.0255, "cost_per_solved": 0.0766 }, "CritPt": { "score": 17.4, "share_of_best": 0.539, "cost_per_attempt": 0.0696, "cost_per_solved": 0.4 }, "SciCode": { "score": 49.7, "share_of_best": 0.802, "cost_per_attempt": 0.015, "cost_per_solved": 0.0302 }, "Terminal-Bench v2.1": { "score": 72.3, "share_of_best": 0.791, "cost_per_attempt": 0.1824, "cost_per_solved": 0.2523 }, "AA-LCR": { "score": 70.3, "share_of_best": 0.844, "cost_per_attempt": 0.1914, "cost_per_solved": 0.2723 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-terra" ], "zaun_gateway": [ "openai.gpt-5.6-terra" ] }, "confidence": "confirmed" } }, { "id": "deepseek/deepseek-v4-flash-0731@max", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "DeepSeek", "model": "DeepSeek V4 Flash 0731", "effort": "max", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": 0.1004, "efficacy_mean": 0.7776, "efficacy_min": 0.513, "weakest_eval": "CritPt", "honesty": { "net_per_100": -14.2833, "accuracy_pct": 40.3833, "answers_when_unsure_pct": 91.697, "grade": null, "confident_errors_per_100": 54.6667, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 90.8081, "share_of_best": 0.943, "cost_per_attempt": 0.012492, "cost_per_solved": 0.0138 }, "HLE": { "score": 38.5542, "share_of_best": 0.652, "cost_per_attempt": 0.04288, "cost_per_solved": 0.1112 }, "CritPt": { "score": 16.5714, "share_of_best": 0.513, "cost_per_attempt": 0.114523, "cost_per_solved": 0.6911 }, "SciCode": { "score": 49.8843, "share_of_best": 0.805, "cost_per_attempt": 0.04223, "cost_per_solved": 0.0847 }, "Terminal-Bench v2.1": { "score": 78.6517, "share_of_best": 0.861, "cost_per_attempt": 0.140712, "cost_per_solved": 0.1789 }, "AA-LCR": { "score": 74.3333, "share_of_best": 0.892, "cost_per_attempt": 0.047567, "cost_per_solved": 0.064 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "deepseek/deepseek-v4-flash-vision@max", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "DeepSeek", "model": "DeepSeek V4 Flash Vision", "effort": "max", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.1051, "efficacy_mean": 0.728, "efficacy_min": 0.336, "weakest_eval": "CritPt", "honesty": { "net_per_100": -17.6333, "accuracy_pct": 38.5667, "answers_when_unsure_pct": 91.4813, "grade": null, "confident_errors_per_100": 56.2, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 91.3131, "share_of_best": 0.949, "cost_per_attempt": 0.016851, "cost_per_solved": 0.0185 }, "HLE": { "score": 34.4764, "share_of_best": 0.583, "cost_per_attempt": 0.037875, "cost_per_solved": 0.1099 }, "CritPt": { "score": 10.8571, "share_of_best": 0.336, "cost_per_attempt": 0.110519, "cost_per_solved": 1.0179 }, "SciCode": { "score": 46.6435, "share_of_best": 0.752, "cost_per_attempt": 0.03439, "cost_per_solved": 0.0737 }, "Terminal-Bench v2.1": { "score": 74.1573, "share_of_best": 0.811, "cost_per_attempt": 0.107886, "cost_per_solved": 0.1455 }, "AA-LCR": { "score": 78.0, "share_of_best": 0.936, "cost_per_attempt": 0.04755, "cost_per_solved": 0.061 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "google/gemini-3.8-flash@medium", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "Google", "model": "Gemini 3.8 Flash", "effort": "medium", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.1057, "efficacy_mean": 0.8074, "efficacy_min": 0.38, "weakest_eval": "CritPt", "honesty": { "net_per_100": 28.58, "accuracy_pct": 52.9833, "answers_when_unsure_pct": 51.8965, "grade": null, "confident_errors_per_100": 24.4, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0125 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 93.54, "share_of_best": 0.972, "cost_per_attempt": 0.007, "cost_per_solved": 0.0075 }, "HLE": { "score": 42.12, "share_of_best": 0.713, "cost_per_attempt": 0.024833, "cost_per_solved": 0.059 }, "CritPt": { "score": 12.29, "share_of_best": 0.38, "cost_per_attempt": 0.101667, "cost_per_solved": 0.8272 }, "SciCode": { "score": 54.4, "share_of_best": 0.877, "cost_per_attempt": 0.029375, "cost_per_solved": 0.054 }, "Terminal-Bench v2.1": { "score": 83.9, "share_of_best": 0.918, "cost_per_attempt": 0.541937, "cost_per_solved": 0.6459 }, "AA-LCR": { "score": 82.0, "share_of_best": 0.984, "cost_per_attempt": 0.089667, "cost_per_solved": 0.1094 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "openai/gpt-5.6-sol@non-reasoning", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "HONESTY_INCONCLUSIVE", "disposition": "REVIEW", "vendor": "OpenAI", "model": "GPT-5.6 Sol", "effort": "non-reasoning", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.1061, "efficacy_mean": 0.5861, "efficacy_min": 0.158, "weakest_eval": "CritPt", "honesty": { "net_per_100": 1.1, "accuracy_pct": 48.7, "answers_when_unsure_pct": 92.7875, "grade": null, "confident_errors_per_100": 47.6, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "inconclusive", "cost_per_net_fact_usd": 0.0545 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 79.0, "share_of_best": 0.821, "cost_per_attempt": 0.0036, "cost_per_solved": 0.0046 }, "HLE": { "score": 16.7, "share_of_best": 0.283, "cost_per_attempt": 0.0055, "cost_per_solved": 0.0329 }, "CritPt": { "score": 5.1, "share_of_best": 0.158, "cost_per_attempt": 0.0471, "cost_per_solved": 0.9235 }, "SciCode": { "score": 47.1, "share_of_best": 0.76, "cost_per_attempt": 0.0189, "cost_per_solved": 0.0401 }, "Terminal-Bench v2.1": { "score": 74.2, "share_of_best": 0.812, "cost_per_attempt": 0.2871, "cost_per_solved": 0.3869 }, "AA-LCR": { "score": 57.0, "share_of_best": 0.684, "cost_per_attempt": 0.3789, "cost_per_solved": 0.6647 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-sol" ], "zaun_gateway": [ "openai.gpt-5.6-sol" ] }, "confidence": "confirmed" } }, { "id": "openai/gpt-5.6-sol@low", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "OpenAI", "model": "GPT-5.6 Sol", "effort": "low", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.108, "efficacy_mean": 0.7784, "efficacy_min": 0.461, "weakest_eval": "CritPt", "honesty": { "net_per_100": 18.87, "accuracy_pct": 57.1667, "answers_when_unsure_pct": 89.4163, "grade": null, "confident_errors_per_100": 38.3, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0313 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 89.8, "share_of_best": 0.933, "cost_per_attempt": 0.0101, "cost_per_solved": 0.0112 }, "HLE": { "score": 39.4, "share_of_best": 0.667, "cost_per_attempt": 0.0224, "cost_per_solved": 0.0569 }, "CritPt": { "score": 14.9, "share_of_best": 0.461, "cost_per_attempt": 0.0751, "cost_per_solved": 0.504 }, "SciCode": { "score": 55.4, "share_of_best": 0.894, "cost_per_attempt": 0.0241, "cost_per_solved": 0.0435 }, "Terminal-Bench v2.1": { "score": 76.8, "share_of_best": 0.84, "cost_per_attempt": 0.1665, "cost_per_solved": 0.2168 }, "AA-LCR": { "score": 73.0, "share_of_best": 0.876, "cost_per_attempt": 0.3817, "cost_per_solved": 0.5229 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-sol" ], "zaun_gateway": [ "openai.gpt-5.6-sol" ] }, "confidence": "confirmed" } }, { "id": "minimax/minimax-m3@default", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "HONESTY_INCONCLUSIVE", "disposition": "REVIEW", "vendor": "MiniMax", "model": "MiniMax-M3", "effort": "default", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": 0.1089, "efficacy_mean": 0.6914, "efficacy_min": 0.115, "weakest_eval": "CritPt", "honesty": { "net_per_100": 1.35, "accuracy_pct": 16.7, "answers_when_unsure_pct": 18.4274, "grade": null, "confident_errors_per_100": 15.35, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "inconclusive", "cost_per_net_fact_usd": 0.259 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 92.9293, "share_of_best": 0.965, "cost_per_attempt": 0.009888, "cost_per_solved": 0.0106 }, "HLE": { "score": 38.9713, "share_of_best": 0.659, "cost_per_attempt": 0.025921, "cost_per_solved": 0.0665 }, "CritPt": { "score": 3.7143, "share_of_best": 0.115, "cost_per_attempt": 0.095545, "cost_per_solved": 2.5724 }, "SciCode": { "score": 45.3704, "share_of_best": 0.732, "cost_per_attempt": 0.025944, "cost_per_solved": 0.0572 }, "Terminal-Bench v2.1": { "score": 65.1685, "share_of_best": 0.713, "cost_per_attempt": 0.281402, "cost_per_solved": 0.4318 }, "AA-LCR": { "score": 80.3333, "share_of_best": 0.964, "cost_per_attempt": 0.029755, "cost_per_solved": 0.037 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "google/gemini-3.7-flash@high", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "Google", "model": "Gemini 3.7 Flash", "effort": "high", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.1104, "efficacy_mean": 0.8416, "efficacy_min": 0.443, "weakest_eval": "CritPt", "honesty": { "net_per_100": 26.48, "accuracy_pct": 55.3167, "answers_when_unsure_pct": 64.5282, "grade": null, "confident_errors_per_100": 28.8333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0147 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 94.5, "share_of_best": 0.982, "cost_per_attempt": 0.01, "cost_per_solved": 0.0106 }, "HLE": { "score": 47.9, "share_of_best": 0.81, "cost_per_attempt": 0.0368, "cost_per_solved": 0.0768 }, "CritPt": { "score": 14.3, "share_of_best": 0.443, "cost_per_attempt": 0.1148, "cost_per_solved": 0.8028 }, "SciCode": { "score": 56.8, "share_of_best": 0.916, "cost_per_attempt": 0.0245, "cost_per_solved": 0.0431 }, "Terminal-Bench v2.1": { "score": 85.8, "share_of_best": 0.939, "cost_per_attempt": 0.494, "cost_per_solved": 0.5758 }, "AA-LCR": { "score": 80.0, "share_of_best": 0.96, "cost_per_attempt": 0.0893, "cost_per_solved": 0.1116 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "nvidia/nemotron-3-super-120b-a12b@reasoning", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "NVIDIA", "model": "Nemotron 3 Super 120B A12B", "effort": "reasoning", "weights": "open", "price_basis": "median-across-providers", "cost_per_solve_usd": 0.1114, "efficacy_mean": 0.501, "efficacy_min": 0.097, "weakest_eval": "CritPt", "honesty": { "net_per_100": -41.5, "accuracy_pct": 24.3167, "answers_when_unsure_pct": 86.9632, "grade": null, "confident_errors_per_100": 65.8167, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 80.0, "share_of_best": 0.831, "cost_per_attempt": 0.018628, "cost_per_solved": 0.0233 }, "HLE": { "score": 20.76, "share_of_best": 0.351, "cost_per_attempt": 0.029018, "cost_per_solved": 0.1398 }, "CritPt": { "score": 3.1429, "share_of_best": 0.097, "cost_per_attempt": 0.048464, "cost_per_solved": 1.542 }, "SciCode": { "score": 35.9954, "share_of_best": 0.581, "cost_per_attempt": 0.002208, "cost_per_solved": 0.0061 }, "Terminal-Bench v2.1": { "score": 38.5768, "share_of_best": 0.422, "cost_per_attempt": 0.555067, "cost_per_solved": 1.4389 }, "AA-LCR": { "score": 60.3333, "share_of_best": 0.724, "cost_per_attempt": 0.026066, "cost_per_solved": 0.0432 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "openai/gpt-5.6-terra@high", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "OpenAI", "model": "GPT-5.6 Terra", "effort": "high", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.1493, "efficacy_mean": 0.8012, "efficacy_min": 0.651, "weakest_eval": "HLE", "honesty": { "net_per_100": -3.47, "accuracy_pct": 45.4833, "answers_when_unsure_pct": 89.7891, "grade": null, "confident_errors_per_100": 48.95, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 89.6, "share_of_best": 0.931, "cost_per_attempt": 0.0184, "cost_per_solved": 0.0205 }, "HLE": { "score": 38.5, "share_of_best": 0.651, "cost_per_attempt": 0.0619, "cost_per_solved": 0.1608 }, "CritPt": { "score": 22.9, "share_of_best": 0.709, "cost_per_attempt": 0.1741, "cost_per_solved": 0.7603 }, "SciCode": { "score": 50.1, "share_of_best": 0.808, "cost_per_attempt": 0.0232, "cost_per_solved": 0.0463 }, "Terminal-Bench v2.1": { "score": 75.7, "share_of_best": 0.828, "cost_per_attempt": 0.2744, "cost_per_solved": 0.3625 }, "AA-LCR": { "score": 73.3, "share_of_best": 0.88, "cost_per_attempt": 0.1929, "cost_per_solved": 0.2632 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-terra" ], "zaun_gateway": [ "openai.gpt-5.6-terra" ] }, "confidence": "confirmed" } }, { "id": "moonshot/kimi-k3@low", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "Moonshot", "model": "Kimi K3", "effort": "low", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": 0.1505, "efficacy_mean": 0.6743, "efficacy_min": 0.097, "weakest_eval": "CritPt", "honesty": { "net_per_100": 3.9167, "accuracy_pct": 45.75, "answers_when_unsure_pct": 77.1121, "grade": null, "confident_errors_per_100": 41.8333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.1285 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 84.2424, "share_of_best": 0.875, "cost_per_attempt": 0.012263, "cost_per_solved": 0.0146 }, "HLE": { "score": 24.9768, "share_of_best": 0.423, "cost_per_attempt": 0.024208, "cost_per_solved": 0.0969 }, "CritPt": { "score": 3.1429, "share_of_best": 0.097, "cost_per_attempt": 0.050265, "cost_per_solved": 1.5993 }, "SciCode": { "score": 51.1574, "share_of_best": 0.825, "cost_per_attempt": 0.01836, "cost_per_solved": 0.0359 }, "Terminal-Bench v2.1": { "score": 82.397, "share_of_best": 0.901, "cost_per_attempt": 0.310731, "cost_per_solved": 0.3771 }, "AA-LCR": { "score": 77.0, "share_of_best": 0.924, "cost_per_attempt": 0.293106, "cost_per_solved": 0.3807 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "openai/gpt-6-astra@non-reasoning", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "OpenAI", "model": "GPT-6 Astra", "effort": "non-reasoning", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.1732, "efficacy_mean": 0.7955, "efficacy_min": 0.61, "weakest_eval": "CritPt", "honesty": { "net_per_100": 26.55, "accuracy_pct": 55.6, "answers_when_unsure_pct": 65.4279, "grade": null, "confident_errors_per_100": 29.05, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0077 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 89.4949, "share_of_best": 0.93, "cost_per_attempt": 0.011033, "cost_per_solved": 0.0123 }, "HLE": { "score": 37.1177, "share_of_best": 0.628, "cost_per_attempt": 0.018966, "cost_per_solved": 0.0511 }, "CritPt": { "score": 19.7143, "share_of_best": 0.61, "cost_per_attempt": 0.132848, "cost_per_solved": 0.6739 }, "SciCode": { "score": 50.463, "share_of_best": 0.814, "cost_per_attempt": 0.046258, "cost_per_solved": 0.0917 }, "Terminal-Bench v2.1": { "score": 89.1386, "share_of_best": 0.975, "cost_per_attempt": 0.443188, "cost_per_solved": 0.4972 }, "AA-LCR": { "score": 68.0, "share_of_best": 0.816, "cost_per_attempt": 0.947588, "cost_per_solved": 1.3935 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "alibaba/qwen3.8-27b@non-reasoning", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "Alibaba", "model": "Qwen3.8 27B", "effort": "non-reasoning", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": 0.1788, "efficacy_mean": 0.4887, "efficacy_min": 0.009, "weakest_eval": "CritPt", "honesty": { "net_per_100": -7.95, "accuracy_pct": 8.5833, "answers_when_unsure_pct": 18.0857, "grade": null, "confident_errors_per_100": 16.5333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 81.8182, "share_of_best": 0.85, "cost_per_attempt": 0.011345, "cost_per_solved": 0.0139 }, "HLE": { "score": 12.1409, "share_of_best": 0.205, "cost_per_attempt": 0.012791, "cost_per_solved": 0.1054 }, "CritPt": { "score": 0.2857, "share_of_best": 0.009, "cost_per_attempt": 0.032444, "cost_per_solved": 11.356 }, "SciCode": { "score": 35.6481, "share_of_best": 0.575, "cost_per_attempt": 0.004821, "cost_per_solved": 0.0135 }, "Terminal-Bench v2.1": { "score": 49.0637, "share_of_best": 0.537, "cost_per_attempt": 0.817051, "cost_per_solved": 1.6653 }, "AA-LCR": { "score": 63.0, "share_of_best": 0.756, "cost_per_attempt": 0.055136, "cost_per_solved": 0.0875 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "openai/gpt-5.6-terra@xhigh", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "OpenAI", "model": "GPT-5.6 Terra", "effort": "xhigh", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.1903, "efficacy_mean": 0.85, "efficacy_min": 0.709, "weakest_eval": "HLE", "honesty": { "net_per_100": -2.98, "accuracy_pct": 45.5167, "answers_when_unsure_pct": 89.018, "grade": null, "confident_errors_per_100": 48.5, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 90.8, "share_of_best": 0.943, "cost_per_attempt": 0.0257, "cost_per_solved": 0.0283 }, "HLE": { "score": 41.9, "share_of_best": 0.709, "cost_per_attempt": 0.0957, "cost_per_solved": 0.2284 }, "CritPt": { "score": 27.1, "share_of_best": 0.839, "cost_per_attempt": 0.2907, "cost_per_solved": 1.0727 }, "SciCode": { "score": 51.6, "share_of_best": 0.832, "cost_per_attempt": 0.0325, "cost_per_solved": 0.063 }, "Terminal-Bench v2.1": { "score": 80.1, "share_of_best": 0.876, "cost_per_attempt": 0.3356, "cost_per_solved": 0.419 }, "AA-LCR": { "score": 75.0, "share_of_best": 0.9, "cost_per_attempt": 0.1948, "cost_per_solved": 0.2597 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-terra" ], "zaun_gateway": [ "openai.gpt-5.6-terra" ] }, "confidence": "confirmed" } }, { "id": "anthropic/claude-sonnet-5@non-reasoning", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "HONESTY_INCONCLUSIVE", "disposition": "REVIEW", "vendor": "Anthropic", "model": "Claude Sonnet 5", "effort": "non-reasoning", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.2034, "efficacy_mean": 0.5977, "efficacy_min": 0.034, "weakest_eval": "CritPt", "honesty": { "net_per_100": -0.65, "accuracy_pct": 33.8, "answers_when_unsure_pct": 52.0393, "grade": null, "confident_errors_per_100": 34.45, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "inconclusive", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 80.0, "share_of_best": 0.831, "cost_per_attempt": 0.0087, "cost_per_solved": 0.0109 }, "HLE": { "score": 19.0, "share_of_best": 0.321, "cost_per_attempt": 0.0201, "cost_per_solved": 0.1058 }, "CritPt": { "score": 1.1, "share_of_best": 0.034, "cost_per_attempt": 0.0733, "cost_per_solved": 6.6636 }, "SciCode": { "score": 48.6, "share_of_best": 0.784, "cost_per_attempt": 0.0131, "cost_per_solved": 0.027 }, "Terminal-Bench v2.1": { "score": 75.3, "share_of_best": 0.824, "cost_per_attempt": 0.8877, "cost_per_solved": 1.1789 }, "AA-LCR": { "score": 66.0, "share_of_best": 0.792, "cost_per_attempt": 0.1921, "cost_per_solved": 0.2911 } }, "match": { "api_ids": { "anthropic": [ "claude-sonnet-5" ], "bedrock": [ "us.anthropic.claude-sonnet-5" ] }, "confidence": "confirmed" } }, { "id": "xai/grok-4.6@medium", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "xAI", "model": "Grok 4.6", "effort": "medium", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.2195, "efficacy_mean": 0.8178, "efficacy_min": 0.548, "weakest_eval": "CritPt", "honesty": { "net_per_100": 28.0, "accuracy_pct": 41.9333, "answers_when_unsure_pct": 23.9954, "grade": null, "confident_errors_per_100": 13.9333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0225 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 93.5, "share_of_best": 0.971, "cost_per_attempt": 0.0302, "cost_per_solved": 0.0323 }, "HLE": { "score": 42.1, "share_of_best": 0.712, "cost_per_attempt": 0.0937, "cost_per_solved": 0.2226 }, "CritPt": { "score": 17.7, "share_of_best": 0.548, "cost_per_attempt": 0.2419, "cost_per_solved": 1.3667 }, "SciCode": { "score": 54.6, "share_of_best": 0.881, "cost_per_attempt": 0.0233, "cost_per_solved": 0.0427 }, "Terminal-Bench v2.1": { "score": 84.3, "share_of_best": 0.922, "cost_per_attempt": 0.8093, "cost_per_solved": 0.96 }, "AA-LCR": { "score": 72.7, "share_of_best": 0.872, "cost_per_attempt": 0.2021, "cost_per_solved": 0.278 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "moonshot/kimi-k2.7-code@default", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "Moonshot", "model": "Kimi K2.7 Code", "effort": "default", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": 0.2265, "efficacy_mean": 0.706, "efficacy_min": 0.31, "weakest_eval": "CritPt", "honesty": { "net_per_100": -10.2167, "accuracy_pct": 39.5667, "answers_when_unsure_pct": 82.3773, "grade": null, "confident_errors_per_100": 49.7833, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 89.596, "share_of_best": 0.931, "cost_per_attempt": 0.039601, "cost_per_solved": 0.0442 }, "HLE": { "score": 35.0324, "share_of_best": 0.593, "cost_per_attempt": 0.104703, "cost_per_solved": 0.2989 }, "CritPt": { "score": 10.0, "share_of_best": 0.31, "cost_per_attempt": 0.256304, "cost_per_solved": 2.563 }, "SciCode": { "score": 47.4537, "share_of_best": 0.765, "cost_per_attempt": 0.023826, "cost_per_solved": 0.0502 }, "Terminal-Bench v2.1": { "score": 67.4157, "share_of_best": 0.738, "cost_per_attempt": 0.414329, "cost_per_solved": 0.6146 }, "AA-LCR": { "score": 75.0, "share_of_best": 0.9, "cost_per_attempt": 0.096936, "cost_per_solved": 0.1292 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "anthropic/claude-opus-5@low", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "Anthropic", "model": "Claude Opus 5", "effort": "low", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.2284, "efficacy_mean": 0.8179, "efficacy_min": 0.715, "weakest_eval": "CritPt", "honesty": { "net_per_100": 28.55, "accuracy_pct": 55.95, "answers_when_unsure_pct": 62.202, "grade": null, "confident_errors_per_100": 27.4, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.014 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 88.9, "share_of_best": 0.924, "cost_per_attempt": 0.0076, "cost_per_solved": 0.0085 }, "HLE": { "score": 43.4, "share_of_best": 0.734, "cost_per_attempt": 0.0625, "cost_per_solved": 0.144 }, "CritPt": { "score": 23.1, "share_of_best": 0.715, "cost_per_attempt": 0.7204, "cost_per_solved": 3.1186 }, "SciCode": { "score": 48.0, "share_of_best": 0.774, "cost_per_attempt": 0.0287, "cost_per_solved": 0.0598 }, "Terminal-Bench v2.1": { "score": 76.4, "share_of_best": 0.836, "cost_per_attempt": 0.4984, "cost_per_solved": 0.6524 }, "AA-LCR": { "score": 77.0, "share_of_best": 0.924, "cost_per_attempt": 0.7306, "cost_per_solved": 0.9488 } }, "match": { "api_ids": { "anthropic": [ "claude-opus-5" ], "bedrock": [ "us.anthropic.claude-opus-5" ] }, "confidence": "confirmed" } }, { "id": "xai/grok-4.6@high", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "xAI", "model": "Grok 4.6", "effort": "high", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.251, "efficacy_mean": 0.8288, "efficacy_min": 0.529, "weakest_eval": "CritPt", "honesty": { "net_per_100": 30.48, "accuracy_pct": 48.2333, "answers_when_unsure_pct": 34.2885, "grade": null, "confident_errors_per_100": 17.75, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0285 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 94.9, "share_of_best": 0.986, "cost_per_attempt": 0.0364, "cost_per_solved": 0.0384 }, "HLE": { "score": 42.9, "share_of_best": 0.726, "cost_per_attempt": 0.1205, "cost_per_solved": 0.2809 }, "CritPt": { "score": 17.1, "share_of_best": 0.529, "cost_per_attempt": 0.316, "cost_per_solved": 1.848 }, "SciCode": { "score": 53.6, "share_of_best": 0.865, "cost_per_attempt": 0.0272, "cost_per_solved": 0.0507 }, "Terminal-Bench v2.1": { "score": 88.4, "share_of_best": 0.967, "cost_per_attempt": 0.8098, "cost_per_solved": 0.9161 }, "AA-LCR": { "score": 75.0, "share_of_best": 0.9, "cost_per_attempt": 0.2026, "cost_per_solved": 0.2701 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "meta/muse-spark-1.2@xhigh", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "Meta", "model": "Muse Spark 1.2", "effort": "xhigh", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.2617, "efficacy_mean": 0.8405, "efficacy_min": 0.548, "weakest_eval": "CritPt", "honesty": { "net_per_100": 27.2, "accuracy_pct": 45.3833, "answers_when_unsure_pct": 33.2926, "grade": null, "confident_errors_per_100": 18.1833, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0384 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 90.404, "share_of_best": 0.939, "cost_per_attempt": 0.039377, "cost_per_solved": 0.0436 }, "HLE": { "score": 45.4588, "share_of_best": 0.769, "cost_per_attempt": 0.100624, "cost_per_solved": 0.2214 }, "CritPt": { "score": 17.7143, "share_of_best": 0.548, "cost_per_attempt": 0.345754, "cost_per_solved": 1.9518 }, "SciCode": { "score": 56.3657, "share_of_best": 0.909, "cost_per_attempt": 0.06048, "cost_per_solved": 0.1073 }, "Terminal-Bench v2.1": { "score": 80.1498, "share_of_best": 0.877, "cost_per_attempt": 0.797218, "cost_per_solved": 0.9947 }, "AA-LCR": { "score": 83.3333, "share_of_best": 1.0, "cost_per_attempt": 0.133205, "cost_per_solved": 0.1598 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "xai/grok-4.6@xhigh", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "xAI", "model": "Grok 4.6", "effort": "xhigh", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.2683, "efficacy_mean": 0.8385, "efficacy_min": 0.61, "weakest_eval": "CritPt", "honesty": { "net_per_100": 29.32, "accuracy_pct": 43.0, "answers_when_unsure_pct": 24.0058, "grade": null, "confident_errors_per_100": 13.6833, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.028 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 93.5, "share_of_best": 0.971, "cost_per_attempt": 0.0383, "cost_per_solved": 0.041 }, "HLE": { "score": 44.1, "share_of_best": 0.746, "cost_per_attempt": 0.1248, "cost_per_solved": 0.283 }, "CritPt": { "score": 19.7, "share_of_best": 0.61, "cost_per_attempt": 0.3109, "cost_per_solved": 1.5782 }, "SciCode": { "score": 51.6, "share_of_best": 0.832, "cost_per_attempt": 0.0296, "cost_per_solved": 0.0574 }, "Terminal-Bench v2.1": { "score": 88.0, "share_of_best": 0.963, "cost_per_attempt": 1.1621, "cost_per_solved": 1.3206 }, "AA-LCR": { "score": 75.7, "share_of_best": 0.908, "cost_per_attempt": 0.2036, "cost_per_solved": 0.269 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "deepseek/deepseek-v4-pro-0813@max", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "HONESTY_INCONCLUSIVE", "disposition": "REVIEW", "vendor": "DeepSeek", "model": "DeepSeek V4 Pro 0813", "effort": "max", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": 0.2734, "efficacy_mean": 0.7955, "efficacy_min": 0.557, "weakest_eval": "CritPt", "honesty": { "net_per_100": 0.83, "accuracy_pct": 49.1, "answers_when_unsure_pct": 94.8265, "grade": null, "confident_errors_per_100": 48.2667, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "inconclusive", "cost_per_net_fact_usd": 1.506 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 92.8, "share_of_best": 0.964, "cost_per_attempt": 0.0598, "cost_per_solved": 0.0644 }, "HLE": { "score": 41.0, "share_of_best": 0.694, "cost_per_attempt": 0.1357, "cost_per_solved": 0.331 }, "CritPt": { "score": 18.0, "share_of_best": 0.557, "cost_per_attempt": 0.3568, "cost_per_solved": 1.9822 }, "SciCode": { "score": 49.2, "share_of_best": 0.794, "cost_per_attempt": 0.0503, "cost_per_solved": 0.1022 }, "Terminal-Bench v2.1": { "score": 78.7, "share_of_best": 0.861, "cost_per_attempt": 0.4124, "cost_per_solved": 0.524 }, "AA-LCR": { "score": 75.3, "share_of_best": 0.904, "cost_per_attempt": 0.1389, "cost_per_solved": 0.1845 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "nvidia/nemotron-3-ultra-550b-a55b@reasoning", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "HONESTY_INCONCLUSIVE", "disposition": "REVIEW", "vendor": "NVIDIA", "model": "Nemotron 3 Ultra 550B A55B", "effort": "reasoning", "weights": "open", "price_basis": "median-across-providers", "cost_per_solve_usd": 0.31, "efficacy_mean": 0.5941, "efficacy_min": 0.097, "weakest_eval": "CritPt", "honesty": { "net_per_100": -0.4, "accuracy_pct": 22.5833, "answers_when_unsure_pct": 29.6878, "grade": null, "confident_errors_per_100": 22.9833, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "inconclusive", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 86.6667, "share_of_best": 0.9, "cost_per_attempt": 0.039585, "cost_per_solved": 0.0457 }, "HLE": { "score": 28.4059, "share_of_best": 0.481, "cost_per_attempt": 0.099637, "cost_per_solved": 0.3508 }, "CritPt": { "score": 3.1429, "share_of_best": 0.097, "cost_per_attempt": 0.222579, "cost_per_solved": 7.082 }, "SciCode": { "score": 39.9306, "share_of_best": 0.644, "cost_per_attempt": 0.012388, "cost_per_solved": 0.031 }, "Terminal-Bench v2.1": { "score": 53.9326, "share_of_best": 0.59, "cost_per_attempt": 1.141376, "cost_per_solved": 2.1163 }, "AA-LCR": { "score": 71.0, "share_of_best": 0.852, "cost_per_attempt": 0.084538, "cost_per_solved": 0.1191 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "zai/glm-5.3@max", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "Z.ai", "model": "GLM-5.3", "effort": "max", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": 0.316, "efficacy_mean": 0.8341, "efficacy_min": 0.591, "weakest_eval": "CritPt", "honesty": { "net_per_100": 14.3, "accuracy_pct": 33.85, "answers_when_unsure_pct": 29.554, "grade": null, "confident_errors_per_100": 19.55, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0797 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 91.7, "share_of_best": 0.953, "cost_per_attempt": 0.0316, "cost_per_solved": 0.0345 }, "HLE": { "score": 42.3, "share_of_best": 0.716, "cost_per_attempt": 0.2328, "cost_per_solved": 0.5504 }, "CritPt": { "score": 19.1, "share_of_best": 0.591, "cost_per_attempt": 0.4522, "cost_per_solved": 2.3675 }, "SciCode": { "score": 56.5, "share_of_best": 0.911, "cost_per_attempt": 0.082, "cost_per_solved": 0.1451 }, "Terminal-Bench v2.1": { "score": 83.9, "share_of_best": 0.918, "cost_per_attempt": 0.6583, "cost_per_solved": 0.7846 }, "AA-LCR": { "score": 76.3, "share_of_best": 0.916, "cost_per_attempt": 0.1486, "cost_per_solved": 0.1948 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "alibaba/qwen3.8-27b@xhigh", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "FAILS_HONESTY", "disposition": "DENY", "vendor": "Alibaba", "model": "Qwen3.8 27B", "effort": "xhigh", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": 0.3174, "efficacy_mean": 0.7006, "efficacy_min": 0.168, "weakest_eval": "CritPt", "honesty": { "net_per_100": -9.9833, "accuracy_pct": 15.5833, "answers_when_unsure_pct": 30.2863, "grade": null, "confident_errors_per_100": 25.5667, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 90.5051, "share_of_best": 0.94, "cost_per_attempt": 0.046367, "cost_per_solved": 0.0512 }, "HLE": { "score": 33.9203, "share_of_best": 0.574, "cost_per_attempt": 0.13257, "cost_per_solved": 0.3908 }, "CritPt": { "score": 5.4286, "share_of_best": 0.168, "cost_per_attempt": 0.321722, "cost_per_solved": 5.9264 }, "SciCode": { "score": 44.6759, "share_of_best": 0.721, "cost_per_attempt": 0.064475, "cost_per_solved": 0.1443 }, "Terminal-Bench v2.1": { "score": 79.7753, "share_of_best": 0.873, "cost_per_attempt": 0.59631, "cost_per_solved": 0.7475 }, "AA-LCR": { "score": 77.3333, "share_of_best": 0.928, "cost_per_attempt": 0.061735, "cost_per_solved": 0.0798 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "google/gemini-3.1-pro-preview@default", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "Google", "model": "Gemini 3.1 Pro Preview", "effort": "default", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.3733, "efficacy_mean": 0.8377, "efficacy_min": 0.548, "weakest_eval": "CritPt", "honesty": { "net_per_100": 31.88, "accuracy_pct": 54.85, "answers_when_unsure_pct": 50.8675, "grade": null, "confident_errors_per_100": 22.9667, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0555 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 94.1, "share_of_best": 0.978, "cost_per_attempt": 0.0715, "cost_per_solved": 0.076 }, "HLE": { "score": 47.0, "share_of_best": 0.795, "cost_per_attempt": 0.1968, "cost_per_solved": 0.4187 }, "CritPt": { "score": 17.7, "share_of_best": 0.548, "cost_per_attempt": 0.4739, "cost_per_solved": 2.6774 }, "SciCode": { "score": 58.9, "share_of_best": 0.95, "cost_per_attempt": 0.0887, "cost_per_solved": 0.1506 }, "Terminal-Bench v2.1": { "score": 73.8, "share_of_best": 0.807, "cost_per_attempt": 0.578, "cost_per_solved": 0.7832 }, "AA-LCR": { "score": 79.0, "share_of_best": 0.948, "cost_per_attempt": 0.2127, "cost_per_solved": 0.2692 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "openai/gpt-5.6-terra@max", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "HONESTY_INCONCLUSIVE", "disposition": "REVIEW", "vendor": "OpenAI", "model": "GPT-5.6 Terra", "effort": "max", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.3826, "efficacy_mean": 0.9007, "efficacy_min": 0.726, "weakest_eval": "HLE", "honesty": { "net_per_100": 0.05, "accuracy_pct": 46.8, "answers_when_unsure_pct": 87.8759, "grade": null, "confident_errors_per_100": 46.75, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "inconclusive", "cost_per_net_fact_usd": 140.6 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 92.5, "share_of_best": 0.961, "cost_per_attempt": 0.073, "cost_per_solved": 0.0789 }, "HLE": { "score": 42.9, "share_of_best": 0.726, "cost_per_attempt": 0.2277, "cost_per_solved": 0.5308 }, "CritPt": { "score": 30.0, "share_of_best": 0.929, "cost_per_attempt": 0.6084, "cost_per_solved": 2.028 }, "SciCode": { "score": 53.9, "share_of_best": 0.869, "cost_per_attempt": 0.1144, "cost_per_solved": 0.2122 }, "Terminal-Bench v2.1": { "score": 88.0, "share_of_best": 0.963, "cost_per_attempt": 0.5732, "cost_per_solved": 0.6514 }, "AA-LCR": { "score": 79.7, "share_of_best": 0.956, "cost_per_attempt": 0.213, "cost_per_solved": 0.2673 } }, "match": { "api_ids": { "openai": [ "gpt-5.6-terra" ], "zaun_gateway": [ "openai.gpt-5.6-terra" ] }, "confidence": "confirmed" } }, { "id": "alibaba/qwen3.8-2.4t-a95b@default", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "Alibaba", "model": "Qwen3.8 2.4T A95B", "effort": "default", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": 0.3971, "efficacy_mean": 0.8238, "efficacy_min": 0.619, "weakest_eval": "CritPt", "honesty": { "net_per_100": 4.3167, "accuracy_pct": 31.25, "answers_when_unsure_pct": 39.1758, "grade": null, "confident_errors_per_100": 26.9333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.6529 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 93.5354, "share_of_best": 0.972, "cost_per_attempt": 0.06581, "cost_per_solved": 0.0704 }, "HLE": { "score": 42.4467, "share_of_best": 0.718, "cost_per_attempt": 0.20157, "cost_per_solved": 0.4749 }, "CritPt": { "score": 20.0, "share_of_best": 0.619, "cost_per_attempt": 0.482792, "cost_per_solved": 2.414 }, "SciCode": { "score": 51.6204, "share_of_best": 0.833, "cost_per_attempt": 0.100608, "cost_per_solved": 0.1949 }, "Terminal-Bench v2.1": { "score": 82.0225, "share_of_best": 0.897, "cost_per_attempt": 0.694074, "cost_per_solved": 0.8462 }, "AA-LCR": { "score": 75.3333, "share_of_best": 0.904, "cost_per_attempt": 0.222036, "cost_per_solved": 0.2947 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "alibaba/qwen3.8-max@default", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "Alibaba", "model": "Qwen3.8 Max", "effort": "default", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 0.4488, "efficacy_mean": 0.8242, "efficacy_min": 0.619, "weakest_eval": "CritPt", "honesty": { "net_per_100": 3.4, "accuracy_pct": 31.85, "answers_when_unsure_pct": 41.7461, "grade": null, "confident_errors_per_100": 28.45, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.8437 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 92.7273, "share_of_best": 0.963, "cost_per_attempt": 0.073518, "cost_per_solved": 0.0793 }, "HLE": { "score": 43.0491, "share_of_best": 0.728, "cost_per_attempt": 0.223442, "cost_per_solved": 0.519 }, "CritPt": { "score": 20.0, "share_of_best": 0.619, "cost_per_attempt": 0.509228, "cost_per_solved": 2.5461 }, "SciCode": { "score": 52.8935, "share_of_best": 0.853, "cost_per_attempt": 0.110631, "cost_per_solved": 0.2092 }, "Terminal-Bench v2.1": { "score": 81.2734, "share_of_best": 0.889, "cost_per_attempt": 1.015721, "cost_per_solved": 1.2498 }, "AA-LCR": { "score": 74.3333, "share_of_best": 0.892, "cost_per_attempt": 0.221785, "cost_per_solved": 0.2984 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "anthropic/claude-sonnet-5@max", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "BELOW_EFFICACY", "disposition": "REVIEW", "vendor": "Anthropic", "model": "Claude Sonnet 5", "effort": "max", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": 1.3267, "efficacy_mean": 0.8063, "efficacy_min": 0.523, "weakest_eval": "CritPt", "honesty": { "net_per_100": 16.45, "accuracy_pct": 40.05, "answers_when_unsure_pct": 39.3661, "grade": null, "confident_errors_per_100": 23.6, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.1763 }, "zero_scored_evals": [], "per_eval": { "GPQA Diamond": { "score": 91.1, "share_of_best": 0.946, "cost_per_attempt": 0.3663, "cost_per_solved": 0.4021 }, "HLE": { "score": 41.3, "share_of_best": 0.699, "cost_per_attempt": 1.0208, "cost_per_solved": 2.4717 }, "CritPt": { "score": 16.9, "share_of_best": 0.523, "cost_per_attempt": 2.3138, "cost_per_solved": 13.6911 }, "SciCode": { "score": 53.6, "share_of_best": 0.865, "cost_per_attempt": 0.1917, "cost_per_solved": 0.3576 }, "Terminal-Bench v2.1": { "score": 80.5, "share_of_best": 0.881, "cost_per_attempt": 1.8879, "cost_per_solved": 2.3452 }, "AA-LCR": { "score": 77.0, "share_of_best": 0.924, "cost_per_attempt": 0.3679, "cost_per_solved": 0.4778 } }, "match": { "api_ids": { "anthropic": [ "claude-sonnet-5" ], "bedrock": [ "us.anthropic.claude-sonnet-5" ] }, "confidence": "confirmed" } }, { "id": "google/gemini-3.5-flash-lite@default", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "COST_UNDEFINED", "disposition": "DENY", "vendor": "Google", "model": "Gemini 3.5 Flash-Lite", "effort": "default", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": null, "efficacy_mean": 0.5552, "efficacy_min": 0.0, "weakest_eval": "CritPt", "honesty": { "net_per_100": 5.23, "accuracy_pct": 29.4667, "answers_when_unsure_pct": 34.3573, "grade": null, "confident_errors_per_100": 24.2333, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "passes", "cost_per_net_fact_usd": 0.0593 }, "zero_scored_evals": [ "CritPt" ], "per_eval": { "GPQA Diamond": { "score": 83.8, "share_of_best": 0.871, "cost_per_attempt": 0.0122, "cost_per_solved": 0.0146 }, "HLE": { "score": 18.8, "share_of_best": 0.318, "cost_per_attempt": 0.0271, "cost_per_solved": 0.1441 }, "CritPt": { "score": 0.0, "share_of_best": 0.0, "cost_per_attempt": 0.0704, "cost_per_solved": null }, "SciCode": { "score": 40.9, "share_of_best": 0.66, "cost_per_attempt": 0.0126, "cost_per_solved": 0.0308 }, "Terminal-Bench v2.1": { "score": 53.6, "share_of_best": 0.586, "cost_per_attempt": 0.2571, "cost_per_solved": 0.4797 }, "AA-LCR": { "score": 74.7, "share_of_best": 0.896, "cost_per_attempt": 0.039, "cost_per_solved": 0.0522 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "anthropic/claude-haiku-4.5@reasoning", "aliases": [ "Claude 4.5 Haiku" ], "rank": null, "rank_by_cost": null, "reason_code": "COST_UNDEFINED", "disposition": "DENY", "vendor": "Anthropic", "model": "Claude Haiku 4.5", "effort": "reasoning", "weights": "closed", "price_basis": "first-party", "cost_per_solve_usd": null, "efficacy_mean": 0.4901, "efficacy_min": 0.0, "weakest_eval": "CritPt", "honesty": { "net_per_100": -4.37, "accuracy_pct": 18.0167, "answers_when_unsure_pct": 27.3023, "grade": null, "confident_errors_per_100": 22.3833, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [ "CritPt" ], "per_eval": { "GPQA Diamond": { "score": 67.2, "share_of_best": 0.698, "cost_per_attempt": 0.0749, "cost_per_solved": 0.1115 }, "HLE": { "score": 10.4, "share_of_best": 0.176, "cost_per_attempt": 0.1245, "cost_per_solved": 1.1971 }, "CritPt": { "score": 0.0, "share_of_best": 0.0, "cost_per_attempt": 0.1648, "cost_per_solved": null }, "SciCode": { "score": 43.3, "share_of_best": 0.698, "cost_per_attempt": 0.0507, "cost_per_solved": 0.1171 }, "Terminal-Bench v2.1": { "score": 44.2, "share_of_best": 0.484, "cost_per_attempt": 0.699, "cost_per_solved": 1.5814 }, "AA-LCR": { "score": 73.7, "share_of_best": 0.884, "cost_per_attempt": 0.1213, "cost_per_solved": 0.1646 } }, "match": { "api_ids": { "anthropic": [ "claude-haiku-4-5", "claude-haiku-4-5-20251001" ], "bedrock": [ "us.anthropic.claude-haiku-4-5-20251001-v1:0" ] }, "confidence": "confirmed" } }, { "id": "alibaba/qwen3.8-27b@medium", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "COST_UNDEFINED", "disposition": "DENY", "vendor": "Alibaba", "model": "Qwen3.8 27B", "effort": "medium", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": null, "efficacy_mean": 0.56, "efficacy_min": 0.0, "weakest_eval": "CritPt", "honesty": { "net_per_100": -36.15, "accuracy_pct": 18.35, "answers_when_unsure_pct": 66.7483, "grade": null, "confident_errors_per_100": 54.5, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [ "CritPt" ], "per_eval": { "GPQA Diamond": { "score": 84.5455, "share_of_best": 0.878, "cost_per_attempt": 0.015165, "cost_per_solved": 0.0179 }, "HLE": { "score": 14.0871, "share_of_best": 0.238, "cost_per_attempt": 0.038998, "cost_per_solved": 0.2768 }, "CritPt": { "score": 0, "share_of_best": 0.0, "cost_per_attempt": 0.091763, "cost_per_solved": null }, "SciCode": { "score": 38.0787, "share_of_best": 0.614, "cost_per_attempt": 0.011675, "cost_per_solved": 0.0307 }, "Terminal-Bench v2.1": { "score": 65.1685, "share_of_best": 0.713, "cost_per_attempt": 0.4084, "cost_per_solved": 0.6267 }, "AA-LCR": { "score": 76.3333, "share_of_best": 0.916, "cost_per_attempt": 0.057596, "cost_per_solved": 0.0755 } }, "match": { "api_ids": {}, "confidence": "unmapped" } }, { "id": "alibaba/qwen3.8-27b@low", "aliases": [], "rank": null, "rank_by_cost": null, "reason_code": "COST_UNDEFINED", "disposition": "DENY", "vendor": "Alibaba", "model": "Qwen3.8 27B", "effort": "low", "weights": "open", "price_basis": "first-party", "cost_per_solve_usd": null, "efficacy_mean": 0.5653, "efficacy_min": 0.0, "weakest_eval": "CritPt", "honesty": { "net_per_100": -26.6667, "accuracy_pct": 17.1333, "answers_when_unsure_pct": 52.856, "grade": null, "confident_errors_per_100": 43.8, "attempt_rate_pct": null, "standard_error": 1.291, "verdict": "fails", "cost_per_net_fact_usd": null }, "zero_scored_evals": [ "CritPt" ], "per_eval": { "GPQA Diamond": { "score": 84.5455, "share_of_best": 0.878, "cost_per_attempt": 0.014086, "cost_per_solved": 0.0167 }, "HLE": { "score": 14.0408, "share_of_best": 0.238, "cost_per_attempt": 0.021691, "cost_per_solved": 0.1545 }, "CritPt": { "score": 0, "share_of_best": 0.0, "cost_per_attempt": 0.050913, "cost_per_solved": null }, "SciCode": { "score": 39.8148, "share_of_best": 0.642, "cost_per_attempt": 0.008242, "cost_per_solved": 0.0207 }, "Terminal-Bench v2.1": { "score": 67.4157, "share_of_best": 0.738, "cost_per_attempt": 0.430274, "cost_per_solved": 0.6382 }, "AA-LCR": { "score": 74.6667, "share_of_best": 0.896, "cost_per_attempt": 0.057146, "cost_per_solved": 0.0765 } }, "match": { "api_ids": {}, "confidence": "unmapped" } } ] }