{
  "snapshot_id": "beta-2026-10-12",
  "title": "Beta results",
  "status": "beta",
  "as_of": "2026-10-12",
  "provenance": {
    "release": "cycle-1-91-2026-09-08-luna-r5",
    "release_digest": "4311a6ee59c3ae1fcb392f1dd768ab00d1d7a7c7ea135271f3e117746233ed8a",
    "method": "Scores for the first ten configurations were computed from their unit-level predictions on September 15-18, 2026, and checked against re-downloaded run receipts (91 case receipts and the same 387 scoreable units for each). Validated native scored exports supply 12 full-document agentic configurations, replacing matching historical rows where available. Their accuracy and confidence counts derive from public prediction units. Significance: 13 of 19 configurations are included in the paired analysis. The remaining 6 configurations were not tested. The correction retains all 171 pairs in the 19-model family across three metrics."
  },
  "cohort": {
    "case_count": 91,
    "unit_count": 387,
    "dismissed_unit_count": 285,
    "decision_window": {
      "start": "2026-06-30",
      "end": "2026-08-07"
    },
    "constant_forecast_micro_brier": 0.1940989123,
    "majority_class_accuracy": 0.7364341085,
    "bootstrap": {
      "method": "Paired case-cluster percentile bootstrap",
      "replicates": 1000000,
      "correction": "Bonferroni across 171 model pairs and three metrics",
      "caveat": "13 of 19 configurations are included in the paired analysis. The remaining 6 configurations were not tested. The correction retains all 171 pairs in the 19-model family across three metrics. Exploratory. Cross-case dependence has not been verified. Unavailable pairs are not tested."
    }
  },
  "significant_pairs": [
    {
      "better": "claude-opus-5",
      "worse": "gpt-4-1",
      "metric": "micro_brier"
    },
    {
      "better": "claude-opus-5",
      "worse": "gpt-4-1",
      "metric": "equal_case_brier"
    },
    {
      "better": "claude-opus-5",
      "worse": "gpt-4-1",
      "metric": "accuracy"
    },
    {
      "better": "claude-opus-5-5",
      "worse": "gpt-4-1",
      "metric": "micro_brier"
    },
    {
      "better": "claude-opus-5-5",
      "worse": "gpt-4-1",
      "metric": "equal_case_brier"
    },
    {
      "better": "claude-opus-5-5",
      "worse": "gpt-4-1",
      "metric": "accuracy"
    },
    {
      "better": "claude-sonnet-5",
      "worse": "gpt-4-1",
      "metric": "micro_brier"
    },
    {
      "better": "claude-sonnet-5",
      "worse": "gpt-4-1",
      "metric": "equal_case_brier"
    },
    {
      "better": "claude-sonnet-5",
      "worse": "gpt-4-1",
      "metric": "accuracy"
    },
    {
      "better": "claude-sonnet-5-5",
      "worse": "gpt-4-1",
      "metric": "micro_brier"
    },
    {
      "better": "claude-sonnet-5-5",
      "worse": "gpt-4-1",
      "metric": "equal_case_brier"
    },
    {
      "better": "claude-sonnet-5-5",
      "worse": "gpt-4-1",
      "metric": "accuracy"
    },
    {
      "better": "gemini-3-1-pro-preview",
      "worse": "gpt-4-1",
      "metric": "micro_brier"
    },
    {
      "better": "gemini-3-1-pro-preview",
      "worse": "gpt-4-1",
      "metric": "equal_case_brier"
    },
    {
      "better": "gemini-3-1-pro-preview",
      "worse": "gpt-4-1",
      "metric": "accuracy"
    },
    {
      "better": "gpt-5-6-luna",
      "worse": "gpt-4-1",
      "metric": "micro_brier"
    },
    {
      "better": "gpt-5-6-luna",
      "worse": "gpt-4-1",
      "metric": "equal_case_brier"
    },
    {
      "better": "gpt-5-6-luna",
      "worse": "gpt-4-1",
      "metric": "accuracy"
    },
    {
      "better": "gpt-5-6-sol",
      "worse": "gpt-4-1",
      "metric": "micro_brier"
    },
    {
      "better": "gpt-5-6-sol",
      "worse": "gpt-4-1",
      "metric": "equal_case_brier"
    },
    {
      "better": "gpt-5-6-sol",
      "worse": "gpt-4-1",
      "metric": "accuracy"
    },
    {
      "better": "gpt-6-1-sol",
      "worse": "gpt-4-1",
      "metric": "micro_brier"
    },
    {
      "better": "gpt-6-1-sol",
      "worse": "gpt-4-1",
      "metric": "equal_case_brier"
    },
    {
      "better": "gpt-6-1-sol",
      "worse": "gpt-4-1",
      "metric": "accuracy"
    },
    {
      "better": "gpt-6-luna",
      "worse": "gpt-4-1",
      "metric": "micro_brier"
    },
    {
      "better": "gpt-6-luna",
      "worse": "gpt-4-1",
      "metric": "equal_case_brier"
    },
    {
      "better": "gpt-6-luna",
      "worse": "gpt-4-1",
      "metric": "accuracy"
    },
    {
      "better": "gpt-6-sol",
      "worse": "gpt-4-1",
      "metric": "micro_brier"
    },
    {
      "better": "gpt-6-sol",
      "worse": "gpt-4-1",
      "metric": "equal_case_brier"
    },
    {
      "better": "gpt-6-sol",
      "worse": "gpt-4-1",
      "metric": "accuracy"
    },
    {
      "better": "grok-4-7",
      "worse": "gpt-4-1",
      "metric": "micro_brier"
    },
    {
      "better": "grok-4-7",
      "worse": "gpt-4-1",
      "metric": "equal_case_brier"
    },
    {
      "better": "grok-4-7",
      "worse": "gpt-4-1",
      "metric": "accuracy"
    },
    {
      "better": "muse-wonder-1-0",
      "worse": "gpt-4-1",
      "metric": "micro_brier"
    },
    {
      "better": "muse-wonder-1-0",
      "worse": "gpt-4-1",
      "metric": "equal_case_brier"
    },
    {
      "better": "muse-wonder-1-0",
      "worse": "gpt-4-1",
      "metric": "accuracy"
    }
  ],
  "models": [
    {
      "slug": "claude-fable-5-1",
      "display_name": "Claude Fable 5.1",
      "provider": "Anthropic",
      "model_key": "anthropic:claude-fable-5-1",
      "access": "Anthropic API",
      "reasoning": "Adaptive thinking (provider default)",
      "release_date": "2026-09-01",
      "training_cutoff": "June 2026 (training-data cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported training-data cutoff (June 2026) is in the month of the first scored decision (June 30, 2026). It is treated as preceding that decision, because a ruling is very unlikely to enter training data the day it issues.",
      "github_run_id": "34307189166",
      "micro_brier": 0.1287705426356589,
      "equal_case_brier": 0.13653489478775194,
      "correct": 318,
      "high_confidence": {
        "count": 81,
        "wrong": 0,
        "mean_confidence": 0.9288888888888889
      },
      "cost": {
        "usd": 83.18,
        "basis": "estimated",
        "note": "Estimated cost with caching. The run was inadvertently uncached (actual successful workload $189.70); the cached cost was reconstructed from saved per-turn token usage and validated against the other Anthropic runs."
      }
    },
    {
      "slug": "grok-4-6",
      "display_name": "Grok 4.6",
      "provider": "SpaceXAI",
      "model_key": "vercel_ai_gateway:spacexai/grok-4.6",
      "access": "Vercel AI Gateway (served by xAI)",
      "reasoning": "High",
      "release_date": "2026-08-12",
      "training_cutoff": "February 1, 2026 (knowledge cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported knowledge cutoff (February 1, 2026) predates the first scored decision on June 30, 2026. xAI's pages report January 2026 and February 1, 2026; the later date is used.",
      "github_run_id": "34294236811",
      "micro_brier": 0.13368062015503876,
      "equal_case_brier": 0.13480444111444112,
      "correct": 312,
      "high_confidence": {
        "count": 112,
        "wrong": 1,
        "mean_confidence": 0.9302678571428571
      },
      "cost": {
        "usd": 15.72,
        "basis": "standard_rate",
        "note": null
      }
    },
    {
      "slug": "claude-opus-5",
      "display_name": "Claude Opus 5",
      "provider": "Anthropic",
      "model_key": "anthropic:claude-opus-5",
      "access": "Anthropic API",
      "reasoning": "Adaptive thinking (provider default: high)",
      "release_date": "2026-07-24",
      "training_cutoff": "May 2026 (training-data cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported training-data cutoff (May 2026) predates the first scored decision on June 30, 2026.",
      "github_run_id": "34845972981",
      "micro_brier": 0.13987105943152456,
      "equal_case_brier": 0.13548394074179787,
      "correct": 310,
      "high_confidence": {
        "count": 121,
        "wrong": 4,
        "mean_confidence": 0.9361983471074381
      },
      "cost": {
        "usd": 45.89,
        "basis": "standard_rate",
        "note": null
      }
    },
    {
      "slug": "gpt-5-6-sol",
      "display_name": "GPT-5.6 Sol",
      "provider": "OpenAI",
      "model_key": "vercel_ai_gateway:openai/gpt-5.6-sol",
      "access": "Vercel AI Gateway (served by OpenAI, Flex)",
      "reasoning": "High",
      "release_date": "2026-06-26",
      "training_cutoff": "February 16, 2026 (knowledge cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported knowledge cutoff (February 16, 2026) predates the first scored decision on June 30, 2026.",
      "github_run_id": "34428487985",
      "micro_brier": 0.14009137209302325,
      "equal_case_brier": 0.1331861701607916,
      "correct": 313,
      "high_confidence": {
        "count": 228,
        "wrong": 26,
        "mean_confidence": 0.97017105263158
      },
      "cost": {
        "usd": 24.91,
        "basis": "standard_rate",
        "note": "Standard-rate equivalent. Flex pricing and a Vercel discount made the actual cost roughly 75% lower."
      }
    },
    {
      "slug": "kimi-k3",
      "display_name": "Kimi K3",
      "provider": "Moonshot",
      "model_key": "vercel_ai_gateway:moonshotai/kimi-k3",
      "access": "Vercel AI Gateway (served by DeepInfra)",
      "reasoning": "High",
      "release_date": "2026-07-17",
      "training_cutoff": null,
      "eligibility": "qualified",
      "eligibility_reason": "The provider has not published a training or knowledge cutoff. Moonshot AI has not published a cutoff.",
      "github_run_id": "34305285043",
      "micro_brier": 0.14129870801033592,
      "equal_case_brier": 0.15366887350744493,
      "correct": 303,
      "high_confidence": {
        "count": 92,
        "wrong": 1,
        "mean_confidence": 0.9284782608695652
      },
      "cost": {
        "usd": 16.87,
        "basis": "standard_rate",
        "note": null
      }
    },
    {
      "slug": "gpt-5-6-luna",
      "display_name": "GPT-5.6 Luna",
      "provider": "OpenAI",
      "model_key": "openai:gpt-5.6-luna",
      "access": "OpenAI API",
      "reasoning": "High",
      "release_date": "2026-06-26",
      "training_cutoff": "February 16, 2026 (knowledge cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported knowledge cutoff (February 16, 2026) predates the first scored decision on June 30, 2026.",
      "github_run_id": "34222142006",
      "micro_brier": 0.1432169250645995,
      "equal_case_brier": 0.15083510810617953,
      "correct": 313,
      "high_confidence": {
        "count": 182,
        "wrong": 20,
        "mean_confidence": 0.9492307692307692
      },
      "cost": {
        "usd": 1.48,
        "basis": "standard_rate",
        "note": "Billing-derived; includes eight extra requests."
      }
    },
    {
      "slug": "gpt-6-astra",
      "display_name": "GPT-6 Astra",
      "provider": "OpenAI",
      "model_key": "openai:gpt-6-astra",
      "access": "OpenAI API (Flex)",
      "reasoning": "High",
      "release_date": "2026-09-03",
      "training_cutoff": "April 30, 2026 (knowledge cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported knowledge cutoff (April 30, 2026) predates the first scored decision on June 30, 2026.",
      "github_run_id": "34305259900",
      "micro_brier": 0.15221040051679585,
      "equal_case_brier": 0.1427474770665842,
      "correct": 305,
      "high_confidence": {
        "count": 207,
        "wrong": 21,
        "mean_confidence": 0.9547584541062801
      },
      "cost": {
        "usd": 59.74,
        "basis": "standard_rate",
        "note": "Standard-rate equivalent. Flex pricing made the actual cost lower."
      }
    },
    {
      "slug": "gemini-3-8-flash",
      "display_name": "Gemini 3.8 Flash",
      "provider": "Google",
      "model_key": "google:gemini-3.8-flash",
      "access": "Google Gemini API",
      "reasoning": "High thinking",
      "release_date": "2026-09-02",
      "training_cutoff": "March 2026 (knowledge cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported knowledge cutoff (March 2026) predates the first scored decision on June 30, 2026.",
      "github_run_id": "34293244222",
      "micro_brier": 0.1533139534883721,
      "equal_case_brier": 0.13899537216751504,
      "correct": 310,
      "high_confidence": {
        "count": 276,
        "wrong": 35,
        "mean_confidence": 0.9518478260869564
      },
      "cost": {
        "usd": 35.61,
        "basis": "standard_rate",
        "note": "Uses Google's Standard introductory pricing; cache usage is imputed for three cases."
      }
    },
    {
      "slug": "claude-sonnet-5",
      "display_name": "Claude Sonnet 5",
      "provider": "Anthropic",
      "model_key": "anthropic:claude-sonnet-5",
      "access": "Anthropic API",
      "reasoning": "Adaptive thinking (provider default)",
      "release_date": "2026-06-30",
      "training_cutoff": "January 2026 (training-data cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported training-data cutoff (January 2026) predates the first scored decision on June 30, 2026.",
      "github_run_id": "34895753581",
      "micro_brier": 0.15848397932816538,
      "equal_case_brier": 0.15963922014493442,
      "correct": 293,
      "high_confidence": {
        "count": 60,
        "wrong": 5,
        "mean_confidence": 0.9256666666666662
      },
      "cost": {
        "usd": 24.68,
        "basis": "standard_rate",
        "note": null
      }
    },
    {
      "slug": "muse-spark-1-3",
      "display_name": "Muse Spark 1.3",
      "provider": "Meta",
      "model_key": "vercel_ai_gateway:meta/muse-spark-1.3-contributor",
      "access": "Vercel AI Gateway (served by Meta, contributor tier)",
      "reasoning": "High",
      "release_date": "2026-09-02",
      "training_cutoff": null,
      "eligibility": "qualified",
      "eligibility_reason": "The provider has not published a training or knowledge cutoff. Meta has not published a cutoff.",
      "github_run_id": "34272944667",
      "micro_brier": 0.18120671834625324,
      "equal_case_brier": 0.17844744224029938,
      "correct": 279,
      "high_confidence": {
        "count": 135,
        "wrong": 15,
        "mean_confidence": 0.9351851851851852
      },
      "cost": {
        "usd": 8.78,
        "basis": "standard_rate",
        "note": "Ran on the contributor-tier endpoint; cost shown at standard rates. Standard-tier execution has not been measured."
      }
    },
    {
      "slug": "gpt-6-sol",
      "display_name": "GPT-6 Sol",
      "provider": "OpenAI",
      "model_key": "openai:gpt-6-sol",
      "access": "OpenAI API",
      "reasoning": "High",
      "release_date": "2026-09-22",
      "training_cutoff": "April 20, 2026 (knowledge cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported knowledge cutoff (April 20, 2026) predates the first scored decision on June 30, 2026.",
      "github_run_id": "35776235829",
      "micro_brier": 0.11762790697674418,
      "equal_case_brier": 0.12949124213088498,
      "correct": 327,
      "high_confidence": {
        "count": 169,
        "wrong": 8,
        "mean_confidence": 0.9558579881656799
      },
      "cost": {
        "usd": 11.0097835,
        "basis": "estimated",
        "note": "Standard-rate estimate for 91 successful cases. Incomplete cache usage or prices use ordinary-input estimates, which may overstate or understate cache-adjusted costs. Successful workload estimate: $5.5049 at Flex rates. Excludes failed attempts and unresolved charges."
      }
    },
    {
      "slug": "gpt-6-luna",
      "display_name": "GPT-6 Luna",
      "provider": "OpenAI",
      "model_key": "openai:gpt-6-luna",
      "access": "OpenAI API",
      "reasoning": "High",
      "release_date": "2026-09-22",
      "training_cutoff": "May 18, 2026 (knowledge cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported knowledge cutoff (May 18, 2026) predates the first scored decision on June 30, 2026.",
      "github_run_id": "35933696453",
      "micro_brier": 0.16367900516795866,
      "equal_case_brier": 0.17379408853051712,
      "correct": 296,
      "high_confidence": {
        "count": 183,
        "wrong": 19,
        "mean_confidence": 0.9541256830601097
      },
      "cost": {
        "usd": 0.62452294,
        "basis": "estimated",
        "note": "Standard-rate estimate for 91 successful cases. Incomplete cache usage or prices use ordinary-input estimates, which may overstate or understate cache-adjusted costs. Successful workload estimate: $0.3123 at Flex rates. Excludes failed attempts and unresolved charges."
      }
    },
    {
      "slug": "claude-opus-5-5",
      "display_name": "Claude Opus 5.5",
      "provider": "Anthropic",
      "model_key": "anthropic:claude-opus-5-5",
      "access": "Anthropic API",
      "reasoning": "High",
      "release_date": "2026-09-22",
      "training_cutoff": "June 2026 (training-data cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported training-data cutoff (June 2026) is in the month of the first scored decision (June 30, 2026). It is treated as preceding that decision, because a ruling is very unlikely to enter training data the day it issues.",
      "github_run_id": "35953489581",
      "micro_brier": 0.14540258397932818,
      "equal_case_brier": 0.1476222901701473,
      "correct": 299,
      "high_confidence": {
        "count": 65,
        "wrong": 0,
        "mean_confidence": 0.9278461538461539
      },
      "cost": {
        "usd": 20.5866604,
        "basis": "estimated",
        "note": "Standard-rate estimate for 91 successful cases. Retains reported cache usage and cache prices. Successful workload estimate: $20.5867. Excludes failed attempts and unresolved charges."
      }
    },
    {
      "slug": "grok-4-7",
      "display_name": "Grok 4.7",
      "provider": "SpaceXAI",
      "model_key": "vercel_ai_gateway:spacexai/grok-4.7",
      "access": "Vercel AI Gateway (served by xAI)",
      "reasoning": "High",
      "release_date": "2026-09-21",
      "training_cutoff": "May 2026 (knowledge cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported knowledge cutoff (May 2026) predates the first scored decision on June 30, 2026.",
      "github_run_id": "35799524684",
      "micro_brier": 0.15832558139534883,
      "equal_case_brier": 0.16454140522176236,
      "correct": 298,
      "high_confidence": {
        "count": 98,
        "wrong": 4,
        "mean_confidence": 0.9293877551020406
      },
      "cost": {
        "usd": 32.999826,
        "basis": "estimated",
        "note": "Standard-rate estimate for 91 successful cases. Incomplete cache usage or prices use ordinary-input estimates, which may overstate or understate cache-adjusted costs. Successful workload charge: $11.5172. Excludes failed attempts and unresolved charges."
      }
    },
    {
      "slug": "gpt-4-1",
      "display_name": "GPT-4.1",
      "provider": "OpenAI",
      "model_key": "openai:gpt-4.1-2025-04-14",
      "access": "OpenAI API (dated snapshot gpt-4.1-2025-04-14)",
      "reasoning": "None (non-reasoning model)",
      "release_date": "2025-04-14",
      "training_cutoff": "June 1, 2024 (knowledge cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported knowledge cutoff (June 1, 2024) predates the first scored decision on June 30, 2026.",
      "github_run_id": "35954543028",
      "micro_brier": 0.6051270671834625,
      "equal_case_brier": 0.5752195578231293,
      "correct": 151,
      "high_confidence": {
        "count": 357,
        "wrong": 229,
        "mean_confidence": 0.9939355742296918
      },
      "cost": {
        "usd": 38.34943,
        "basis": "estimated",
        "note": "Standard-rate estimate for 91 successful cases. Incomplete cache usage or prices use ordinary-input estimates, which may overstate or understate cache-adjusted costs. Successful workload estimate: $38.3494. Excludes failed attempts and unresolved charges."
      }
    },
    {
      "slug": "gemini-3-1-pro-preview",
      "display_name": "Gemini 3.1 Pro Preview",
      "provider": "Google",
      "model_key": "google:gemini-3.1-pro-preview",
      "access": "Google Gemini API (preview alias)",
      "reasoning": "High thinking",
      "release_date": "2026-02-19",
      "training_cutoff": "January 2025 (knowledge cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported knowledge cutoff (January 2025) predates the first scored decision on June 30, 2026.",
      "github_run_id": "36075557088",
      "micro_brier": 0.1834496124031008,
      "equal_case_brier": 0.18079850209314494,
      "correct": 302,
      "high_confidence": {
        "count": 320,
        "wrong": 57,
        "mean_confidence": 0.9544374999999989
      },
      "cost": {
        "usd": 59.044026,
        "basis": "estimated",
        "note": "Standard-rate estimate for 91 successful cases. Incomplete cache usage or prices use ordinary-input estimates, which may overstate or understate cache-adjusted costs. Successful workload estimate: $59.0440. Excludes failed attempts and unresolved charges."
      }
    },
    {
      "slug": "gpt-6-1-sol",
      "display_name": "GPT-6.1 Sol",
      "provider": "OpenAI",
      "model_key": "openai:gpt-6.1-sol",
      "access": "OpenAI API",
      "reasoning": "High",
      "release_date": "2026-09-29",
      "training_cutoff": "April 30, 2026 (knowledge cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported knowledge cutoff (April 30, 2026) predates the first scored decision on June 30, 2026.",
      "github_run_id": "36673524599",
      "micro_brier": 0.14947861757105943,
      "equal_case_brier": 0.13885242237920808,
      "correct": 304,
      "high_confidence": {
        "count": 199,
        "wrong": 18,
        "mean_confidence": 0.9565577889447233
      },
      "cost": {
        "usd": 10.677807,
        "basis": "estimated",
        "note": "Standard-rate estimate for 91 successful cases. Incomplete cache usage or prices use ordinary-input estimates, which may overstate or understate cache-adjusted costs. Successful workload estimate: $10.6778. Excludes failed attempts and unresolved charges."
      }
    },
    {
      "slug": "claude-sonnet-5-5",
      "display_name": "Claude Sonnet 5.5",
      "provider": "Anthropic",
      "model_key": "anthropic:claude-sonnet-5-5",
      "access": "Anthropic API",
      "reasoning": "High",
      "release_date": "2026-09-28",
      "training_cutoff": "June 2026 (training-data cutoff)",
      "eligibility": "eligible",
      "eligibility_reason": "The reported training-data cutoff (June 2026) is in the month of the first scored decision (June 30, 2026). It is treated as preceding that decision, because a ruling is very unlikely to enter training data the day it issues.",
      "github_run_id": "36673527185",
      "micro_brier": 0.17297777777777779,
      "equal_case_brier": 0.17183453419596278,
      "correct": 295,
      "high_confidence": {
        "count": 63,
        "wrong": 5,
        "mean_confidence": 0.9157142857142853
      },
      "cost": {
        "usd": 6.992451,
        "basis": "estimated",
        "note": "Standard-rate estimate for 91 successful cases. Retains reported cache usage and cache prices. Successful workload estimate: $6.9925. Excludes failed attempts and unresolved charges."
      }
    },
    {
      "slug": "muse-wonder-1-0",
      "display_name": "Muse Wonder",
      "provider": "Meta",
      "model_key": "vercel_ai_gateway:meta/muse-wonder-1.0",
      "access": "Vercel AI Gateway (served by Meta)",
      "reasoning": "High",
      "release_date": "2026-10-12",
      "training_cutoff": null,
      "eligibility": "qualified",
      "eligibility_reason": "The provider has not published a training or knowledge cutoff. Meta's announcement, model card and developer documentation do not report a training-data or knowledge cutoff.",
      "github_run_id": "38300603621",
      "micro_brier": 0.16535968992248062,
      "equal_case_brier": 0.16900526973026972,
      "correct": 299,
      "high_confidence": {
        "count": 118,
        "wrong": 10,
        "mean_confidence": 0.933644067796611
      },
      "cost": {
        "usd": 57.593452,
        "basis": "standard_rate",
        "note": "Successful workload reported charge: $39.3794; accounting covers 91 of 91 cases. Standard-rate estimate covers all 91 cases. Missing cache read/write dimensions: 132/605 responses; missing cache prices: 0 responses; missing response usage: 0 cases. Successful case receipts only; excludes failed attempts, retries, unresolved charges and summary preparation. These are workload charges or usage estimates, not provider invoice totals. Standard repricing uses ordinary input estimates when cache dimensions or frozen cache rates are missing."
      }
    }
  ]
}