{
  "_about": "Architecture-differential utility finding — SINGLE SOURCE OF TRUTH. Generated by scripts/build-adiff-stats.mjs from huggingface/utility/utility-prereg-aggregate.json (the preregistered eval's primary aggregate). DO NOT HAND-EDIT. The flagship page, the landing-page link, and the HuggingFace card all derive their numbers from this file so they cannot drift (the August stat-drift defect class, fixed at the root).",
  "generated_at": "2026-08-07T14:22:16.254Z",
  "source": "huggingface/utility/utility-prereg-aggregate.json",
  "preregistration": "docs/utility-eval-preregistration.md (locked 2026-06-18)",
  "run_date": "2026-07-15",
  "family_alpha": 0.05,
  "metric": "T = treatment (Atlas-exposed revision) wins, P = placebo (self-reflection) wins, over decided majority-vote instances. One-sided exact binomial + Holm across 5 consumers.",
  "registered_predictions_total": 5,
  "registered_predictions_confirmed": 5,
  "all_confirmed": true,
  "gradient": [
    {
      "model": "GPT-4o",
      "effect": "improved",
      "tier": "positive"
    },
    {
      "model": "Gemini",
      "effect": "improved",
      "tier": "positive"
    },
    {
      "model": "Grok",
      "effect": "no effect",
      "tier": "null"
    },
    {
      "model": "DeepSeek",
      "effect": "no effect",
      "tier": "null"
    },
    {
      "model": "Claude",
      "effect": "degraded",
      "tier": "negative"
    }
  ],
  "consumers": [
    {
      "model": "GPT-4o",
      "registered_prediction": "H1: treatment beats placebo",
      "registered_hypothesis": "H1",
      "tier": "positive",
      "effect": "improved",
      "T": 148,
      "P": 12,
      "decided": 160,
      "win_rate_treatment": 0.925,
      "p_two_sided": 0,
      "p_holm": 0,
      "h3_survives_paraphrase": true,
      "effect_at_both_caps": true,
      "inter_judge_agreement": 0.704,
      "confirmed": true,
      "verdict": "H1 CONFIRMED — significant after Holm, survives paraphrase, holds at both caps"
    },
    {
      "model": "Gemini",
      "registered_prediction": "H1: treatment beats placebo",
      "registered_hypothesis": "H1",
      "tier": "positive",
      "effect": "improved",
      "T": 137,
      "P": 35,
      "decided": 172,
      "win_rate_treatment": 0.797,
      "p_two_sided": 0,
      "p_holm": 0,
      "h3_survives_paraphrase": true,
      "effect_at_both_caps": true,
      "inter_judge_agreement": 0.717,
      "confirmed": true,
      "verdict": "H1 CONFIRMED — significant after Holm, survives paraphrase, holds at both caps"
    },
    {
      "model": "Grok",
      "registered_prediction": "H2: null (no advantage)",
      "registered_hypothesis": "H2",
      "tier": "null",
      "effect": "no effect",
      "T": 72,
      "P": 93,
      "decided": 165,
      "win_rate_treatment": 0.436,
      "p_two_sided": 0.119205,
      "p_holm": 1,
      "h3_survives_paraphrase": false,
      "effect_at_both_caps": false,
      "inter_judge_agreement": 0.629,
      "confirmed": true,
      "verdict": "H2 supported — null as registered"
    },
    {
      "model": "DeepSeek",
      "registered_prediction": "H2: null (no advantage)",
      "registered_hypothesis": "H2",
      "tier": "null",
      "effect": "no effect",
      "T": 90,
      "P": 68,
      "decided": 158,
      "win_rate_treatment": 0.57,
      "p_two_sided": 0.094475,
      "p_holm": 0.141712,
      "h3_survives_paraphrase": false,
      "effect_at_both_caps": false,
      "inter_judge_agreement": 0.628,
      "confirmed": true,
      "verdict": "H2 supported — null as registered"
    },
    {
      "model": "Claude",
      "registered_prediction": "H2: null (no advantage)",
      "registered_hypothesis": "H2",
      "tier": "negative",
      "effect": "degraded",
      "T": 35,
      "P": 126,
      "decided": 161,
      "win_rate_treatment": 0.217,
      "p_two_sided": 0,
      "p_holm": 1,
      "h3_survives_paraphrase": false,
      "effect_at_both_caps": false,
      "inter_judge_agreement": 0.677,
      "confirmed": true,
      "verdict": "H2 supported (no treatment advantage) — NOTE: significant in the REVERSE direction (placebo beat treatment); reported at full strength"
    }
  ],
  "task_type_breakdown": {
    "status": "see analysis/adiff-tasktype-2026-08.md (Work Item A)",
    "note": "Per-task-type breakdown of the Claude degradation lives in the analysis/ file; its mechanism reading is curator-gated and NOT rendered here."
  },
  "refs": {
    "evidence": "utility-evidence-v2.md",
    "preregistration": "docs/utility-eval-preregistration.md",
    "claim": "claims.json#divergence-improves-reasoning"
  }
}