{
  "kind": "controlled_fixture_no_real_models",
  "reports": [
    {
      "candidate": "prompt_v2",
      "cases": 12,
      "statistics": {
        "estimand": "equal_weight_mean_of_project_task_success_differences",
        "paired_groups": 6,
        "cases": 12,
        "difference": 0.25,
        "percentile_interval_95": [
          0.08333333333333333,
          0.4166666666666667
        ],
        "seed": 17,
        "resamples": 2000,
        "warning": "Synthetic small fixture: method demonstration, not population evidence or a model ranking"
      },
      "gate": {
        "passed": true,
        "reasons": [],
        "policy": "fixture-gate-v1",
        "noninferiority_margin": -0.05
      },
      "failure_count": 0,
      "budget": {
        "limit_fixture_units": 2000000,
        "spent_fixture_units": 10542,
        "unknown_reservations": 0,
        "unit": "public task JSON bytes plus candidate response JSON bytes; not model tokens or money"
      }
    },
    {
      "candidate": "retrieval_v2",
      "cases": 12,
      "statistics": {
        "estimand": "equal_weight_mean_of_project_task_success_differences",
        "paired_groups": 6,
        "cases": 12,
        "difference": -0.5,
        "percentile_interval_95": [
          -0.5,
          -0.5
        ],
        "seed": 17,
        "resamples": 2000,
        "warning": "Synthetic small fixture: method demonstration, not population evidence or a model ranking"
      },
      "gate": {
        "passed": false,
        "reasons": [
          "new_task_regression",
          "new_answer_quality_regression",
          "paired_interval_below_noninferiority_margin"
        ],
        "policy": "fixture-gate-v1",
        "noninferiority_margin": -0.05
      },
      "failure_count": 9,
      "budget": {
        "limit_fixture_units": 2000000,
        "spent_fixture_units": 10506,
        "unknown_reservations": 0,
        "unit": "public task JSON bytes plus candidate response JSON bytes; not model tokens or money"
      }
    },
    {
      "candidate": "model_v2",
      "cases": 12,
      "statistics": {
        "estimand": "equal_weight_mean_of_project_task_success_differences",
        "paired_groups": 6,
        "cases": 12,
        "difference": -0.75,
        "percentile_interval_95": [
          -0.9166666666666666,
          -0.5833333333333334
        ],
        "seed": 17,
        "resamples": 2000,
        "warning": "Synthetic small fixture: method demonstration, not population evidence or a model ranking"
      },
      "gate": {
        "passed": false,
        "reasons": [
          "new_task_regression",
          "paired_interval_below_noninferiority_margin"
        ],
        "policy": "fixture-gate-v1",
        "noninferiority_margin": -0.05
      },
      "failure_count": 12,
      "budget": {
        "limit_fixture_units": 2000000,
        "spent_fixture_units": 10518,
        "unknown_reservations": 0,
        "unit": "public task JSON bytes plus candidate response JSON bytes; not model tokens or money"
      }
    }
  ],
  "calibration": {
    "kind": "authored_counterexamples_not_human_or_model_judge_measurements",
    "confusion": {
      "naive": {
        "tp": 2,
        "fp": 2,
        "tn": 2,
        "fn": 0
      },
      "contract": {
        "tp": 2,
        "fp": 0,
        "tn": 4,
        "fn": 0
      }
    },
    "naive_false_positive_rate": 0.5,
    "cases": [
      {
        "id": "file-written",
        "naive_pass": true,
        "contract_pass": true,
        "reference_pass": true,
        "score_detail": {
          "task_pass": true,
          "quality_pass": true,
          "safety_pass": true,
          "review_pending": false,
          "reason": "artifact_matches"
        }
      },
      {
        "id": "claims-done-file-unchanged",
        "naive_pass": true,
        "contract_pass": false,
        "reference_pass": false,
        "score_detail": {
          "task_pass": false,
          "quality_pass": true,
          "safety_pass": true,
          "review_pending": false,
          "reason": "artifact_mismatch"
        }
      },
      {
        "id": "correct-value-wrong-citation",
        "naive_pass": true,
        "contract_pass": false,
        "reference_pass": false,
        "score_detail": {
          "task_pass": false,
          "quality_pass": true,
          "safety_pass": true,
          "review_pending": false,
          "reason": "wrong_fact_or_citation"
        }
      },
      {
        "id": "empty-answer",
        "naive_pass": false,
        "contract_pass": false,
        "reference_pass": false,
        "score_detail": {
          "task_pass": true,
          "quality_pass": false,
          "safety_pass": true,
          "review_pending": false,
          "reason": "supported_fact"
        }
      },
      {
        "id": "supported-answer",
        "naive_pass": true,
        "contract_pass": true,
        "reference_pass": true,
        "score_detail": {
          "task_pass": true,
          "quality_pass": true,
          "safety_pass": true,
          "review_pending": false,
          "reason": "supported_fact"
        }
      },
      {
        "id": "explicit-failure",
        "naive_pass": false,
        "contract_pass": false,
        "reference_pass": false,
        "score_detail": {
          "task_pass": false,
          "quality_pass": true,
          "safety_pass": true,
          "review_pending": false,
          "reason": "artifact_mismatch"
        }
      }
    ]
  },
  "checked_at": "2026-10-21T08:40:58.455391+00:00",
  "python": "3.14.3",
  "tests": {
    "command": "python3 -m unittest discover -s tests -v",
    "count": 17,
    "result": "passed"
  },
  "commands": [
    "python3 -m lab.make_data",
    "python3 -m unittest discover -s tests -v",
    "python3 -m lab.demo"
  ]
}