{
  "schema": "MAKNOW_AIE004_LAB_FIXTURES_v1",
  "cases": [
    {
      "id": "baseline_all_pass",
      "candidate": {
        "candidate_id": "candidate-atlas",
        "model_version": "atlas-v1",
        "configuration_version": "cfg-v1",
        "prompt_version": "prompt-v1"
      },
      "dataset_version": "evalset-v1",
      "judge_tolerance": 1.0,
      "required_slices": [
        "extract_en_low",
        "extract_fa_low",
        "sum_en_high",
        "sum_fa_high"
      ],
      "rows": [
        {
          "slice_id": "extract_en_low",
          "task": "structured_extraction",
          "locale": "en-US",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.86,
          "human_rating": 4.4,
          "judge_rating": 4.3,
          "hard_gate_pass": true
        },
        {
          "slice_id": "extract_fa_low",
          "task": "structured_extraction",
          "locale": "fa-IR",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.84,
          "human_rating": 4.2,
          "judge_rating": 4.1,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_en_high",
          "task": "grounded_summarization",
          "locale": "en-US",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.88,
          "human_rating": 4.5,
          "judge_rating": 4.4,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_fa_high",
          "task": "grounded_summarization",
          "locale": "fa-IR",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.82,
          "human_rating": 4.1,
          "judge_rating": 4.0,
          "hard_gate_pass": true
        }
      ]
    },
    {
      "id": "aggregate_trap_fa_high",
      "candidate": {
        "candidate_id": "candidate-atlas",
        "model_version": "atlas-v1",
        "configuration_version": "cfg-v1",
        "prompt_version": "prompt-v1"
      },
      "dataset_version": "evalset-v1",
      "judge_tolerance": 1.0,
      "required_slices": [
        "extract_en_low",
        "extract_fa_low",
        "sum_en_high",
        "sum_fa_high"
      ],
      "rows": [
        {
          "slice_id": "extract_en_low",
          "task": "structured_extraction",
          "locale": "en-US",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.98,
          "human_rating": 4.8,
          "judge_rating": 4.8,
          "hard_gate_pass": true
        },
        {
          "slice_id": "extract_fa_low",
          "task": "structured_extraction",
          "locale": "fa-IR",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.96,
          "human_rating": 4.7,
          "judge_rating": 4.6,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_en_high",
          "task": "grounded_summarization",
          "locale": "en-US",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.95,
          "human_rating": 4.7,
          "judge_rating": 4.6,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_fa_high",
          "task": "grounded_summarization",
          "locale": "fa-IR",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.93,
          "human_rating": 4.6,
          "judge_rating": 4.6,
          "hard_gate_pass": false
        }
      ]
    },
    {
      "id": "judge_human_disagreement",
      "candidate": {
        "candidate_id": "candidate-atlas",
        "model_version": "atlas-v1",
        "configuration_version": "cfg-v1",
        "prompt_version": "prompt-v1"
      },
      "dataset_version": "evalset-v1",
      "judge_tolerance": 1.0,
      "required_slices": [
        "extract_en_low",
        "extract_fa_low",
        "sum_en_high",
        "sum_fa_high"
      ],
      "rows": [
        {
          "slice_id": "extract_en_low",
          "task": "structured_extraction",
          "locale": "en-US",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.9,
          "human_rating": 4.5,
          "judge_rating": 4.4,
          "hard_gate_pass": true
        },
        {
          "slice_id": "extract_fa_low",
          "task": "structured_extraction",
          "locale": "fa-IR",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.88,
          "human_rating": 2.0,
          "judge_rating": 4.5,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_en_high",
          "task": "grounded_summarization",
          "locale": "en-US",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.87,
          "human_rating": 4.3,
          "judge_rating": 4.2,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_fa_high",
          "task": "grounded_summarization",
          "locale": "fa-IR",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.86,
          "human_rating": 4.2,
          "judge_rating": 4.1,
          "hard_gate_pass": true
        }
      ]
    },
    {
      "id": "missing_persian_high_risk",
      "candidate": {
        "candidate_id": "candidate-atlas",
        "model_version": "atlas-v1",
        "configuration_version": "cfg-v1",
        "prompt_version": "prompt-v1"
      },
      "dataset_version": "evalset-v1",
      "judge_tolerance": 1.0,
      "required_slices": [
        "extract_en_low",
        "extract_fa_low",
        "sum_en_high",
        "sum_fa_high"
      ],
      "rows": [
        {
          "slice_id": "extract_en_low",
          "task": "structured_extraction",
          "locale": "en-US",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.89,
          "human_rating": 4.4,
          "judge_rating": 4.3,
          "hard_gate_pass": true
        },
        {
          "slice_id": "extract_fa_low",
          "task": "structured_extraction",
          "locale": "fa-IR",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.85,
          "human_rating": 4.2,
          "judge_rating": 4.1,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_en_high",
          "task": "grounded_summarization",
          "locale": "en-US",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.9,
          "human_rating": 4.5,
          "judge_rating": 4.4,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_fa_high",
          "task": "grounded_summarization",
          "locale": "fa-IR",
          "risk": "high",
          "evidence_state": "UNKNOWN",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": null,
          "human_rating": null,
          "judge_rating": null,
          "hard_gate_pass": true
        }
      ]
    },
    {
      "id": "schema_hard_gate_failure",
      "candidate": {
        "candidate_id": "candidate-atlas",
        "model_version": "atlas-v1",
        "configuration_version": "cfg-v1",
        "prompt_version": "prompt-v1"
      },
      "dataset_version": "evalset-v1",
      "judge_tolerance": 1.0,
      "required_slices": [
        "extract_en_low",
        "extract_fa_low",
        "sum_en_high",
        "sum_fa_high"
      ],
      "rows": [
        {
          "slice_id": "extract_en_low",
          "task": "structured_extraction",
          "locale": "en-US",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.91,
          "human_rating": 4.5,
          "judge_rating": 4.4,
          "hard_gate_pass": true
        },
        {
          "slice_id": "extract_fa_low",
          "task": "structured_extraction",
          "locale": "fa-IR",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.9,
          "human_rating": 4.4,
          "judge_rating": 4.4,
          "hard_gate_pass": false
        },
        {
          "slice_id": "sum_en_high",
          "task": "grounded_summarization",
          "locale": "en-US",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.88,
          "human_rating": 4.3,
          "judge_rating": 4.2,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_fa_high",
          "task": "grounded_summarization",
          "locale": "fa-IR",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.87,
          "human_rating": 4.2,
          "judge_rating": 4.2,
          "hard_gate_pass": true
        }
      ]
    },
    {
      "id": "qualified_lower_soft_mean",
      "candidate": {
        "candidate_id": "candidate-atlas",
        "model_version": "atlas-v1",
        "configuration_version": "cfg-v1",
        "prompt_version": "prompt-v1"
      },
      "dataset_version": "evalset-v1",
      "judge_tolerance": 1.0,
      "required_slices": [
        "extract_en_low",
        "extract_fa_low",
        "sum_en_high",
        "sum_fa_high"
      ],
      "rows": [
        {
          "slice_id": "extract_en_low",
          "task": "structured_extraction",
          "locale": "en-US",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.74,
          "human_rating": 3.8,
          "judge_rating": 3.8,
          "hard_gate_pass": true
        },
        {
          "slice_id": "extract_fa_low",
          "task": "structured_extraction",
          "locale": "fa-IR",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.73,
          "human_rating": 3.7,
          "judge_rating": 3.7,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_en_high",
          "task": "grounded_summarization",
          "locale": "en-US",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.72,
          "human_rating": 3.6,
          "judge_rating": 3.6,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_fa_high",
          "task": "grounded_summarization",
          "locale": "fa-IR",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.71,
          "human_rating": 3.6,
          "judge_rating": 3.5,
          "hard_gate_pass": true
        }
      ]
    },
    {
      "id": "prompt_version_changed",
      "candidate": {
        "candidate_id": "candidate-atlas",
        "model_version": "atlas-v1",
        "configuration_version": "cfg-v1",
        "prompt_version": "prompt-v2"
      },
      "dataset_version": "evalset-v1",
      "judge_tolerance": 1.0,
      "required_slices": [
        "extract_en_low",
        "extract_fa_low",
        "sum_en_high",
        "sum_fa_high"
      ],
      "rows": [
        {
          "slice_id": "extract_en_low",
          "task": "structured_extraction",
          "locale": "en-US",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v2",
          "quality_score": 0.9,
          "human_rating": 4.4,
          "judge_rating": 4.3,
          "hard_gate_pass": true
        },
        {
          "slice_id": "extract_fa_low",
          "task": "structured_extraction",
          "locale": "fa-IR",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v2",
          "quality_score": 0.86,
          "human_rating": 4.2,
          "judge_rating": 4.1,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_en_high",
          "task": "grounded_summarization",
          "locale": "en-US",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v2",
          "quality_score": 0.88,
          "human_rating": 4.3,
          "judge_rating": 4.2,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_fa_high",
          "task": "grounded_summarization",
          "locale": "fa-IR",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v1",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.84,
          "human_rating": 4.1,
          "judge_rating": 4.0,
          "hard_gate_pass": true
        }
      ]
    },
    {
      "id": "transfer_model_v2_refusal_change",
      "candidate": {
        "candidate_id": "candidate-atlas",
        "model_version": "atlas-v2",
        "configuration_version": "cfg-v1",
        "prompt_version": "prompt-v1"
      },
      "dataset_version": "evalset-v1",
      "judge_tolerance": 1.0,
      "required_slices": [
        "extract_en_low",
        "extract_fa_low",
        "sum_en_high",
        "sum_fa_high"
      ],
      "rows": [
        {
          "slice_id": "extract_en_low",
          "task": "structured_extraction",
          "locale": "en-US",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v2",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.95,
          "human_rating": 4.7,
          "judge_rating": 4.6,
          "hard_gate_pass": true
        },
        {
          "slice_id": "extract_fa_low",
          "task": "structured_extraction",
          "locale": "fa-IR",
          "risk": "low",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v2",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.92,
          "human_rating": 4.5,
          "judge_rating": 4.4,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_en_high",
          "task": "grounded_summarization",
          "locale": "en-US",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v2",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.94,
          "human_rating": 4.6,
          "judge_rating": 4.5,
          "hard_gate_pass": true
        },
        {
          "slice_id": "sum_fa_high",
          "task": "grounded_summarization",
          "locale": "fa-IR",
          "risk": "high",
          "evidence_state": "OBSERVED",
          "model_version": "atlas-v2",
          "configuration_version": "cfg-v1",
          "prompt_version": "prompt-v1",
          "quality_score": 0.89,
          "human_rating": 4.0,
          "judge_rating": 4.0,
          "hard_gate_pass": false
        }
      ]
    }
  ]
}