evidence/recipe_quality.json

{
  "schema": "lfm25-quant-reference/2",
  "mode": "int8hu",
  "head_quant": "block32",
  "head_sym": true,
  "int4_block": 32,
  "rescue_regex": null,
  "quant_attn": "keep",
  "quant_embedding": "keep",
  "quality": {
    "teacher_forced_top1_agree": 122,
    "positions": 125,
    "sequences": 5,
    "cosine_min": 0.9970501333100017,
    "cosine_mean": 0.9997748702116448,
    "per_sequence": [
      {
        "case": 0,
        "agree": 20,
        "positions": 21,
        "cosine_min": 0.9970501333100017
      },
      {
        "case": 1,
        "agree": 20,
        "positions": 20,
        "cosine_min": 0.9988472272394007
      },
      {
        "case": 2,
        "agree": 24,
        "positions": 24,
        "cosine_min": 0.9997840496708345
      },
      {
        "case": 3,
        "agree": 27,
        "positions": 28,
        "cosine_min": 0.9995996541432354
      },
      {
        "case": 4,
        "agree": 31,
        "positions": 32,
        "cosine_min": 0.9985697415741762
      }
    ]
  },
  "expected": [
    {
      "prompt": "The capital of France is",
      "input_ids": [
        597,
        5205,
        302,
        3980,
        355
      ],
      "gen_ids": [
        4741,
        22,
        342,
        41,
        514,
        18,
        266,
        496,
        597,
        5205,
        302,
        3980,
        355,
        4741,
        2426,
        342
      ]
    },
    {
      "prompt": "def fibonacci(n):",
      "input_ids": [
        3162,
        116773,
        7946,
        2770
      ],
      "gen_ids": [
        1446,
        395,
        5173,
        401,
        20877,
        316,
        309,
        11615,
        278,
        92189,
        91230,
        1579,
        22,
        116773,
        16,
        24
      ]
    },
    {
      "prompt": "The second law of thermodynamics states that",
      "input_ids": [
        597,
        1840,
        2535,
        302,
        8502,
        47235,
        3977,
        395
      ],
      "gen_ids": [
        278,
        2925,
        35378,
        302,
        401,
        13696,
        1177,
        510,
        2966,
        10135,
        890,
        893,
        22,
        969,
        355,
        267
      ]
    },
    {
      "prompt": "Q: What is 17 * 23?\nA:",
      "input_ids": [
        57,
        34,
        3246,
        355,
        229,
        916,
        767,
        229,
        1467,
        1283,
        41,
        34
      ],
      "gen_ids": [
        229,
        30284,
        8,
        597,
        4695,
        355,
        13564,
        374,
        267,
        3547,
        37207,
        22,
        358,
        1252,
        2675,
        278
      ]
    },
    {
      "prompt": "Once upon a time, in a small village at the edge of a forest,",
      "input_ids": [
        13641,
        3879,
        267,
        893,
        20,
        296,
        267,
        1710,
        6813,
        462,
        278,
        8527,
        302,
        267,
        7421,
        20
      ],
      "gen_ids": [
        912,
        8158,
        267,
        2943,
        5893,
        5167,
        402,
        5451,
        22,
        2298,
        452,
        2037,
        374,
        949,
        41467,
        309
      ]
    }
  ]
}