{
  "direct_report": {
    "final_dir": "/content/yandex_disk_cache/ml/subgroup_distillation/group_e/direct/alexyalunin-rubiobert/final",
    "model_name": "alexyalunin/RuBioBERT",
    "run_slug": "direct_hard_training",
    "selection_metric": "hit@3",
    "selection_metric_value": 0.835,
    "test_metrics": {
      "hit@1": 0.628140703517588,
      "hit@3": 0.8291457286432161,
      "macro_f1": 0.5157875868327818,
      "micro_f1": 0.48293963254593175,
      "mrr": 0.744193071244231,
      "recall@3": 0.8197654941373534,
      "subset_accuracy": 0.1708542713567839,
      "weighted_f1": 0.5126833520048437
    },
    "train_duration_sec": 100.4,
    "train_metrics": {
      "epoch": 9.0,
      "total_flos": 2177031722148864.0,
      "train_loss": 0.47315873063128927,
      "train_runtime": 100.0873,
      "train_samples_per_second": 110.184,
      "train_steps_per_second": 13.788
    },
    "val_metrics": {
      "hit@1": 0.645,
      "hit@3": 0.835,
      "macro_f1": 0.5239739935027187,
      "micro_f1": 0.48812664907651715,
      "mrr": 0.750362070235822,
      "recall@3": 0.8273333333333333,
      "subset_accuracy": 0.175,
      "weighted_f1": 0.5181260346642617
    }
  },
  "direct_test": {
    "hit@1": 0.628140703517588,
    "hit@3": 0.8291457286432161,
    "macro_f1": 0.5157875868327818,
    "micro_f1": 0.48293963254593175,
    "mrr": 0.744193071244231,
    "recall@3": 0.8197654941373534,
    "subset_accuracy": 0.1708542713567839,
    "weighted_f1": 0.5126833520048437
  },
  "direct_val": {
    "hit@1": 0.645,
    "hit@3": 0.835,
    "macro_f1": 0.5239739935027187,
    "micro_f1": 0.48812664907651715,
    "mrr": 0.750362070235822,
    "recall@3": 0.8273333333333333,
    "subset_accuracy": 0.175,
    "weighted_f1": 0.5181260346642617
  },
  "per_label": [
    {
      "f1": 0.3076923076923077,
      "fn": 2,
      "fp": 34,
      "label": "E01",
      "precision": 0.19047619047619047,
      "recall": 0.8,
      "support": 10,
      "tp": 8
    },
    {
      "f1": 0.0,
      "fn": 1,
      "fp": 1,
      "label": "E02",
      "precision": 0.0,
      "recall": 0.0,
      "support": 1,
      "tp": 0
    },
    {
      "f1": 0.15384615384615385,
      "fn": 6,
      "fp": 16,
      "label": "E03",
      "precision": 0.1111111111111111,
      "recall": 0.25,
      "support": 8,
      "tp": 2
    },
    {
      "f1": 0.5,
      "fn": 3,
      "fp": 19,
      "label": "E04",
      "precision": 0.36666666666666664,
      "recall": 0.7857142857142857,
      "support": 14,
      "tp": 11
    },
    {
      "f1": 0.4444444444444444,
      "fn": 5,
      "fp": 10,
      "label": "E05",
      "precision": 0.375,
      "recall": 0.5454545454545454,
      "support": 11,
      "tp": 6
    },
    {
      "f1": 0.5528455284552846,
      "fn": 2,
      "fp": 53,
      "label": "E06",
      "precision": 0.39080459770114945,
      "recall": 0.9444444444444444,
      "support": 36,
      "tp": 34
    },
    {
      "f1": 0.5,
      "fn": 1,
      "fp": 1,
      "label": "E07",
      "precision": 0.5,
      "recall": 0.5,
      "support": 2,
      "tp": 1
    },
    {
      "f1": 0.7058823529411765,
      "fn": 1,
      "fp": 4,
      "label": "E10",
      "precision": 0.6,
      "recall": 0.8571428571428571,
      "support": 7,
      "tp": 6
    },
    {
      "f1": 0.6956521739130435,
      "fn": 0,
      "fp": 14,
      "label": "E11",
      "precision": 0.5333333333333333,
      "recall": 1.0,
      "support": 16,
      "tp": 16
    },
    {
      "f1": 1.0,
      "fn": 0,
      "fp": 0,
      "label": "E12",
      "precision": 1.0,
      "recall": 1.0,
      "support": 2,
      "tp": 2
    },
    {
      "f1": 1.0,
      "fn": 0,
      "fp": 0,
      "label": "E13",
      "precision": 1.0,
      "recall": 1.0,
      "support": 4,
      "tp": 4
    },
    {
      "f1": 1.0,
      "fn": 0,
      "fp": 0,
      "label": "E14",
      "precision": 1.0,
      "recall": 1.0,
      "support": 4,
      "tp": 4
    },
    {
      "f1": 0.0,
      "fn": 0,
      "fp": 0,
      "label": "E16",
      "precision": 0.0,
      "recall": 0.0,
      "support": 0,
      "tp": 0
    },
    {
      "f1": 1.0,
      "fn": 0,
      "fp": 0,
      "label": "E20",
      "precision": 1.0,
      "recall": 1.0,
      "support": 1,
      "tp": 1
    },
    {
      "f1": 0.7272727272727273,
      "fn": 0,
      "fp": 3,
      "label": "E21",
      "precision": 0.5714285714285714,
      "recall": 1.0,
      "support": 4,
      "tp": 4
    },
    {
      "f1": 0.38095238095238093,
      "fn": 1,
      "fp": 12,
      "label": "E22",
      "precision": 0.25,
      "recall": 0.8,
      "support": 5,
      "tp": 4
    },
    {
      "f1": 0.5,
      "fn": 1,
      "fp": 1,
      "label": "E23",
      "precision": 0.5,
      "recall": 0.5,
      "support": 2,
      "tp": 1
    },
    {
      "f1": 0.0,
      "fn": 1,
      "fp": 1,
      "label": "E25",
      "precision": 0.0,
      "recall": 0.0,
      "support": 1,
      "tp": 0
    },
    {
      "f1": 0.4,
      "fn": 2,
      "fp": 1,
      "label": "E27",
      "precision": 0.5,
      "recall": 0.3333333333333333,
      "support": 3,
      "tp": 1
    },
    {
      "f1": 0.4827586206896552,
      "fn": 0,
      "fp": 15,
      "label": "E28",
      "precision": 0.3181818181818182,
      "recall": 1.0,
      "support": 7,
      "tp": 7
    },
    {
      "f1": 0.5,
      "fn": 1,
      "fp": 1,
      "label": "E29",
      "precision": 0.5,
      "recall": 0.5,
      "support": 2,
      "tp": 1
    },
    {
      "f1": 1.0,
      "fn": 0,
      "fp": 0,
      "label": "E30",
      "precision": 1.0,
      "recall": 1.0,
      "support": 1,
      "tp": 1
    },
    {
      "f1": 1.0,
      "fn": 0,
      "fp": 0,
      "label": "E31",
      "precision": 1.0,
      "recall": 1.0,
      "support": 1,
      "tp": 1
    },
    {
      "f1": 0.0,
      "fn": 1,
      "fp": 0,
      "label": "E35",
      "precision": 0.0,
      "recall": 0.0,
      "support": 1,
      "tp": 0
    },
    {
      "f1": 1.0,
      "fn": 0,
      "fp": 0,
      "label": "E43",
      "precision": 1.0,
      "recall": 1.0,
      "support": 1,
      "tp": 1
    },
    {
      "f1": 0.6666666666666666,
      "fn": 1,
      "fp": 0,
      "label": "E44",
      "precision": 1.0,
      "recall": 0.5,
      "support": 2,
      "tp": 1
    },
    {
      "f1": 1.0,
      "fn": 0,
      "fp": 0,
      "label": "E46",
      "precision": 1.0,
      "recall": 1.0,
      "support": 1,
      "tp": 1
    },
    {
      "f1": 1.0,
      "fn": 0,
      "fp": 0,
      "label": "E53",
      "precision": 1.0,
      "recall": 1.0,
      "support": 1,
      "tp": 1
    },
    {
      "f1": 0.0,
      "fn": 1,
      "fp": 0,
      "label": "E55",
      "precision": 0.0,
      "recall": 0.0,
      "support": 1,
      "tp": 0
    },
    {
      "f1": 0.6666666666666666,
      "fn": 2,
      "fp": 0,
      "label": "E61",
      "precision": 1.0,
      "recall": 0.5,
      "support": 4,
      "tp": 2
    },
    {
      "f1": 0.5806451612903226,
      "fn": 5,
      "fp": 21,
      "label": "E66",
      "precision": 0.46153846153846156,
      "recall": 0.782608695652174,
      "support": 23,
      "tp": 18
    },
    {
      "f1": 0.6666666666666666,
      "fn": 0,
      "fp": 1,
      "label": "E67",
      "precision": 0.5,
      "recall": 1.0,
      "support": 1,
      "tp": 1
    },
    {
      "f1": 0.5,
      "fn": 1,
      "fp": 1,
      "label": "E70",
      "precision": 0.5,
      "recall": 0.5,
      "support": 2,
      "tp": 1
    },
    {
      "f1": 0.6666666666666666,
      "fn": 0,
      "fp": 7,
      "label": "E71",
      "precision": 0.5,
      "recall": 1.0,
      "support": 7,
      "tp": 7
    },
    {
      "f1": 0.8,
      "fn": 0,
      "fp": 1,
      "label": "E72",
      "precision": 0.6666666666666666,
      "recall": 1.0,
      "support": 2,
      "tp": 2
    },
    {
      "f1": 0.5,
      "fn": 2,
      "fp": 4,
      "label": "E74",
      "precision": 0.42857142857142855,
      "recall": 0.6,
      "support": 5,
      "tp": 3
    },
    {
      "f1": 0.9090909090909091,
      "fn": 0,
      "fp": 1,
      "label": "E75",
      "precision": 0.8333333333333334,
      "recall": 1.0,
      "support": 5,
      "tp": 5
    },
    {
      "f1": 0.8,
      "fn": 0,
      "fp": 3,
      "label": "E76",
      "precision": 0.6666666666666666,
      "recall": 1.0,
      "support": 6,
      "tp": 6
    },
    {
      "f1": 0.0,
      "fn": 1,
      "fp": 0,
      "label": "E78",
      "precision": 0.0,
      "recall": 0.0,
      "support": 1,
      "tp": 0
    },
    {
      "f1": 0.8571428571428571,
      "fn": 0,
      "fp": 1,
      "label": "E79",
      "precision": 0.75,
      "recall": 1.0,
      "support": 3,
      "tp": 3
    },
    {
      "f1": 1.0,
      "fn": 0,
      "fp": 0,
      "label": "E80",
      "precision": 1.0,
      "recall": 1.0,
      "support": 1,
      "tp": 1
    },
    {
      "f1": 0.5,
      "fn": 2,
      "fp": 0,
      "label": "E83",
      "precision": 1.0,
      "recall": 0.3333333333333333,
      "support": 3,
      "tp": 1
    },
    {
      "f1": 1.0,
      "fn": 0,
      "fp": 0,
      "label": "E84",
      "precision": 1.0,
      "recall": 1.0,
      "support": 1,
      "tp": 1
    },
    {
      "f1": 1.0,
      "fn": 0,
      "fp": 0,
      "label": "E85",
      "precision": 1.0,
      "recall": 1.0,
      "support": 1,
      "tp": 1
    },
    {
      "f1": 0.5833333333333334,
      "fn": 1,
      "fp": 9,
      "label": "E89",
      "precision": 0.4375,
      "recall": 0.875,
      "support": 8,
      "tp": 7
    }
  ],
  "quality_gate": {
    "direct_hit3_threshold": 0.9,
    "force_push_weak": true,
    "hit3_drop_tolerance": 0.02,
    "hit3_floor": 0.8242211055276382,
    "macro_f1_drop_tolerance": 0.03,
    "macro_floor": 0.65047534381331,
    "min_macro_f1_for_push": 0.3,
    "passed": false,
    "push_allowed": true,
    "push_policy": "always_push_when_push_to_hub_enabled"
  },
  "teacher_errors": {},
  "teacher_reports": [
    {
      "final_dir": "/content/yandex_disk_cache/ml/subgroup_distillation/group_e/teachers/alexyalunin-rubioroberta/final",
      "model_name": "alexyalunin/RuBioRoBERTa",
      "pushed_to_hub": false,
      "run_slug": "alexyalunin-rubioroberta",
      "test_metrics": {
        "hit@1": 0.6482412060301508,
        "hit@3": 0.8592964824120602,
        "macro_f1": 0.6865864313407604,
        "micro_f1": 0.6485436893203883,
        "mrr": 0.7635155763702912,
        "recall@3": 0.8532663316582914,
        "subset_accuracy": 0.507537688442211,
        "weighted_f1": 0.6514279861261107
      },
      "train_duration_sec": 314.8,
      "train_metrics": {
        "epoch": 12.0,
        "total_flos": 1.0278832248336384e+16,
        "train_loss": 0.31829168511473616,
        "train_runtime": 299.4665,
        "train_samples_per_second": 36.825,
        "train_steps_per_second": 4.608
      },
      "val_metrics": {
        "hit@1": 0.665,
        "hit@3": 0.83,
        "macro_f1": 0.6816071208348493,
        "micro_f1": 0.6377358490566037,
        "mrr": 0.7651676992824679,
        "recall@3": 0.8214999999999999,
        "subset_accuracy": 0.48,
        "weighted_f1": 0.654032532287305
      }
    },
    {
      "final_dir": "/content/yandex_disk_cache/ml/subgroup_distillation/group_e/teachers/ai-forever-rubert-base/final",
      "model_name": "ai-forever/ruBert-base",
      "pushed_to_hub": false,
      "run_slug": "ai-forever-rubert-base",
      "test_metrics": {
        "hit@1": 0.6633165829145728,
        "hit@3": 0.8241206030150754,
        "macro_f1": 0.567726321569097,
        "micro_f1": 0.5497630331753555,
        "mrr": 0.7552527410980668,
        "recall@3": 0.8164154103852594,
        "subset_accuracy": 0.3015075376884422,
        "weighted_f1": 0.5636139940062055
      },
      "train_duration_sec": 144.6,
      "train_metrics": {
        "epoch": 12.0,
        "total_flos": 2902708962865152.0,
        "train_loss": 0.4238858779271444,
        "train_runtime": 133.7006,
        "train_samples_per_second": 82.483,
        "train_steps_per_second": 10.322
      },
      "val_metrics": {
        "hit@1": 0.65,
        "hit@3": 0.82,
        "macro_f1": 0.5512155779574014,
        "micro_f1": 0.5529953917050692,
        "mrr": 0.7548125116338592,
        "recall@3": 0.814,
        "subset_accuracy": 0.315,
        "weighted_f1": 0.5726346327425935
      }
    },
    {
      "final_dir": "/content/yandex_disk_cache/ml/subgroup_distillation/group_e/teachers/deeppavlov-rubert-base-cased/final",
      "model_name": "DeepPavlov/rubert-base-cased",
      "pushed_to_hub": false,
      "run_slug": "deeppavlov-rubert-base-cased",
      "test_metrics": {
        "hit@1": 0.6130653266331658,
        "hit@3": 0.7939698492462312,
        "macro_f1": 0.5623096108520506,
        "micro_f1": 0.5097451274362819,
        "mrr": 0.7264908372260924,
        "recall@3": 0.7879396984924623,
        "subset_accuracy": 0.2613065326633166,
        "weighted_f1": 0.5247350484683884
      },
      "train_duration_sec": 145.8,
      "train_metrics": {
        "epoch": 12.0,
        "total_flos": 2902708962865152.0,
        "train_loss": 0.45112168028734734,
        "train_runtime": 136.666,
        "train_samples_per_second": 80.693,
        "train_steps_per_second": 10.098
      },
      "val_metrics": {
        "hit@1": 0.66,
        "hit@3": 0.81,
        "macro_f1": 0.5291229985764946,
        "micro_f1": 0.5123010130246021,
        "mrr": 0.755827678749699,
        "recall@3": 0.8014999999999999,
        "subset_accuracy": 0.23,
        "weighted_f1": 0.5291843956354477
      }
    }
  ],
  "teacher_test": {
    "hit@1": 0.678391959798995,
    "hit@3": 0.8442211055276382,
    "macro_f1": 0.68047534381331,
    "micro_f1": 0.6541353383458647,
    "mrr": 0.7735248354904848,
    "recall@3": 0.8381909547738693,
    "subset_accuracy": 0.4824120603015075,
    "weighted_f1": 0.6548964986426676
  },
  "teacher_val": {
    "hit@1": 0.675,
    "hit@3": 0.84,
    "macro_f1": 0.6592846608011086,
    "micro_f1": 0.6483516483516484,
    "mrr": 0.7742923534798536,
    "recall@3": 0.834,
    "subset_accuracy": 0.48,
    "weighted_f1": 0.6615922586454719
  },
  "test": {
    "hit@1": 0.6482412060301508,
    "hit@3": 0.8241206030150754,
    "macro_f1": 0.6121827915051391,
    "micro_f1": 0.5606299212598426,
    "mrr": 0.7515661740428731,
    "recall@3": 0.8147403685092127,
    "subset_accuracy": 0.2964824120603015,
    "weighted_f1": 0.5830428669342762
  },
  "threshold": 0.5,
  "val_best": {
    "hit@1": 0.635,
    "hit@3": 0.8,
    "macro_f1": 0.6257847034784294,
    "micro_f1": 0.5578446909667195,
    "mrr": 0.7417874066293184,
    "recall@3": 0.7939999999999999,
    "subset_accuracy": 0.3,
    "weighted_f1": 0.581422089009702
  }
}