data/exp_kv_decay/EleutherAI--pythia-2.8b_kv_decay_seq4096.json

{
  "model": "EleutherAI/pythia-2.8b",
  "seq_len": 4096,
  "n_chunks": 30,
  "theta": 10000.0,
  "T_train": 2048,
  "gamma_used": 0.674,
  "gamma_pade_at_T_train": 0.7470064429851826,
  "gamma_pade_at_seq_len": 0.5508312818257147,
  "d_horizon": 2754.0837594601494,
  "D_f_f0_9": 3033,
  "f_retain": 0.9,
  "modes": {
    "baseline": {
      "ppl": 113.53278675197002,
      "nll": 4.732091665267944,
      "std": 0.21647391872688937,
      "delta_ppl": 0.0
    },
    "hard_cutoff": {
      "ppl": 120.23211487669701,
      "nll": 4.789424165089925,
      "std": 0.2085787817992625,
      "delta_ppl": 6.699328124726989
    },
    "soft_decay": {
      "ppl": 118.84341363067055,
      "nll": 4.777806774775187,
      "std": 0.20823771624144086,
      "delta_ppl": 5.310626878700532
    },
    "hard_df": {
      "ppl": 120.91026393710165,
      "nll": 4.7950486501057945,
      "std": 0.21036379187805948,
      "delta_ppl": 7.3774771851316245
    }
  },
  "verdict": "SOFT_DECAY_WINS \u2014 idea 4 v25 confirmed"
}