data/exp_kv_decay/EleutherAI--pythia-1.4b_kv_decay_seq4096.json

{
  "model": "EleutherAI/pythia-1.4b",
  "seq_len": 4096,
  "n_chunks": 30,
  "theta": 10000.0,
  "T_train": 2048,
  "gamma_used": 0.705,
  "gamma_pade_at_T_train": 0.7470064429851826,
  "gamma_pade_at_seq_len": 0.5508312818257147,
  "d_horizon": 2446.879770674857,
  "D_f_f0_9": 2960,
  "f_retain": 0.9,
  "modes": {
    "baseline": {
      "ppl": 53.65674917940454,
      "nll": 3.9826072613398233,
      "std": 0.2010077981665634,
      "delta_ppl": 0.0
    },
    "hard_cutoff": {
      "ppl": 31.65464082972793,
      "nll": 3.4548847675323486,
      "std": 0.206414518381513,
      "delta_ppl": -22.00210834967661
    },
    "soft_decay": {
      "ppl": 25.16093004937277,
      "nll": 3.2252923965454103,
      "std": 0.19782993796895576,
      "delta_ppl": -28.495819130031773
    },
    "hard_df": {
      "ppl": 48.386144400553235,
      "nll": 3.879213500022888,
      "std": 0.18416963648784698,
      "delta_ppl": -5.2706047788513075
    }
  },
  "verdict": "SOFT_DECAY_WINS \u2014 idea 4 v25 confirmed"
}