{
 "episodes": [
  {
   "episode": 1,
   "loss": "0.1168",
   "reward": "-0.06",
   "reward_std": "0.05",
   "kl": "1.487",
   "completion_length": "121.2",
   "clipped_ratio": "0",
   "learning_rate": "0",
   "grad_norm": "1.441",
   "epoch": "0.002"
  },
  {
   "episode": 2,
   "loss": "0.03414",
   "reward": "-0.085",
   "reward_std": "0",
   "kl": "1.728",
   "completion_length": "82",
   "clipped_ratio": "0",
   "learning_rate": "1e-06",
   "grad_norm": "0.06773",
   "epoch": "0.004"
  },
  {
   "episode": 3,
   "loss": "0.03677",
   "reward": "-0.035",
   "reward_std": "0.05774",
   "kl": "1.472",
   "completion_length": "110.5",
   "clipped_ratio": "0",
   "learning_rate": "2e-06",
   "grad_norm": "1.649",
   "epoch": "0.006"
  },
  {
   "episode": 4,
   "loss": "0.1546",
   "reward": "-0.06",
   "reward_std": "0.05",
   "kl": "1.565",
   "completion_length": "97.25",
   "clipped_ratio": "0",
   "learning_rate": "3e-06",
   "grad_norm": "1.336",
   "epoch": "0.008"
  },
  {
   "episode": 5,
   "loss": "0.0292",
   "reward": "-0.085",
   "reward_std": "0",
   "kl": "1.48",
   "completion_length": "105.8",
   "clipped_ratio": "0",
   "learning_rate": "4e-06",
   "grad_norm": "0.05263",
   "epoch": "0.01"
  },
  {
   "episode": 6,
   "loss": "-0.05279",
   "reward": "-0.06",
   "reward_std": "0.05",
   "kl": "1.577",
   "completion_length": "113",
   "clipped_ratio": "0",
   "learning_rate": "5e-06",
   "grad_norm": "1.791",
   "epoch": "0.012"
  },
  {
   "episode": 7,
   "loss": "0.1339",
   "reward": "-0.06",
   "reward_std": "0.05",
   "kl": "1.539",
   "completion_length": "126.2",
   "clipped_ratio": "0",
   "learning_rate": "6e-06",
   "grad_norm": "1.452",
   "epoch": "0.014"
  },
  {
   "episode": 8,
   "loss": "0.03526",
   "reward": "-0.085",
   "reward_std": "0",
   "kl": "1.845",
   "completion_length": "89",
   "clipped_ratio": "0",
   "learning_rate": "7e-06",
   "grad_norm": "0.1091",
   "epoch": "0.016"
  },
  {
   "episode": 9,
   "loss": "-0.03297",
   "reward": "-0.03875",
   "reward_std": "0.05375",
   "kl": "1.484",
   "completion_length": "114.5",
   "clipped_ratio": "0",
   "learning_rate": "8e-06",
   "grad_norm": "1.834",
   "epoch": "0.018"
  },
  {
   "episode": 10,
   "loss": "0.02409",
   "reward": "-0.085",
   "reward_std": "0",
   "kl": "1.3",
   "completion_length": "158",
   "clipped_ratio": "0",
   "learning_rate": "9e-06",
   "grad_norm": "0.08162",
   "epoch": "0.02"
  },
  {
   "episode": 11,
   "loss": "-0.07952",
   "reward": "-0.035",
   "reward_std": "0.05774",
   "kl": "1.526",
   "completion_length": "114.5",
   "clipped_ratio": "0",
   "learning_rate": "1e-05",
   "grad_norm": "1.697",
   "epoch": "0.022"
  },
  {
   "episode": 12,
   "loss": "0.04861",
   "reward": "-0.035",
   "reward_std": "0.05774",
   "kl": "1.59",
   "completion_length": "99",
   "clipped_ratio": "0",
   "learning_rate": "1.1e-05",
   "grad_norm": "1.777",
   "epoch": "0.024"
  },
  {
   "episode": 13,
   "loss": "0.09983",
   "reward": "-0.03875",
   "reward_std": "0.06237",
   "kl": "1.888",
   "completion_length": "93.75",
   "clipped_ratio": "0",
   "learning_rate": "1.2e-05",
   "grad_norm": "2.288",
   "epoch": "0.026"
  },
  {
   "episode": 14,
   "loss": "0.1242",
   "reward": "-0.06",
   "reward_std": "0.05",
   "kl": "1.648",
   "completion_length": "112.8",
   "clipped_ratio": "0",
   "learning_rate": "1.3e-05",
   "grad_norm": "1.58",
   "epoch": "0.028"
  },
  {
   "episode": 15,
   "loss": "0.04319",
   "reward": "-0.06",
   "reward_std": "0.05",
   "kl": "1.545",
   "completion_length": "100.5",
   "clipped_ratio": "0",
   "learning_rate": "1.4e-05",
   "grad_norm": "1.571",
   "epoch": "0.03"
  },
  {
   "episode": 16,
   "loss": "0.03368",
   "reward": "-0.085",
   "reward_std": "0",
   "kl": "1.721",
   "completion_length": "82",
   "clipped_ratio": "0",
   "learning_rate": "1.5e-05",
   "grad_norm": "0.1079",
   "epoch": "0.032"
  },
  {
   "episode": 17,
   "loss": "0.03411",
   "reward": "-0.085",
   "reward_std": "0",
   "kl": "1.737",
   "completion_length": "96",
   "clipped_ratio": "0",
   "learning_rate": "1.6e-05",
   "grad_norm": "0.05412",
   "epoch": "0.034"
  },
  {
   "episode": 18,
   "loss": "0.02681",
   "reward": "-0.085",
   "reward_std": "0",
   "kl": "1.425",
   "completion_length": "116.2",
   "clipped_ratio": "0",
   "learning_rate": "1.7e-05",
   "grad_norm": "0.09016",
   "epoch": "0.036"
  },
  {
   "episode": 19,
   "loss": "-0.1867",
   "reward": "0.1367",
   "reward_std": "0.4434",
   "kl": "1.459",
   "completion_length": "138.5",
   "clipped_ratio": "0",
   "learning_rate": "1.8e-05",
   "grad_norm": "1.965",
   "epoch": "0.038"
  },
  {
   "episode": 20,
   "loss": "0.02446",
   "reward": "-0.03875",
   "reward_std": "0.06237",
   "kl": "1.662",
   "completion_length": "97.75",
   "clipped_ratio": "0",
   "learning_rate": "1.9e-05",
   "grad_norm": "1.648",
   "epoch": "0.04"
  },
  {
   "episode": 21,
   "loss": "0.02844",
   "reward": "-0.085",
   "reward_std": "0",
   "kl": "1.462",
   "completion_length": "103.5",
   "clipped_ratio": "0",
   "learning_rate": "2e-05",
   "grad_norm": "0.05372",
   "epoch": "0.042"
  },
  {
   "episode": 22,
   "loss": "0.1425",
   "reward": "-0.06375",
   "reward_std": "0.05297",
   "kl": "1.547",
   "completion_length": "140",
   "clipped_ratio": "0",
   "learning_rate": "2.1e-05",
   "grad_norm": "1.714",
   "epoch": "0.044"
  },
  {
   "episode": 23,
   "loss": "0.06482",
   "reward": "0.1825",
   "reward_std": "0.4707",
   "kl": "1.429",
   "completion_length": "101.2",
   "clipped_ratio": "0",
   "learning_rate": "2.2e-05",
   "grad_norm": "1.443",
   "epoch": "0.046"
  },
  {
   "episode": 24,
   "loss": "0.03516",
   "reward": "-0.035",
   "reward_std": "0.05774",
   "kl": "1.529",
   "completion_length": "93.5",
   "clipped_ratio": "0",
   "learning_rate": "2.3e-05",
   "grad_norm": "1.834",
   "epoch": "0.048"
  },
  {
   "episode": 25,
   "loss": "0.2135",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.349",
   "completion_length": "140.5",
   "clipped_ratio": "0",
   "learning_rate": "2.4e-05",
   "grad_norm": "1.748",
   "epoch": "0.05"
  },
  {
   "episode": 26,
   "loss": "-0.0988",
   "reward": "-0.03875",
   "reward_std": "0.05375",
   "kl": "1.452",
   "completion_length": "110.8",
   "clipped_ratio": "0",
   "learning_rate": "2.5e-05",
   "grad_norm": "1.824",
   "epoch": "0.052"
  },
  {
   "episode": 27,
   "loss": "0.08622",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.599",
   "completion_length": "92",
   "clipped_ratio": "0",
   "learning_rate": "2.6e-05",
   "grad_norm": "1.645",
   "epoch": "0.054"
  },
  {
   "episode": 28,
   "loss": "0.2779",
   "reward": "-0.035",
   "reward_std": "0.05774",
   "kl": "1.387",
   "completion_length": "158",
   "clipped_ratio": "0",
   "learning_rate": "2.7e-05",
   "grad_norm": "1.36",
   "epoch": "0.056"
  },
  {
   "episode": 29,
   "loss": "0.03327",
   "reward": "-0.085",
   "reward_std": "0",
   "kl": "1.678",
   "completion_length": "83",
   "clipped_ratio": "0",
   "learning_rate": "2.8e-05",
   "grad_norm": "0.0967",
   "epoch": "0.058"
  },
  {
   "episode": 30,
   "loss": "0.02572",
   "reward": "-0.06",
   "reward_std": "0.05",
   "kl": "1.724",
   "completion_length": "89.5",
   "clipped_ratio": "0",
   "learning_rate": "2.9e-05",
   "grad_norm": "1.624",
   "epoch": "0.06"
  },
  {
   "episode": 31,
   "loss": "0.02943",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.541",
   "completion_length": "103.5",
   "clipped_ratio": "0",
   "learning_rate": "3e-05",
   "grad_norm": "0.04806",
   "epoch": "0.062"
  },
  {
   "episode": 32,
   "loss": "0.01911",
   "reward": "-0.03875",
   "reward_std": "0.06237",
   "kl": "1.364",
   "completion_length": "98",
   "clipped_ratio": "0",
   "learning_rate": "3.1e-05",
   "grad_norm": "1.499",
   "epoch": "0.064"
  },
  {
   "episode": 33,
   "loss": "-0.3277",
   "reward": "-0.06375",
   "reward_std": "0.05297",
   "kl": "1.405",
   "completion_length": "77.75",
   "clipped_ratio": "0",
   "learning_rate": "3.2e-05",
   "grad_norm": "1.937",
   "epoch": "0.066"
  },
  {
   "episode": 34,
   "loss": "-0.1782",
   "reward": "-0.08875",
   "reward_std": "0.0075",
   "kl": "1.544",
   "completion_length": "88.25",
   "clipped_ratio": "0",
   "learning_rate": "3.3e-05",
   "grad_norm": "1.418",
   "epoch": "0.068"
  },
  {
   "episode": 35,
   "loss": "0.08649",
   "reward": "-0.035",
   "reward_std": "0.05774",
   "kl": "1.487",
   "completion_length": "106",
   "clipped_ratio": "0",
   "learning_rate": "3.4e-05",
   "grad_norm": "1.569",
   "epoch": "0.07"
  },
  {
   "episode": 36,
   "loss": "-0.0006697",
   "reward": "-0.035",
   "reward_std": "0.05774",
   "kl": "1.382",
   "completion_length": "99.75",
   "clipped_ratio": "0",
   "learning_rate": "3.5e-05",
   "grad_norm": "1.449",
   "epoch": "0.072"
  },
  {
   "episode": 37,
   "loss": "-0.02239",
   "reward": "0.01125",
   "reward_std": "0.0075",
   "kl": "1.371",
   "completion_length": "114.5",
   "clipped_ratio": "0",
   "learning_rate": "3.6e-05",
   "grad_norm": "1.406",
   "epoch": "0.074"
  },
  {
   "episode": 38,
   "loss": "0.03101",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.58",
   "completion_length": "95.75",
   "clipped_ratio": "0",
   "learning_rate": "3.7e-05",
   "grad_norm": "0.06464",
   "epoch": "0.076"
  },
  {
   "episode": 39,
   "loss": "0.05389",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.723",
   "completion_length": "89.5",
   "clipped_ratio": "0",
   "learning_rate": "3.8e-05",
   "grad_norm": "1.589",
   "epoch": "0.078"
  },
  {
   "episode": 40,
   "loss": "0.01259",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.451",
   "completion_length": "107.5",
   "clipped_ratio": "0",
   "learning_rate": "3.9e-05",
   "grad_norm": "1.351",
   "epoch": "0.08"
  },
  {
   "episode": 41,
   "loss": "0.07202",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.776",
   "completion_length": "92.25",
   "clipped_ratio": "0",
   "learning_rate": "4e-05",
   "grad_norm": "1.879",
   "epoch": "0.082"
  },
  {
   "episode": 42,
   "loss": "0.04993",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.674",
   "completion_length": "82.25",
   "clipped_ratio": "0",
   "learning_rate": "4.1e-05",
   "grad_norm": "1.664",
   "epoch": "0.084"
  },
  {
   "episode": 43,
   "loss": "-0.009573",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.592",
   "completion_length": "88.25",
   "clipped_ratio": "0",
   "learning_rate": "4.2e-05",
   "grad_norm": "1.511",
   "epoch": "0.086"
  },
  {
   "episode": 44,
   "loss": "-0.1045",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.62",
   "completion_length": "96.25",
   "clipped_ratio": "0",
   "learning_rate": "4.3e-05",
   "grad_norm": "1.58",
   "epoch": "0.088"
  },
  {
   "episode": 45,
   "loss": "0.09343",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.692",
   "completion_length": "87.5",
   "clipped_ratio": "0",
   "learning_rate": "4.4e-05",
   "grad_norm": "1.955",
   "epoch": "0.09"
  },
  {
   "episode": 46,
   "loss": "0.03445",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.762",
   "completion_length": "87.75",
   "clipped_ratio": "0",
   "learning_rate": "4.5e-05",
   "grad_norm": "0.1226",
   "epoch": "0.092"
  },
  {
   "episode": 47,
   "loss": "-0.09708",
   "reward": "-0.06",
   "reward_std": "0.05",
   "kl": "1.557",
   "completion_length": "90.75",
   "clipped_ratio": "0",
   "learning_rate": "4.6e-05",
   "grad_norm": "1.661",
   "epoch": "0.094"
  },
  {
   "episode": 48,
   "loss": "0.03128",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.661",
   "completion_length": "90",
   "clipped_ratio": "0",
   "learning_rate": "4.7e-05",
   "grad_norm": "0.07655",
   "epoch": "0.096"
  },
  {
   "episode": 49,
   "loss": "0.03093",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.609",
   "completion_length": "102.5",
   "clipped_ratio": "0",
   "learning_rate": "4.8e-05",
   "grad_norm": "0.06835",
   "epoch": "0.098"
  },
  {
   "episode": 50,
   "loss": "0.0294",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.499",
   "completion_length": "100.2",
   "clipped_ratio": "0",
   "learning_rate": "4.9e-05",
   "grad_norm": "0.04857",
   "epoch": "0.1"
  },
  {
   "episode": 51,
   "loss": "-0.08929",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.549",
   "completion_length": "104",
   "clipped_ratio": "0",
   "learning_rate": "5e-05",
   "grad_norm": "1.356",
   "epoch": "0.102"
  },
  {
   "episode": 52,
   "loss": "0.2245",
   "reward": "-0.035",
   "reward_std": "0.05774",
   "kl": "1.576",
   "completion_length": "137.8",
   "clipped_ratio": "0",
   "learning_rate": "4.989e-05",
   "grad_norm": "1.731",
   "epoch": "0.104"
  },
  {
   "episode": 53,
   "loss": "0.2212",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.381",
   "completion_length": "118",
   "clipped_ratio": "0",
   "learning_rate": "4.978e-05",
   "grad_norm": "1.558",
   "epoch": "0.106"
  },
  {
   "episode": 54,
   "loss": "0.03415",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.736",
   "completion_length": "89.25",
   "clipped_ratio": "0",
   "learning_rate": "4.967e-05",
   "grad_norm": "0.07583",
   "epoch": "0.108"
  },
  {
   "episode": 55,
   "loss": "0.03483",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.754",
   "completion_length": "97",
   "clipped_ratio": "0",
   "learning_rate": "4.956e-05",
   "grad_norm": "0.09679",
   "epoch": "0.11"
  },
  {
   "episode": 56,
   "loss": "0.05668",
   "reward": "-0.01375",
   "reward_std": "0.04802",
   "kl": "1.489",
   "completion_length": "107.2",
   "clipped_ratio": "0",
   "learning_rate": "4.944e-05",
   "grad_norm": "1.737",
   "epoch": "0.112"
  },
  {
   "episode": 57,
   "loss": "0.03395",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.723",
   "completion_length": "88.5",
   "clipped_ratio": "0",
   "learning_rate": "4.933e-05",
   "grad_norm": "0.07683",
   "epoch": "0.114"
  },
  {
   "episode": 58,
   "loss": "0.03335",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.681",
   "completion_length": "83",
   "clipped_ratio": "0",
   "learning_rate": "4.922e-05",
   "grad_norm": "0.0568",
   "epoch": "0.116"
  },
  {
   "episode": 59,
   "loss": "0.03198",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.614",
   "completion_length": "100",
   "clipped_ratio": "0",
   "learning_rate": "4.911e-05",
   "grad_norm": "0.06101",
   "epoch": "0.118"
  },
  {
   "episode": 60,
   "loss": "0.03004",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.525",
   "completion_length": "90.75",
   "clipped_ratio": "0",
   "learning_rate": "4.9e-05",
   "grad_norm": "0.06637",
   "epoch": "0.12"
  },
  {
   "episode": 61,
   "loss": "0.1298",
   "reward": "-0.035",
   "reward_std": "0.05774",
   "kl": "1.597",
   "completion_length": "103.8",
   "clipped_ratio": "0",
   "learning_rate": "4.889e-05",
   "grad_norm": "1.57",
   "epoch": "0.122"
  },
  {
   "episode": 62,
   "loss": "0.03358",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.721",
   "completion_length": "87",
   "clipped_ratio": "0",
   "learning_rate": "4.878e-05",
   "grad_norm": "0.05775",
   "epoch": "0.124"
  },
  {
   "episode": 63,
   "loss": "0.05816",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.336",
   "completion_length": "120.2",
   "clipped_ratio": "0",
   "learning_rate": "4.867e-05",
   "grad_norm": "1.434",
   "epoch": "0.126"
  },
  {
   "episode": 64,
   "loss": "-0.1117",
   "reward": "-0.0175",
   "reward_std": "0.05545",
   "kl": "1.377",
   "completion_length": "109",
   "clipped_ratio": "0",
   "learning_rate": "4.856e-05",
   "grad_norm": "1.512",
   "epoch": "0.128"
  },
  {
   "episode": 65,
   "loss": "0.0283",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.431",
   "completion_length": "112.5",
   "clipped_ratio": "0",
   "learning_rate": "4.844e-05",
   "grad_norm": "0.05251",
   "epoch": "0.13"
  },
  {
   "episode": 66,
   "loss": "0.02255",
   "reward": "-0.03875",
   "reward_std": "0.06237",
   "kl": "1.395",
   "completion_length": "103.2",
   "clipped_ratio": "0",
   "learning_rate": "4.833e-05",
   "grad_norm": "1.503",
   "epoch": "0.132"
  },
  {
   "episode": 67,
   "loss": "0.0259",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.301",
   "completion_length": "111.5",
   "clipped_ratio": "0",
   "learning_rate": "4.822e-05",
   "grad_norm": "0.05156",
   "epoch": "0.134"
  },
  {
   "episode": 68,
   "loss": "0.02639",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.364",
   "completion_length": "118.2",
   "clipped_ratio": "0",
   "learning_rate": "4.811e-05",
   "grad_norm": "0.03931",
   "epoch": "0.136"
  },
  {
   "episode": 69,
   "loss": "0.02672",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.366",
   "completion_length": "103",
   "clipped_ratio": "0",
   "learning_rate": "4.8e-05",
   "grad_norm": "0.04079",
   "epoch": "0.138"
  },
  {
   "episode": 70,
   "loss": "0.05654",
   "reward": "-0.01375",
   "reward_std": "0.0575",
   "kl": "1.344",
   "completion_length": "108.5",
   "clipped_ratio": "0",
   "learning_rate": "4.789e-05",
   "grad_norm": "1.761",
   "epoch": "0.14"
  },
  {
   "episode": 71,
   "loss": "0.03006",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.524",
   "completion_length": "106.8",
   "clipped_ratio": "0",
   "learning_rate": "4.778e-05",
   "grad_norm": "0.1124",
   "epoch": "0.142"
  },
  {
   "episode": 72,
   "loss": "0.02528",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.279",
   "completion_length": "125.2",
   "clipped_ratio": "0",
   "learning_rate": "4.767e-05",
   "grad_norm": "0.05816",
   "epoch": "0.144"
  },
  {
   "episode": 73,
   "loss": "0.208",
   "reward": "-0.035",
   "reward_std": "0.05774",
   "kl": "1.469",
   "completion_length": "131",
   "clipped_ratio": "0",
   "learning_rate": "4.756e-05",
   "grad_norm": "1.413",
   "epoch": "0.146"
  },
  {
   "episode": 74,
   "loss": "0.06406",
   "reward": "-0.035",
   "reward_std": "0.05774",
   "kl": "1.347",
   "completion_length": "121.2",
   "clipped_ratio": "0",
   "learning_rate": "4.744e-05",
   "grad_norm": "1.338",
   "epoch": "0.148"
  },
  {
   "episode": 75,
   "loss": "0.02868",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.465",
   "completion_length": "92.25",
   "clipped_ratio": "0",
   "learning_rate": "4.733e-05",
   "grad_norm": "0.05968",
   "epoch": "0.15"
  },
  {
   "episode": 76,
   "loss": "0.03725",
   "reward": "-0.01375",
   "reward_std": "0.0575",
   "kl": "1.292",
   "completion_length": "119.2",
   "clipped_ratio": "0",
   "learning_rate": "4.722e-05",
   "grad_norm": "2.02",
   "epoch": "0.152"
  },
  {
   "episode": 77,
   "loss": "0.02851",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.446",
   "completion_length": "118",
   "clipped_ratio": "0",
   "learning_rate": "4.711e-05",
   "grad_norm": "0.15",
   "epoch": "0.154"
  },
  {
   "episode": 78,
   "loss": "0.07068",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "3.348",
   "completion_length": "104",
   "clipped_ratio": "0",
   "learning_rate": "4.7e-05",
   "grad_norm": "4.452",
   "epoch": "0.156"
  },
  {
   "episode": 79,
   "loss": "0.02455",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.25",
   "completion_length": "132.2",
   "clipped_ratio": "0",
   "learning_rate": "4.689e-05",
   "grad_norm": "0.03926",
   "epoch": "0.158"
  },
  {
   "episode": 80,
   "loss": "0.02639",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.361",
   "completion_length": "116.8",
   "clipped_ratio": "0",
   "learning_rate": "4.678e-05",
   "grad_norm": "0.1211",
   "epoch": "0.16"
  },
  {
   "episode": 81,
   "loss": "0.02788",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.419",
   "completion_length": "100.5",
   "clipped_ratio": "0",
   "learning_rate": "4.667e-05",
   "grad_norm": "0.03858",
   "epoch": "0.162"
  },
  {
   "episode": 82,
   "loss": "0.03285",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.66",
   "completion_length": "89.75",
   "clipped_ratio": "0",
   "learning_rate": "4.656e-05",
   "grad_norm": "0.3443",
   "epoch": "0.164"
  },
  {
   "episode": 83,
   "loss": "0.02415",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.255",
   "completion_length": "125.2",
   "clipped_ratio": "0",
   "learning_rate": "4.644e-05",
   "grad_norm": "0.04701",
   "epoch": "0.166"
  },
  {
   "episode": 84,
   "loss": "0.1476",
   "reward": "-0.06375",
   "reward_std": "0.05297",
   "kl": "1.054",
   "completion_length": "144.2",
   "clipped_ratio": "0.25",
   "learning_rate": "4.633e-05",
   "grad_norm": "1.043",
   "epoch": "0.168"
  },
  {
   "episode": 85,
   "loss": "0.02414",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.324",
   "completion_length": "133",
   "clipped_ratio": "0",
   "learning_rate": "4.622e-05",
   "grad_norm": "0.1377",
   "epoch": "0.17"
  },
  {
   "episode": 86,
   "loss": "0.4588",
   "reward": "-0.0175",
   "reward_std": "0.05545",
   "kl": "1.243",
   "completion_length": "134.2",
   "clipped_ratio": "0.25",
   "learning_rate": "4.611e-05",
   "grad_norm": "1.549",
   "epoch": "0.172"
  },
  {
   "episode": 87,
   "loss": "0.05721",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.323",
   "completion_length": "116.8",
   "clipped_ratio": "0",
   "learning_rate": "4.6e-05",
   "grad_norm": "1.334",
   "epoch": "0.174"
  },
  {
   "episode": 88,
   "loss": "0.0259",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.336",
   "completion_length": "105.8",
   "clipped_ratio": "0",
   "learning_rate": "4.589e-05",
   "grad_norm": "0.04917",
   "epoch": "0.176"
  },
  {
   "episode": 89,
   "loss": "0.03014",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.515",
   "completion_length": "104.5",
   "clipped_ratio": "0",
   "learning_rate": "4.578e-05",
   "grad_norm": "0.1387",
   "epoch": "0.178"
  },
  {
   "episode": 90,
   "loss": "0.02943",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.476",
   "completion_length": "117.5",
   "clipped_ratio": "0",
   "learning_rate": "4.567e-05",
   "grad_norm": "0.2111",
   "epoch": "0.18"
  },
  {
   "episode": 91,
   "loss": "0.1492",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.187",
   "completion_length": "135.8",
   "clipped_ratio": "0",
   "learning_rate": "4.556e-05",
   "grad_norm": "1.204",
   "epoch": "0.182"
  },
  {
   "episode": 92,
   "loss": "0.00894",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.269",
   "completion_length": "115.8",
   "clipped_ratio": "0",
   "learning_rate": "4.544e-05",
   "grad_norm": "1.163",
   "epoch": "0.184"
  },
  {
   "episode": 93,
   "loss": "0.02674",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.44",
   "completion_length": "102.5",
   "clipped_ratio": "0",
   "learning_rate": "4.533e-05",
   "grad_norm": "0.1047",
   "epoch": "0.186"
  },
  {
   "episode": 94,
   "loss": "0.02273",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.198",
   "completion_length": "133.2",
   "clipped_ratio": "0",
   "learning_rate": "4.522e-05",
   "grad_norm": "0.05807",
   "epoch": "0.188"
  },
  {
   "episode": 95,
   "loss": "0.02433",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.232",
   "completion_length": "124",
   "clipped_ratio": "0",
   "learning_rate": "4.511e-05",
   "grad_norm": "0.04012",
   "epoch": "0.19"
  },
  {
   "episode": 96,
   "loss": "0.02564",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.31",
   "completion_length": "111.5",
   "clipped_ratio": "0",
   "learning_rate": "4.5e-05",
   "grad_norm": "0.06366",
   "epoch": "0.192"
  },
  {
   "episode": 97,
   "loss": "0.02609",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.337",
   "completion_length": "112.2",
   "clipped_ratio": "0",
   "learning_rate": "4.489e-05",
   "grad_norm": "0.05895",
   "epoch": "0.194"
  },
  {
   "episode": 98,
   "loss": "0.02595",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.339",
   "completion_length": "102.8",
   "clipped_ratio": "0",
   "learning_rate": "4.478e-05",
   "grad_norm": "0.07423",
   "epoch": "0.196"
  },
  {
   "episode": 99,
   "loss": "-0.2551",
   "reward": "-0.01375",
   "reward_std": "0.0575",
   "kl": "1.134",
   "completion_length": "131.2",
   "clipped_ratio": "0",
   "learning_rate": "4.467e-05",
   "grad_norm": "1.3",
   "epoch": "0.198"
  },
  {
   "episode": 100,
   "loss": "0.0307",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.706",
   "completion_length": "85.5",
   "clipped_ratio": "0",
   "learning_rate": "4.456e-05",
   "grad_norm": "0.113",
   "epoch": "0.2"
  },
  {
   "episode": 101,
   "loss": "0.02324",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.188",
   "completion_length": "121.8",
   "clipped_ratio": "0",
   "learning_rate": "4.444e-05",
   "grad_norm": "0.04005",
   "epoch": "0.202"
  },
  {
   "episode": 102,
   "loss": "0.02365",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.209",
   "completion_length": "119.2",
   "clipped_ratio": "0",
   "learning_rate": "4.433e-05",
   "grad_norm": "0.03859",
   "epoch": "0.204"
  },
  {
   "episode": 103,
   "loss": "-0.1069",
   "reward": "-0.01375",
   "reward_std": "0.0575",
   "kl": "1.013",
   "completion_length": "123.5",
   "clipped_ratio": "0",
   "learning_rate": "4.422e-05",
   "grad_norm": "1.101",
   "epoch": "0.206"
  },
  {
   "episode": 104,
   "loss": "-0.06919",
   "reward": "-0.01375",
   "reward_std": "0.0575",
   "kl": "1.118",
   "completion_length": "129.8",
   "clipped_ratio": "0",
   "learning_rate": "4.411e-05",
   "grad_norm": "1.286",
   "epoch": "0.208"
  },
  {
   "episode": 105,
   "loss": "0.02123",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.138",
   "completion_length": "142",
   "clipped_ratio": "0",
   "learning_rate": "4.4e-05",
   "grad_norm": "0.04361",
   "epoch": "0.21"
  },
  {
   "episode": 106,
   "loss": "0.02399",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.212",
   "completion_length": "128",
   "clipped_ratio": "0",
   "learning_rate": "4.389e-05",
   "grad_norm": "0.04805",
   "epoch": "0.212"
  },
  {
   "episode": 107,
   "loss": "0.3564",
   "reward": "-0.01375",
   "reward_std": "0.0575",
   "kl": "1.031",
   "completion_length": "152.8",
   "clipped_ratio": "0.25",
   "learning_rate": "4.378e-05",
   "grad_norm": "1.439",
   "epoch": "0.214"
  },
  {
   "episode": 108,
   "loss": "0.02283",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.166",
   "completion_length": "129.2",
   "clipped_ratio": "0",
   "learning_rate": "4.367e-05",
   "grad_norm": "0.05553",
   "epoch": "0.216"
  },
  {
   "episode": 109,
   "loss": "-0.04348",
   "reward": "-0.0675",
   "reward_std": "0.05545",
   "kl": "0.5512",
   "completion_length": "202.8",
   "clipped_ratio": "0",
   "learning_rate": "4.356e-05",
   "grad_norm": "1.05",
   "epoch": "0.218"
  },
  {
   "episode": 110,
   "loss": "0.02031",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.051",
   "completion_length": "137.5",
   "clipped_ratio": "0",
   "learning_rate": "4.344e-05",
   "grad_norm": "0.04941",
   "epoch": "0.22"
  },
  {
   "episode": 111,
   "loss": "0.1065",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.289",
   "completion_length": "118.5",
   "clipped_ratio": "0",
   "learning_rate": "4.333e-05",
   "grad_norm": "1.255",
   "epoch": "0.222"
  },
  {
   "episode": 112,
   "loss": "0.02114",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.096",
   "completion_length": "133.5",
   "clipped_ratio": "0",
   "learning_rate": "4.322e-05",
   "grad_norm": "0.06218",
   "epoch": "0.224"
  },
  {
   "episode": 113,
   "loss": "0.02359",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.246",
   "completion_length": "112",
   "clipped_ratio": "0",
   "learning_rate": "4.311e-05",
   "grad_norm": "0.03646",
   "epoch": "0.226"
  },
  {
   "episode": 114,
   "loss": "-0.02765",
   "reward": "-0.0175",
   "reward_std": "0.05545",
   "kl": "1.039",
   "completion_length": "101",
   "clipped_ratio": "0",
   "learning_rate": "4.3e-05",
   "grad_norm": "1.376",
   "epoch": "0.228"
  },
  {
   "episode": 115,
   "loss": "0.0184",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "0.9662",
   "completion_length": "165.5",
   "clipped_ratio": "0.25",
   "learning_rate": "4.289e-05",
   "grad_norm": "0.04314",
   "epoch": "0.23"
  },
  {
   "episode": 116,
   "loss": "0.1954",
   "reward": "-0.03875",
   "reward_std": "0.06237",
   "kl": "1.071",
   "completion_length": "158",
   "clipped_ratio": "0.25",
   "learning_rate": "4.278e-05",
   "grad_norm": "1.173",
   "epoch": "0.232"
  },
  {
   "episode": 117,
   "loss": "0.02071",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.051",
   "completion_length": "130",
   "clipped_ratio": "0",
   "learning_rate": "4.267e-05",
   "grad_norm": "0.02952",
   "epoch": "0.234"
  },
  {
   "episode": 118,
   "loss": "0.1281",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "1.111",
   "completion_length": "128.5",
   "clipped_ratio": "0",
   "learning_rate": "4.256e-05",
   "grad_norm": "1.226",
   "epoch": "0.236"
  },
  {
   "episode": 119,
   "loss": "0.1146",
   "reward": "-0.01",
   "reward_std": "0.05",
   "kl": "0.9954",
   "completion_length": "124.2",
   "clipped_ratio": "0",
   "learning_rate": "4.244e-05",
   "grad_norm": "1.035",
   "epoch": "0.238"
  },
  {
   "episode": 120,
   "loss": "0.09495",
   "reward": "-0.0425",
   "reward_std": "0.0664",
   "kl": "0.6759",
   "completion_length": "192.2",
   "clipped_ratio": "0.25",
   "learning_rate": "4.233e-05",
   "grad_norm": "0.9413",
   "epoch": "0.24"
  },
  {
   "episode": 121,
   "loss": "0.01714",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "0.894",
   "completion_length": "164.8",
   "clipped_ratio": "0",
   "learning_rate": "4.222e-05",
   "grad_norm": "0.04506",
   "epoch": "0.242"
  },
  {
   "episode": 122,
   "loss": "-0.1289",
   "reward": "-0.01375",
   "reward_std": "0.0575",
   "kl": "1.226",
   "completion_length": "102.2",
   "clipped_ratio": "0",
   "learning_rate": "4.211e-05",
   "grad_norm": "1.349",
   "epoch": "0.244"
  },
  {
   "episode": 123,
   "loss": "0.02344",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.195",
   "completion_length": "115.2",
   "clipped_ratio": "0",
   "learning_rate": "4.2e-05",
   "grad_norm": "0.0458",
   "epoch": "0.246"
  },
  {
   "episode": 124,
   "loss": "0.02134",
   "reward": "0.015",
   "reward_std": "0",
   "kl": "1.169",
   "completion_length": "125.2",
   "clipped_ratio": "0",
   "learning_rate": "4.189e-05",
   "grad_norm": "0.06143",
   "epoch": "0.248"
  },
  {
   "episode": 125,
   "loss": "-0.1984",
   "reward": "0.01125",
   "reward_std": "0.0075",
   "kl": "1.124",
   "completion_length": "116.8",
   "clipped_ratio": "0",
   "learning_rate": "4.178e-05",
   "grad_norm": "1.044",
   "epoch": "0.25"
  },
  {
   "episode": 126,
   "loss": "0.0192",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.9959",
   "completion_length": "133.5",
   "clipped_ratio": "0",
   "learning_rate": "4.167e-05",
   "grad_norm": "0.04122",
   "epoch": "0.252"
  },
  {
   "episode": 127,
   "loss": "0.01727",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.977",
   "completion_length": "147.5",
   "clipped_ratio": "0.25",
   "learning_rate": "4.156e-05",
   "grad_norm": "0.02834",
   "epoch": "0.254"
  },
  {
   "episode": 128,
   "loss": "0.009876",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "1.032",
   "completion_length": "156.8",
   "clipped_ratio": "0",
   "learning_rate": "4.144e-05",
   "grad_norm": "0.9371",
   "epoch": "0.256"
  },
  {
   "episode": 129,
   "loss": "-0.09492",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "1.074",
   "completion_length": "148.5",
   "clipped_ratio": "0.25",
   "learning_rate": "4.133e-05",
   "grad_norm": "0.845",
   "epoch": "0.258"
  },
  {
   "episode": 130,
   "loss": "0.02401",
   "reward": "0",
   "reward_std": "0",
   "kl": "1.203",
   "completion_length": "104.5",
   "clipped_ratio": "0",
   "learning_rate": "4.122e-05",
   "grad_norm": "0.05182",
   "epoch": "0.26"
  },
  {
   "episode": 131,
   "loss": "-0.2198",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.9641",
   "completion_length": "162",
   "clipped_ratio": "0.25",
   "learning_rate": "4.111e-05",
   "grad_norm": "0.8911",
   "epoch": "0.262"
  },
  {
   "episode": 132,
   "loss": "0.08979",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "1.071",
   "completion_length": "107.2",
   "clipped_ratio": "0",
   "learning_rate": "4.1e-05",
   "grad_norm": "1.38",
   "epoch": "0.264"
  },
  {
   "episode": 133,
   "loss": "0.0148",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.8439",
   "completion_length": "204",
   "clipped_ratio": "0.5",
   "learning_rate": "4.089e-05",
   "grad_norm": "0.03386",
   "epoch": "0.266"
  },
  {
   "episode": 134,
   "loss": "0.0148",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.8089",
   "completion_length": "192",
   "clipped_ratio": "0.5",
   "learning_rate": "4.078e-05",
   "grad_norm": "0.0373",
   "epoch": "0.268"
  },
  {
   "episode": 135,
   "loss": "0.2167",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.8866",
   "completion_length": "182.2",
   "clipped_ratio": "0.5",
   "learning_rate": "4.067e-05",
   "grad_norm": "0.9069",
   "epoch": "0.27"
  },
  {
   "episode": 136,
   "loss": "0.02339",
   "reward": "0",
   "reward_std": "0",
   "kl": "1.197",
   "completion_length": "107.5",
   "clipped_ratio": "0",
   "learning_rate": "4.056e-05",
   "grad_norm": "0.06824",
   "epoch": "0.272"
  },
  {
   "episode": 137,
   "loss": "-0.1594",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "1.195",
   "completion_length": "150.2",
   "clipped_ratio": "0.25",
   "learning_rate": "4.044e-05",
   "grad_norm": "0.8749",
   "epoch": "0.274"
  },
  {
   "episode": 138,
   "loss": "0.021",
   "reward": "0",
   "reward_std": "0",
   "kl": "1.15",
   "completion_length": "121.8",
   "clipped_ratio": "0",
   "learning_rate": "4.033e-05",
   "grad_norm": "0.0369",
   "epoch": "0.276"
  },
  {
   "episode": 139,
   "loss": "0.0129",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.7527",
   "completion_length": "185",
   "clipped_ratio": "0.5",
   "learning_rate": "4.022e-05",
   "grad_norm": "0.02335",
   "epoch": "0.278"
  },
  {
   "episode": 140,
   "loss": "0.0135",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.7518",
   "completion_length": "191.5",
   "clipped_ratio": "0.5",
   "learning_rate": "4.011e-05",
   "grad_norm": "0.03046",
   "epoch": "0.28"
  },
  {
   "episode": 141,
   "loss": "0.01559",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.8364",
   "completion_length": "197",
   "clipped_ratio": "0.25",
   "learning_rate": "4e-05",
   "grad_norm": "0.1297",
   "epoch": "0.282"
  },
  {
   "episode": 142,
   "loss": "0.02244",
   "reward": "0",
   "reward_std": "0",
   "kl": "1.18",
   "completion_length": "119.5",
   "clipped_ratio": "0",
   "learning_rate": "3.989e-05",
   "grad_norm": "0.04509",
   "epoch": "0.284"
  },
  {
   "episode": 143,
   "loss": "-0.06772",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "1.04",
   "completion_length": "113",
   "clipped_ratio": "0",
   "learning_rate": "3.978e-05",
   "grad_norm": "1.059",
   "epoch": "0.286"
  },
  {
   "episode": 144,
   "loss": "-0.1361",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.8591",
   "completion_length": "164",
   "clipped_ratio": "0",
   "learning_rate": "3.967e-05",
   "grad_norm": "0.7481",
   "epoch": "0.288"
  },
  {
   "episode": 145,
   "loss": "0.009652",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.7233",
   "completion_length": "170.8",
   "clipped_ratio": "0",
   "learning_rate": "3.956e-05",
   "grad_norm": "0.8965",
   "epoch": "0.29"
  },
  {
   "episode": 146,
   "loss": "0.01383",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.8261",
   "completion_length": "172",
   "clipped_ratio": "0.25",
   "learning_rate": "3.944e-05",
   "grad_norm": "0.01843",
   "epoch": "0.292"
  },
  {
   "episode": 147,
   "loss": "0.1679",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.7532",
   "completion_length": "195.5",
   "clipped_ratio": "0.5",
   "learning_rate": "3.933e-05",
   "grad_norm": "0.8574",
   "epoch": "0.294"
  },
  {
   "episode": 148,
   "loss": "0.01479",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.8126",
   "completion_length": "161.5",
   "clipped_ratio": "0.25",
   "learning_rate": "3.922e-05",
   "grad_norm": "0.04673",
   "epoch": "0.296"
  },
  {
   "episode": 149,
   "loss": "0.09016",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.6905",
   "completion_length": "222",
   "clipped_ratio": "0.25",
   "learning_rate": "3.911e-05",
   "grad_norm": "0.9023",
   "epoch": "0.298"
  },
  {
   "episode": 150,
   "loss": "0.01703",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.9712",
   "completion_length": "154.8",
   "clipped_ratio": "0.25",
   "learning_rate": "3.9e-05",
   "grad_norm": "0.04395",
   "epoch": "0.3"
  },
  {
   "episode": 151,
   "loss": "0.01369",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.8077",
   "completion_length": "179.5",
   "clipped_ratio": "0.5",
   "learning_rate": "3.889e-05",
   "grad_norm": "0.02968",
   "epoch": "0.302"
  },
  {
   "episode": 152,
   "loss": "0.01495",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.7448",
   "completion_length": "236.2",
   "clipped_ratio": "0.5",
   "learning_rate": "3.878e-05",
   "grad_norm": "0.2067",
   "epoch": "0.304"
  },
  {
   "episode": 153,
   "loss": "0.01446",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.8869",
   "completion_length": "174.5",
   "clipped_ratio": "0.5",
   "learning_rate": "3.867e-05",
   "grad_norm": "0.02209",
   "epoch": "0.306"
  },
  {
   "episode": 154,
   "loss": "0.01308",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.7685",
   "completion_length": "191",
   "clipped_ratio": "0.25",
   "learning_rate": "3.856e-05",
   "grad_norm": "0.02081",
   "epoch": "0.308"
  },
  {
   "episode": 155,
   "loss": "0.09928",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.7042",
   "completion_length": "218.2",
   "clipped_ratio": "0.75",
   "learning_rate": "3.844e-05",
   "grad_norm": "0.8511",
   "epoch": "0.31"
  },
  {
   "episode": 156,
   "loss": "0.1472",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.8815",
   "completion_length": "202",
   "clipped_ratio": "0.5",
   "learning_rate": "3.833e-05",
   "grad_norm": "1.008",
   "epoch": "0.312"
  },
  {
   "episode": 157,
   "loss": "0.01473",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.7722",
   "completion_length": "184.5",
   "clipped_ratio": "0.25",
   "learning_rate": "3.822e-05",
   "grad_norm": "0.1047",
   "epoch": "0.314"
  },
  {
   "episode": 158,
   "loss": "0.01633",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.8847",
   "completion_length": "166.5",
   "clipped_ratio": "0.25",
   "learning_rate": "3.811e-05",
   "grad_norm": "0.06288",
   "epoch": "0.316"
  },
  {
   "episode": 159,
   "loss": "0.01355",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.7771",
   "completion_length": "177.2",
   "clipped_ratio": "0.25",
   "learning_rate": "3.8e-05",
   "grad_norm": "0.05157",
   "epoch": "0.318"
  },
  {
   "episode": 160,
   "loss": "0.06912",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.592",
   "completion_length": "229.5",
   "clipped_ratio": "0.5",
   "learning_rate": "3.789e-05",
   "grad_norm": "0.7783",
   "epoch": "0.32"
  },
  {
   "episode": 161,
   "loss": "0.01657",
   "reward": "0",
   "reward_std": "0",
   "kl": "1.046",
   "completion_length": "135.5",
   "clipped_ratio": "0.25",
   "learning_rate": "3.778e-05",
   "grad_norm": "0.02938",
   "epoch": "0.322"
  },
  {
   "episode": 162,
   "loss": "0.01172",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.6083",
   "completion_length": "232.5",
   "clipped_ratio": "0.75",
   "learning_rate": "3.767e-05",
   "grad_norm": "0.02841",
   "epoch": "0.324"
  },
  {
   "episode": 163,
   "loss": "0.1041",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.9652",
   "completion_length": "218.5",
   "clipped_ratio": "0.75",
   "learning_rate": "3.756e-05",
   "grad_norm": "0.9357",
   "epoch": "0.326"
  },
  {
   "episode": 164,
   "loss": "0.02038",
   "reward": "0",
   "reward_std": "0",
   "kl": "1.034",
   "completion_length": "114.8",
   "clipped_ratio": "0",
   "learning_rate": "3.744e-05",
   "grad_norm": "0.03502",
   "epoch": "0.328"
  },
  {
   "episode": 165,
   "loss": "-0.2373",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.6106",
   "completion_length": "219.5",
   "clipped_ratio": "0.75",
   "learning_rate": "3.733e-05",
   "grad_norm": "0.6781",
   "epoch": "0.33"
  },
  {
   "episode": 166,
   "loss": "-0.1384",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "1.235",
   "completion_length": "123.5",
   "clipped_ratio": "0",
   "learning_rate": "3.722e-05",
   "grad_norm": "0.8199",
   "epoch": "0.332"
  },
  {
   "episode": 167,
   "loss": "0.1289",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.6907",
   "completion_length": "207.8",
   "clipped_ratio": "0.5",
   "learning_rate": "3.711e-05",
   "grad_norm": "0.8933",
   "epoch": "0.334"
  },
  {
   "episode": 168,
   "loss": "0.05713",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.5286",
   "completion_length": "234",
   "clipped_ratio": "0.75",
   "learning_rate": "3.7e-05",
   "grad_norm": "0.6729",
   "epoch": "0.336"
  },
  {
   "episode": 169,
   "loss": "0.01499",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.8495",
   "completion_length": "174.2",
   "clipped_ratio": "0.25",
   "learning_rate": "3.689e-05",
   "grad_norm": "0.1389",
   "epoch": "0.338"
  },
  {
   "episode": 170,
   "loss": "0.01559",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.8565",
   "completion_length": "168.5",
   "clipped_ratio": "0.25",
   "learning_rate": "3.678e-05",
   "grad_norm": "0.04733",
   "epoch": "0.34"
  },
  {
   "episode": 171,
   "loss": "-0.09501",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.8351",
   "completion_length": "168",
   "clipped_ratio": "0.25",
   "learning_rate": "3.667e-05",
   "grad_norm": "0.84",
   "epoch": "0.342"
  },
  {
   "episode": 172,
   "loss": "-0.2282",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.862",
   "completion_length": "177",
   "clipped_ratio": "0.5",
   "learning_rate": "3.656e-05",
   "grad_norm": "0.6844",
   "epoch": "0.344"
  },
  {
   "episode": 173,
   "loss": "0.01553",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.8731",
   "completion_length": "185.5",
   "clipped_ratio": "0.25",
   "learning_rate": "3.644e-05",
   "grad_norm": "0.2459",
   "epoch": "0.346"
  },
  {
   "episode": 174,
   "loss": "0.0121",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.68",
   "completion_length": "206.8",
   "clipped_ratio": "0.5",
   "learning_rate": "3.633e-05",
   "grad_norm": "0.02169",
   "epoch": "0.348"
  },
  {
   "episode": 175,
   "loss": "0.01206",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.6513",
   "completion_length": "199",
   "clipped_ratio": "0.5",
   "learning_rate": "3.622e-05",
   "grad_norm": "0.04694",
   "epoch": "0.35"
  },
  {
   "episode": 176,
   "loss": "0.2827",
   "reward": "-0.05",
   "reward_std": "0.05774",
   "kl": "0.7719",
   "completion_length": "170.5",
   "clipped_ratio": "0.25",
   "learning_rate": "3.611e-05",
   "grad_norm": "0.7261",
   "epoch": "0.352"
  },
  {
   "episode": 177,
   "loss": "0.1534",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.7106",
   "completion_length": "200",
   "clipped_ratio": "0.25",
   "learning_rate": "3.6e-05",
   "grad_norm": "0.8582",
   "epoch": "0.354"
  },
  {
   "episode": 178,
   "loss": "0.01187",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.5987",
   "completion_length": "222.5",
   "clipped_ratio": "0.25",
   "learning_rate": "3.589e-05",
   "grad_norm": "0.02377",
   "epoch": "0.356"
  },
  {
   "episode": 179,
   "loss": "0.01027",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.5137",
   "completion_length": "256",
   "clipped_ratio": "1",
   "learning_rate": "3.578e-05",
   "grad_norm": "0.03761",
   "epoch": "0.358"
  },
  {
   "episode": 180,
   "loss": "0.009968",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.5013",
   "completion_length": "246",
   "clipped_ratio": "0.5",
   "learning_rate": "3.567e-05",
   "grad_norm": "0.03003",
   "epoch": "0.36"
  },
  {
   "episode": 181,
   "loss": "0.0136",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.6799",
   "completion_length": "256",
   "clipped_ratio": "1",
   "learning_rate": "3.556e-05",
   "grad_norm": "0.6516",
   "epoch": "0.362"
  },
  {
   "episode": 182,
   "loss": "0.0167",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.8716",
   "completion_length": "147",
   "clipped_ratio": "0",
   "learning_rate": "3.544e-05",
   "grad_norm": "0.07296",
   "epoch": "0.364"
  },
  {
   "episode": 183,
   "loss": "0.0114",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.6002",
   "completion_length": "206",
   "clipped_ratio": "0.25",
   "learning_rate": "3.533e-05",
   "grad_norm": "0.03717",
   "epoch": "0.366"
  },
  {
   "episode": 184,
   "loss": "0.07569",
   "reward": "-0.05",
   "reward_std": "0.05774",
   "kl": "0.4309",
   "completion_length": "237.5",
   "clipped_ratio": "0.75",
   "learning_rate": "3.522e-05",
   "grad_norm": "0.6907",
   "epoch": "0.368"
  },
  {
   "episode": 185,
   "loss": "0.01558",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.8331",
   "completion_length": "182.2",
   "clipped_ratio": "0.25",
   "learning_rate": "3.511e-05",
   "grad_norm": "0.1109",
   "epoch": "0.37"
  },
  {
   "episode": 186,
   "loss": "0.08199",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.6914",
   "completion_length": "225",
   "clipped_ratio": "0.75",
   "learning_rate": "3.5e-05",
   "grad_norm": "0.7589",
   "epoch": "0.372"
  },
  {
   "episode": 187,
   "loss": "0.01244",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.6683",
   "completion_length": "198.5",
   "clipped_ratio": "0.5",
   "learning_rate": "3.489e-05",
   "grad_norm": "0.02421",
   "epoch": "0.374"
  },
  {
   "episode": 188,
   "loss": "0.009995",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.5007",
   "completion_length": "243",
   "clipped_ratio": "0.75",
   "learning_rate": "3.478e-05",
   "grad_norm": "0.0273",
   "epoch": "0.376"
  },
  {
   "episode": 189,
   "loss": "0.07081",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.5754",
   "completion_length": "228.5",
   "clipped_ratio": "0.75",
   "learning_rate": "3.467e-05",
   "grad_norm": "0.5827",
   "epoch": "0.378"
  },
  {
   "episode": 190,
   "loss": "-0.2955",
   "reward": "-0.05",
   "reward_std": "0.05774",
   "kl": "0.5758",
   "completion_length": "189",
   "clipped_ratio": "0.5",
   "learning_rate": "3.456e-05",
   "grad_norm": "0.9176",
   "epoch": "0.38"
  },
  {
   "episode": 191,
   "loss": "0.01553",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.7765",
   "completion_length": "256",
   "clipped_ratio": "1",
   "learning_rate": "3.444e-05",
   "grad_norm": "0.8604",
   "epoch": "0.382"
  },
  {
   "episode": 192,
   "loss": "0.01256",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.6766",
   "completion_length": "182.2",
   "clipped_ratio": "0.25",
   "learning_rate": "3.433e-05",
   "grad_norm": "0.01865",
   "epoch": "0.384"
  },
  {
   "episode": 193,
   "loss": "-0.06049",
   "reward": "-0.05",
   "reward_std": "0.05774",
   "kl": "0.5448",
   "completion_length": "236.5",
   "clipped_ratio": "0.75",
   "learning_rate": "3.422e-05",
   "grad_norm": "0.6642",
   "epoch": "0.386"
  },
  {
   "episode": 194,
   "loss": "0.01536",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.8527",
   "completion_length": "160.8",
   "clipped_ratio": "0.25",
   "learning_rate": "3.411e-05",
   "grad_norm": "0.03851",
   "epoch": "0.388"
  },
  {
   "episode": 195,
   "loss": "0.03194",
   "reward": "0",
   "reward_std": "0",
   "kl": "1.676",
   "completion_length": "193",
   "clipped_ratio": "0.25",
   "learning_rate": "3.4e-05",
   "grad_norm": "2.288",
   "epoch": "0.39"
  },
  {
   "episode": 196,
   "loss": "-0.03057",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.5017",
   "completion_length": "249.2",
   "clipped_ratio": "0.75",
   "learning_rate": "3.389e-05",
   "grad_norm": "0.6679",
   "epoch": "0.392"
  },
  {
   "episode": 197,
   "loss": "0.01438",
   "reward": "-0.05",
   "reward_std": "0.05774",
   "kl": "0.6339",
   "completion_length": "255.5",
   "clipped_ratio": "0.75",
   "learning_rate": "3.378e-05",
   "grad_norm": "0.6842",
   "epoch": "0.394"
  },
  {
   "episode": 198,
   "loss": "0.01817",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.5629",
   "completion_length": "252.5",
   "clipped_ratio": "0.75",
   "learning_rate": "3.367e-05",
   "grad_norm": "0.699",
   "epoch": "0.396"
  },
  {
   "episode": 199,
   "loss": "0.05561",
   "reward": "-0.05",
   "reward_std": "0.05774",
   "kl": "0.5969",
   "completion_length": "235.5",
   "clipped_ratio": "0.5",
   "learning_rate": "3.356e-05",
   "grad_norm": "0.662",
   "epoch": "0.398"
  },
  {
   "episode": 200,
   "loss": "0.01062",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.5544",
   "completion_length": "223",
   "clipped_ratio": "0.5",
   "learning_rate": "3.344e-05",
   "grad_norm": "0.03785",
   "epoch": "0.4"
  },
  {
   "episode": 201,
   "loss": "-0.09729",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.4514",
   "completion_length": "239",
   "clipped_ratio": "0.75",
   "learning_rate": "3.333e-05",
   "grad_norm": "0.6039",
   "epoch": "0.402"
  },
  {
   "episode": 202,
   "loss": "0.01084",
   "reward": "0",
   "reward_std": "0",
   "kl": "0.5435",
   "completion_length": "249.8",
   "clipped_ratio": "0.75",
   "learning_rate": "3.322e-05",
   "grad_norm": "0.05489",
   "epoch": "0.404"
  },
  {
   "episode": 203,
   "loss": "0.01082",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.5408",
   "completion_length": "256",
   "clipped_ratio": "1",
   "learning_rate": "3.311e-05",
   "grad_norm": "0.5738",
   "epoch": "0.406"
  },
  {
   "episode": 204,
   "loss": "0.01224",
   "reward": "-0.025",
   "reward_std": "0.05",
   "kl": "0.6122",
   "completion_length": "256",
   "clipped_ratio": "1",
   "learning_rate": "3.3e-05",
   "grad_norm": "0.6763",
   "epoch": "0.408"
  }
 ]
}