Spaces:
Sleeping
Sleeping
| { | |
| "episodes": [ | |
| { | |
| "episode": 1, | |
| "loss": "0.1168", | |
| "reward": "-0.06", | |
| "reward_std": "0.05", | |
| "kl": "1.487", | |
| "completion_length": "121.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "0", | |
| "grad_norm": "1.441", | |
| "epoch": "0.002" | |
| }, | |
| { | |
| "episode": 2, | |
| "loss": "0.03414", | |
| "reward": "-0.085", | |
| "reward_std": "0", | |
| "kl": "1.728", | |
| "completion_length": "82", | |
| "clipped_ratio": "0", | |
| "learning_rate": "1e-06", | |
| "grad_norm": "0.06773", | |
| "epoch": "0.004" | |
| }, | |
| { | |
| "episode": 3, | |
| "loss": "0.03677", | |
| "reward": "-0.035", | |
| "reward_std": "0.05774", | |
| "kl": "1.472", | |
| "completion_length": "110.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "2e-06", | |
| "grad_norm": "1.649", | |
| "epoch": "0.006" | |
| }, | |
| { | |
| "episode": 4, | |
| "loss": "0.1546", | |
| "reward": "-0.06", | |
| "reward_std": "0.05", | |
| "kl": "1.565", | |
| "completion_length": "97.25", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3e-06", | |
| "grad_norm": "1.336", | |
| "epoch": "0.008" | |
| }, | |
| { | |
| "episode": 5, | |
| "loss": "0.0292", | |
| "reward": "-0.085", | |
| "reward_std": "0", | |
| "kl": "1.48", | |
| "completion_length": "105.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4e-06", | |
| "grad_norm": "0.05263", | |
| "epoch": "0.01" | |
| }, | |
| { | |
| "episode": 6, | |
| "loss": "-0.05279", | |
| "reward": "-0.06", | |
| "reward_std": "0.05", | |
| "kl": "1.577", | |
| "completion_length": "113", | |
| "clipped_ratio": "0", | |
| "learning_rate": "5e-06", | |
| "grad_norm": "1.791", | |
| "epoch": "0.012" | |
| }, | |
| { | |
| "episode": 7, | |
| "loss": "0.1339", | |
| "reward": "-0.06", | |
| "reward_std": "0.05", | |
| "kl": "1.539", | |
| "completion_length": "126.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "6e-06", | |
| "grad_norm": "1.452", | |
| "epoch": "0.014" | |
| }, | |
| { | |
| "episode": 8, | |
| "loss": "0.03526", | |
| "reward": "-0.085", | |
| "reward_std": "0", | |
| "kl": "1.845", | |
| "completion_length": "89", | |
| "clipped_ratio": "0", | |
| "learning_rate": "7e-06", | |
| "grad_norm": "0.1091", | |
| "epoch": "0.016" | |
| }, | |
| { | |
| "episode": 9, | |
| "loss": "-0.03297", | |
| "reward": "-0.03875", | |
| "reward_std": "0.05375", | |
| "kl": "1.484", | |
| "completion_length": "114.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "8e-06", | |
| "grad_norm": "1.834", | |
| "epoch": "0.018" | |
| }, | |
| { | |
| "episode": 10, | |
| "loss": "0.02409", | |
| "reward": "-0.085", | |
| "reward_std": "0", | |
| "kl": "1.3", | |
| "completion_length": "158", | |
| "clipped_ratio": "0", | |
| "learning_rate": "9e-06", | |
| "grad_norm": "0.08162", | |
| "epoch": "0.02" | |
| }, | |
| { | |
| "episode": 11, | |
| "loss": "-0.07952", | |
| "reward": "-0.035", | |
| "reward_std": "0.05774", | |
| "kl": "1.526", | |
| "completion_length": "114.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "1e-05", | |
| "grad_norm": "1.697", | |
| "epoch": "0.022" | |
| }, | |
| { | |
| "episode": 12, | |
| "loss": "0.04861", | |
| "reward": "-0.035", | |
| "reward_std": "0.05774", | |
| "kl": "1.59", | |
| "completion_length": "99", | |
| "clipped_ratio": "0", | |
| "learning_rate": "1.1e-05", | |
| "grad_norm": "1.777", | |
| "epoch": "0.024" | |
| }, | |
| { | |
| "episode": 13, | |
| "loss": "0.09983", | |
| "reward": "-0.03875", | |
| "reward_std": "0.06237", | |
| "kl": "1.888", | |
| "completion_length": "93.75", | |
| "clipped_ratio": "0", | |
| "learning_rate": "1.2e-05", | |
| "grad_norm": "2.288", | |
| "epoch": "0.026" | |
| }, | |
| { | |
| "episode": 14, | |
| "loss": "0.1242", | |
| "reward": "-0.06", | |
| "reward_std": "0.05", | |
| "kl": "1.648", | |
| "completion_length": "112.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "1.3e-05", | |
| "grad_norm": "1.58", | |
| "epoch": "0.028" | |
| }, | |
| { | |
| "episode": 15, | |
| "loss": "0.04319", | |
| "reward": "-0.06", | |
| "reward_std": "0.05", | |
| "kl": "1.545", | |
| "completion_length": "100.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "1.4e-05", | |
| "grad_norm": "1.571", | |
| "epoch": "0.03" | |
| }, | |
| { | |
| "episode": 16, | |
| "loss": "0.03368", | |
| "reward": "-0.085", | |
| "reward_std": "0", | |
| "kl": "1.721", | |
| "completion_length": "82", | |
| "clipped_ratio": "0", | |
| "learning_rate": "1.5e-05", | |
| "grad_norm": "0.1079", | |
| "epoch": "0.032" | |
| }, | |
| { | |
| "episode": 17, | |
| "loss": "0.03411", | |
| "reward": "-0.085", | |
| "reward_std": "0", | |
| "kl": "1.737", | |
| "completion_length": "96", | |
| "clipped_ratio": "0", | |
| "learning_rate": "1.6e-05", | |
| "grad_norm": "0.05412", | |
| "epoch": "0.034" | |
| }, | |
| { | |
| "episode": 18, | |
| "loss": "0.02681", | |
| "reward": "-0.085", | |
| "reward_std": "0", | |
| "kl": "1.425", | |
| "completion_length": "116.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "1.7e-05", | |
| "grad_norm": "0.09016", | |
| "epoch": "0.036" | |
| }, | |
| { | |
| "episode": 19, | |
| "loss": "-0.1867", | |
| "reward": "0.1367", | |
| "reward_std": "0.4434", | |
| "kl": "1.459", | |
| "completion_length": "138.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "1.8e-05", | |
| "grad_norm": "1.965", | |
| "epoch": "0.038" | |
| }, | |
| { | |
| "episode": 20, | |
| "loss": "0.02446", | |
| "reward": "-0.03875", | |
| "reward_std": "0.06237", | |
| "kl": "1.662", | |
| "completion_length": "97.75", | |
| "clipped_ratio": "0", | |
| "learning_rate": "1.9e-05", | |
| "grad_norm": "1.648", | |
| "epoch": "0.04" | |
| }, | |
| { | |
| "episode": 21, | |
| "loss": "0.02844", | |
| "reward": "-0.085", | |
| "reward_std": "0", | |
| "kl": "1.462", | |
| "completion_length": "103.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "2e-05", | |
| "grad_norm": "0.05372", | |
| "epoch": "0.042" | |
| }, | |
| { | |
| "episode": 22, | |
| "loss": "0.1425", | |
| "reward": "-0.06375", | |
| "reward_std": "0.05297", | |
| "kl": "1.547", | |
| "completion_length": "140", | |
| "clipped_ratio": "0", | |
| "learning_rate": "2.1e-05", | |
| "grad_norm": "1.714", | |
| "epoch": "0.044" | |
| }, | |
| { | |
| "episode": 23, | |
| "loss": "0.06482", | |
| "reward": "0.1825", | |
| "reward_std": "0.4707", | |
| "kl": "1.429", | |
| "completion_length": "101.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "2.2e-05", | |
| "grad_norm": "1.443", | |
| "epoch": "0.046" | |
| }, | |
| { | |
| "episode": 24, | |
| "loss": "0.03516", | |
| "reward": "-0.035", | |
| "reward_std": "0.05774", | |
| "kl": "1.529", | |
| "completion_length": "93.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "2.3e-05", | |
| "grad_norm": "1.834", | |
| "epoch": "0.048" | |
| }, | |
| { | |
| "episode": 25, | |
| "loss": "0.2135", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.349", | |
| "completion_length": "140.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "2.4e-05", | |
| "grad_norm": "1.748", | |
| "epoch": "0.05" | |
| }, | |
| { | |
| "episode": 26, | |
| "loss": "-0.0988", | |
| "reward": "-0.03875", | |
| "reward_std": "0.05375", | |
| "kl": "1.452", | |
| "completion_length": "110.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "2.5e-05", | |
| "grad_norm": "1.824", | |
| "epoch": "0.052" | |
| }, | |
| { | |
| "episode": 27, | |
| "loss": "0.08622", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.599", | |
| "completion_length": "92", | |
| "clipped_ratio": "0", | |
| "learning_rate": "2.6e-05", | |
| "grad_norm": "1.645", | |
| "epoch": "0.054" | |
| }, | |
| { | |
| "episode": 28, | |
| "loss": "0.2779", | |
| "reward": "-0.035", | |
| "reward_std": "0.05774", | |
| "kl": "1.387", | |
| "completion_length": "158", | |
| "clipped_ratio": "0", | |
| "learning_rate": "2.7e-05", | |
| "grad_norm": "1.36", | |
| "epoch": "0.056" | |
| }, | |
| { | |
| "episode": 29, | |
| "loss": "0.03327", | |
| "reward": "-0.085", | |
| "reward_std": "0", | |
| "kl": "1.678", | |
| "completion_length": "83", | |
| "clipped_ratio": "0", | |
| "learning_rate": "2.8e-05", | |
| "grad_norm": "0.0967", | |
| "epoch": "0.058" | |
| }, | |
| { | |
| "episode": 30, | |
| "loss": "0.02572", | |
| "reward": "-0.06", | |
| "reward_std": "0.05", | |
| "kl": "1.724", | |
| "completion_length": "89.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "2.9e-05", | |
| "grad_norm": "1.624", | |
| "epoch": "0.06" | |
| }, | |
| { | |
| "episode": 31, | |
| "loss": "0.02943", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.541", | |
| "completion_length": "103.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3e-05", | |
| "grad_norm": "0.04806", | |
| "epoch": "0.062" | |
| }, | |
| { | |
| "episode": 32, | |
| "loss": "0.01911", | |
| "reward": "-0.03875", | |
| "reward_std": "0.06237", | |
| "kl": "1.364", | |
| "completion_length": "98", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.1e-05", | |
| "grad_norm": "1.499", | |
| "epoch": "0.064" | |
| }, | |
| { | |
| "episode": 33, | |
| "loss": "-0.3277", | |
| "reward": "-0.06375", | |
| "reward_std": "0.05297", | |
| "kl": "1.405", | |
| "completion_length": "77.75", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.2e-05", | |
| "grad_norm": "1.937", | |
| "epoch": "0.066" | |
| }, | |
| { | |
| "episode": 34, | |
| "loss": "-0.1782", | |
| "reward": "-0.08875", | |
| "reward_std": "0.0075", | |
| "kl": "1.544", | |
| "completion_length": "88.25", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.3e-05", | |
| "grad_norm": "1.418", | |
| "epoch": "0.068" | |
| }, | |
| { | |
| "episode": 35, | |
| "loss": "0.08649", | |
| "reward": "-0.035", | |
| "reward_std": "0.05774", | |
| "kl": "1.487", | |
| "completion_length": "106", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.4e-05", | |
| "grad_norm": "1.569", | |
| "epoch": "0.07" | |
| }, | |
| { | |
| "episode": 36, | |
| "loss": "-0.0006697", | |
| "reward": "-0.035", | |
| "reward_std": "0.05774", | |
| "kl": "1.382", | |
| "completion_length": "99.75", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.5e-05", | |
| "grad_norm": "1.449", | |
| "epoch": "0.072" | |
| }, | |
| { | |
| "episode": 37, | |
| "loss": "-0.02239", | |
| "reward": "0.01125", | |
| "reward_std": "0.0075", | |
| "kl": "1.371", | |
| "completion_length": "114.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.6e-05", | |
| "grad_norm": "1.406", | |
| "epoch": "0.074" | |
| }, | |
| { | |
| "episode": 38, | |
| "loss": "0.03101", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.58", | |
| "completion_length": "95.75", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.7e-05", | |
| "grad_norm": "0.06464", | |
| "epoch": "0.076" | |
| }, | |
| { | |
| "episode": 39, | |
| "loss": "0.05389", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.723", | |
| "completion_length": "89.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.8e-05", | |
| "grad_norm": "1.589", | |
| "epoch": "0.078" | |
| }, | |
| { | |
| "episode": 40, | |
| "loss": "0.01259", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.451", | |
| "completion_length": "107.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.9e-05", | |
| "grad_norm": "1.351", | |
| "epoch": "0.08" | |
| }, | |
| { | |
| "episode": 41, | |
| "loss": "0.07202", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.776", | |
| "completion_length": "92.25", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4e-05", | |
| "grad_norm": "1.879", | |
| "epoch": "0.082" | |
| }, | |
| { | |
| "episode": 42, | |
| "loss": "0.04993", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.674", | |
| "completion_length": "82.25", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.1e-05", | |
| "grad_norm": "1.664", | |
| "epoch": "0.084" | |
| }, | |
| { | |
| "episode": 43, | |
| "loss": "-0.009573", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.592", | |
| "completion_length": "88.25", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.2e-05", | |
| "grad_norm": "1.511", | |
| "epoch": "0.086" | |
| }, | |
| { | |
| "episode": 44, | |
| "loss": "-0.1045", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.62", | |
| "completion_length": "96.25", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.3e-05", | |
| "grad_norm": "1.58", | |
| "epoch": "0.088" | |
| }, | |
| { | |
| "episode": 45, | |
| "loss": "0.09343", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.692", | |
| "completion_length": "87.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.4e-05", | |
| "grad_norm": "1.955", | |
| "epoch": "0.09" | |
| }, | |
| { | |
| "episode": 46, | |
| "loss": "0.03445", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.762", | |
| "completion_length": "87.75", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.5e-05", | |
| "grad_norm": "0.1226", | |
| "epoch": "0.092" | |
| }, | |
| { | |
| "episode": 47, | |
| "loss": "-0.09708", | |
| "reward": "-0.06", | |
| "reward_std": "0.05", | |
| "kl": "1.557", | |
| "completion_length": "90.75", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.6e-05", | |
| "grad_norm": "1.661", | |
| "epoch": "0.094" | |
| }, | |
| { | |
| "episode": 48, | |
| "loss": "0.03128", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.661", | |
| "completion_length": "90", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.7e-05", | |
| "grad_norm": "0.07655", | |
| "epoch": "0.096" | |
| }, | |
| { | |
| "episode": 49, | |
| "loss": "0.03093", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.609", | |
| "completion_length": "102.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.8e-05", | |
| "grad_norm": "0.06835", | |
| "epoch": "0.098" | |
| }, | |
| { | |
| "episode": 50, | |
| "loss": "0.0294", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.499", | |
| "completion_length": "100.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.9e-05", | |
| "grad_norm": "0.04857", | |
| "epoch": "0.1" | |
| }, | |
| { | |
| "episode": 51, | |
| "loss": "-0.08929", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.549", | |
| "completion_length": "104", | |
| "clipped_ratio": "0", | |
| "learning_rate": "5e-05", | |
| "grad_norm": "1.356", | |
| "epoch": "0.102" | |
| }, | |
| { | |
| "episode": 52, | |
| "loss": "0.2245", | |
| "reward": "-0.035", | |
| "reward_std": "0.05774", | |
| "kl": "1.576", | |
| "completion_length": "137.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.989e-05", | |
| "grad_norm": "1.731", | |
| "epoch": "0.104" | |
| }, | |
| { | |
| "episode": 53, | |
| "loss": "0.2212", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.381", | |
| "completion_length": "118", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.978e-05", | |
| "grad_norm": "1.558", | |
| "epoch": "0.106" | |
| }, | |
| { | |
| "episode": 54, | |
| "loss": "0.03415", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.736", | |
| "completion_length": "89.25", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.967e-05", | |
| "grad_norm": "0.07583", | |
| "epoch": "0.108" | |
| }, | |
| { | |
| "episode": 55, | |
| "loss": "0.03483", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.754", | |
| "completion_length": "97", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.956e-05", | |
| "grad_norm": "0.09679", | |
| "epoch": "0.11" | |
| }, | |
| { | |
| "episode": 56, | |
| "loss": "0.05668", | |
| "reward": "-0.01375", | |
| "reward_std": "0.04802", | |
| "kl": "1.489", | |
| "completion_length": "107.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.944e-05", | |
| "grad_norm": "1.737", | |
| "epoch": "0.112" | |
| }, | |
| { | |
| "episode": 57, | |
| "loss": "0.03395", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.723", | |
| "completion_length": "88.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.933e-05", | |
| "grad_norm": "0.07683", | |
| "epoch": "0.114" | |
| }, | |
| { | |
| "episode": 58, | |
| "loss": "0.03335", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.681", | |
| "completion_length": "83", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.922e-05", | |
| "grad_norm": "0.0568", | |
| "epoch": "0.116" | |
| }, | |
| { | |
| "episode": 59, | |
| "loss": "0.03198", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.614", | |
| "completion_length": "100", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.911e-05", | |
| "grad_norm": "0.06101", | |
| "epoch": "0.118" | |
| }, | |
| { | |
| "episode": 60, | |
| "loss": "0.03004", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.525", | |
| "completion_length": "90.75", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.9e-05", | |
| "grad_norm": "0.06637", | |
| "epoch": "0.12" | |
| }, | |
| { | |
| "episode": 61, | |
| "loss": "0.1298", | |
| "reward": "-0.035", | |
| "reward_std": "0.05774", | |
| "kl": "1.597", | |
| "completion_length": "103.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.889e-05", | |
| "grad_norm": "1.57", | |
| "epoch": "0.122" | |
| }, | |
| { | |
| "episode": 62, | |
| "loss": "0.03358", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.721", | |
| "completion_length": "87", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.878e-05", | |
| "grad_norm": "0.05775", | |
| "epoch": "0.124" | |
| }, | |
| { | |
| "episode": 63, | |
| "loss": "0.05816", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.336", | |
| "completion_length": "120.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.867e-05", | |
| "grad_norm": "1.434", | |
| "epoch": "0.126" | |
| }, | |
| { | |
| "episode": 64, | |
| "loss": "-0.1117", | |
| "reward": "-0.0175", | |
| "reward_std": "0.05545", | |
| "kl": "1.377", | |
| "completion_length": "109", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.856e-05", | |
| "grad_norm": "1.512", | |
| "epoch": "0.128" | |
| }, | |
| { | |
| "episode": 65, | |
| "loss": "0.0283", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.431", | |
| "completion_length": "112.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.844e-05", | |
| "grad_norm": "0.05251", | |
| "epoch": "0.13" | |
| }, | |
| { | |
| "episode": 66, | |
| "loss": "0.02255", | |
| "reward": "-0.03875", | |
| "reward_std": "0.06237", | |
| "kl": "1.395", | |
| "completion_length": "103.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.833e-05", | |
| "grad_norm": "1.503", | |
| "epoch": "0.132" | |
| }, | |
| { | |
| "episode": 67, | |
| "loss": "0.0259", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.301", | |
| "completion_length": "111.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.822e-05", | |
| "grad_norm": "0.05156", | |
| "epoch": "0.134" | |
| }, | |
| { | |
| "episode": 68, | |
| "loss": "0.02639", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.364", | |
| "completion_length": "118.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.811e-05", | |
| "grad_norm": "0.03931", | |
| "epoch": "0.136" | |
| }, | |
| { | |
| "episode": 69, | |
| "loss": "0.02672", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.366", | |
| "completion_length": "103", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.8e-05", | |
| "grad_norm": "0.04079", | |
| "epoch": "0.138" | |
| }, | |
| { | |
| "episode": 70, | |
| "loss": "0.05654", | |
| "reward": "-0.01375", | |
| "reward_std": "0.0575", | |
| "kl": "1.344", | |
| "completion_length": "108.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.789e-05", | |
| "grad_norm": "1.761", | |
| "epoch": "0.14" | |
| }, | |
| { | |
| "episode": 71, | |
| "loss": "0.03006", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.524", | |
| "completion_length": "106.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.778e-05", | |
| "grad_norm": "0.1124", | |
| "epoch": "0.142" | |
| }, | |
| { | |
| "episode": 72, | |
| "loss": "0.02528", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.279", | |
| "completion_length": "125.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.767e-05", | |
| "grad_norm": "0.05816", | |
| "epoch": "0.144" | |
| }, | |
| { | |
| "episode": 73, | |
| "loss": "0.208", | |
| "reward": "-0.035", | |
| "reward_std": "0.05774", | |
| "kl": "1.469", | |
| "completion_length": "131", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.756e-05", | |
| "grad_norm": "1.413", | |
| "epoch": "0.146" | |
| }, | |
| { | |
| "episode": 74, | |
| "loss": "0.06406", | |
| "reward": "-0.035", | |
| "reward_std": "0.05774", | |
| "kl": "1.347", | |
| "completion_length": "121.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.744e-05", | |
| "grad_norm": "1.338", | |
| "epoch": "0.148" | |
| }, | |
| { | |
| "episode": 75, | |
| "loss": "0.02868", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.465", | |
| "completion_length": "92.25", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.733e-05", | |
| "grad_norm": "0.05968", | |
| "epoch": "0.15" | |
| }, | |
| { | |
| "episode": 76, | |
| "loss": "0.03725", | |
| "reward": "-0.01375", | |
| "reward_std": "0.0575", | |
| "kl": "1.292", | |
| "completion_length": "119.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.722e-05", | |
| "grad_norm": "2.02", | |
| "epoch": "0.152" | |
| }, | |
| { | |
| "episode": 77, | |
| "loss": "0.02851", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.446", | |
| "completion_length": "118", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.711e-05", | |
| "grad_norm": "0.15", | |
| "epoch": "0.154" | |
| }, | |
| { | |
| "episode": 78, | |
| "loss": "0.07068", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "3.348", | |
| "completion_length": "104", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.7e-05", | |
| "grad_norm": "4.452", | |
| "epoch": "0.156" | |
| }, | |
| { | |
| "episode": 79, | |
| "loss": "0.02455", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.25", | |
| "completion_length": "132.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.689e-05", | |
| "grad_norm": "0.03926", | |
| "epoch": "0.158" | |
| }, | |
| { | |
| "episode": 80, | |
| "loss": "0.02639", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.361", | |
| "completion_length": "116.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.678e-05", | |
| "grad_norm": "0.1211", | |
| "epoch": "0.16" | |
| }, | |
| { | |
| "episode": 81, | |
| "loss": "0.02788", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.419", | |
| "completion_length": "100.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.667e-05", | |
| "grad_norm": "0.03858", | |
| "epoch": "0.162" | |
| }, | |
| { | |
| "episode": 82, | |
| "loss": "0.03285", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.66", | |
| "completion_length": "89.75", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.656e-05", | |
| "grad_norm": "0.3443", | |
| "epoch": "0.164" | |
| }, | |
| { | |
| "episode": 83, | |
| "loss": "0.02415", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.255", | |
| "completion_length": "125.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.644e-05", | |
| "grad_norm": "0.04701", | |
| "epoch": "0.166" | |
| }, | |
| { | |
| "episode": 84, | |
| "loss": "0.1476", | |
| "reward": "-0.06375", | |
| "reward_std": "0.05297", | |
| "kl": "1.054", | |
| "completion_length": "144.2", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "4.633e-05", | |
| "grad_norm": "1.043", | |
| "epoch": "0.168" | |
| }, | |
| { | |
| "episode": 85, | |
| "loss": "0.02414", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.324", | |
| "completion_length": "133", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.622e-05", | |
| "grad_norm": "0.1377", | |
| "epoch": "0.17" | |
| }, | |
| { | |
| "episode": 86, | |
| "loss": "0.4588", | |
| "reward": "-0.0175", | |
| "reward_std": "0.05545", | |
| "kl": "1.243", | |
| "completion_length": "134.2", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "4.611e-05", | |
| "grad_norm": "1.549", | |
| "epoch": "0.172" | |
| }, | |
| { | |
| "episode": 87, | |
| "loss": "0.05721", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.323", | |
| "completion_length": "116.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.6e-05", | |
| "grad_norm": "1.334", | |
| "epoch": "0.174" | |
| }, | |
| { | |
| "episode": 88, | |
| "loss": "0.0259", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.336", | |
| "completion_length": "105.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.589e-05", | |
| "grad_norm": "0.04917", | |
| "epoch": "0.176" | |
| }, | |
| { | |
| "episode": 89, | |
| "loss": "0.03014", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.515", | |
| "completion_length": "104.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.578e-05", | |
| "grad_norm": "0.1387", | |
| "epoch": "0.178" | |
| }, | |
| { | |
| "episode": 90, | |
| "loss": "0.02943", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.476", | |
| "completion_length": "117.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.567e-05", | |
| "grad_norm": "0.2111", | |
| "epoch": "0.18" | |
| }, | |
| { | |
| "episode": 91, | |
| "loss": "0.1492", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.187", | |
| "completion_length": "135.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.556e-05", | |
| "grad_norm": "1.204", | |
| "epoch": "0.182" | |
| }, | |
| { | |
| "episode": 92, | |
| "loss": "0.00894", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.269", | |
| "completion_length": "115.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.544e-05", | |
| "grad_norm": "1.163", | |
| "epoch": "0.184" | |
| }, | |
| { | |
| "episode": 93, | |
| "loss": "0.02674", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.44", | |
| "completion_length": "102.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.533e-05", | |
| "grad_norm": "0.1047", | |
| "epoch": "0.186" | |
| }, | |
| { | |
| "episode": 94, | |
| "loss": "0.02273", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.198", | |
| "completion_length": "133.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.522e-05", | |
| "grad_norm": "0.05807", | |
| "epoch": "0.188" | |
| }, | |
| { | |
| "episode": 95, | |
| "loss": "0.02433", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.232", | |
| "completion_length": "124", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.511e-05", | |
| "grad_norm": "0.04012", | |
| "epoch": "0.19" | |
| }, | |
| { | |
| "episode": 96, | |
| "loss": "0.02564", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.31", | |
| "completion_length": "111.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.5e-05", | |
| "grad_norm": "0.06366", | |
| "epoch": "0.192" | |
| }, | |
| { | |
| "episode": 97, | |
| "loss": "0.02609", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.337", | |
| "completion_length": "112.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.489e-05", | |
| "grad_norm": "0.05895", | |
| "epoch": "0.194" | |
| }, | |
| { | |
| "episode": 98, | |
| "loss": "0.02595", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.339", | |
| "completion_length": "102.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.478e-05", | |
| "grad_norm": "0.07423", | |
| "epoch": "0.196" | |
| }, | |
| { | |
| "episode": 99, | |
| "loss": "-0.2551", | |
| "reward": "-0.01375", | |
| "reward_std": "0.0575", | |
| "kl": "1.134", | |
| "completion_length": "131.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.467e-05", | |
| "grad_norm": "1.3", | |
| "epoch": "0.198" | |
| }, | |
| { | |
| "episode": 100, | |
| "loss": "0.0307", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.706", | |
| "completion_length": "85.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.456e-05", | |
| "grad_norm": "0.113", | |
| "epoch": "0.2" | |
| }, | |
| { | |
| "episode": 101, | |
| "loss": "0.02324", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.188", | |
| "completion_length": "121.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.444e-05", | |
| "grad_norm": "0.04005", | |
| "epoch": "0.202" | |
| }, | |
| { | |
| "episode": 102, | |
| "loss": "0.02365", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.209", | |
| "completion_length": "119.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.433e-05", | |
| "grad_norm": "0.03859", | |
| "epoch": "0.204" | |
| }, | |
| { | |
| "episode": 103, | |
| "loss": "-0.1069", | |
| "reward": "-0.01375", | |
| "reward_std": "0.0575", | |
| "kl": "1.013", | |
| "completion_length": "123.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.422e-05", | |
| "grad_norm": "1.101", | |
| "epoch": "0.206" | |
| }, | |
| { | |
| "episode": 104, | |
| "loss": "-0.06919", | |
| "reward": "-0.01375", | |
| "reward_std": "0.0575", | |
| "kl": "1.118", | |
| "completion_length": "129.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.411e-05", | |
| "grad_norm": "1.286", | |
| "epoch": "0.208" | |
| }, | |
| { | |
| "episode": 105, | |
| "loss": "0.02123", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.138", | |
| "completion_length": "142", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.4e-05", | |
| "grad_norm": "0.04361", | |
| "epoch": "0.21" | |
| }, | |
| { | |
| "episode": 106, | |
| "loss": "0.02399", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.212", | |
| "completion_length": "128", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.389e-05", | |
| "grad_norm": "0.04805", | |
| "epoch": "0.212" | |
| }, | |
| { | |
| "episode": 107, | |
| "loss": "0.3564", | |
| "reward": "-0.01375", | |
| "reward_std": "0.0575", | |
| "kl": "1.031", | |
| "completion_length": "152.8", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "4.378e-05", | |
| "grad_norm": "1.439", | |
| "epoch": "0.214" | |
| }, | |
| { | |
| "episode": 108, | |
| "loss": "0.02283", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.166", | |
| "completion_length": "129.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.367e-05", | |
| "grad_norm": "0.05553", | |
| "epoch": "0.216" | |
| }, | |
| { | |
| "episode": 109, | |
| "loss": "-0.04348", | |
| "reward": "-0.0675", | |
| "reward_std": "0.05545", | |
| "kl": "0.5512", | |
| "completion_length": "202.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.356e-05", | |
| "grad_norm": "1.05", | |
| "epoch": "0.218" | |
| }, | |
| { | |
| "episode": 110, | |
| "loss": "0.02031", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.051", | |
| "completion_length": "137.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.344e-05", | |
| "grad_norm": "0.04941", | |
| "epoch": "0.22" | |
| }, | |
| { | |
| "episode": 111, | |
| "loss": "0.1065", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.289", | |
| "completion_length": "118.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.333e-05", | |
| "grad_norm": "1.255", | |
| "epoch": "0.222" | |
| }, | |
| { | |
| "episode": 112, | |
| "loss": "0.02114", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.096", | |
| "completion_length": "133.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.322e-05", | |
| "grad_norm": "0.06218", | |
| "epoch": "0.224" | |
| }, | |
| { | |
| "episode": 113, | |
| "loss": "0.02359", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.246", | |
| "completion_length": "112", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.311e-05", | |
| "grad_norm": "0.03646", | |
| "epoch": "0.226" | |
| }, | |
| { | |
| "episode": 114, | |
| "loss": "-0.02765", | |
| "reward": "-0.0175", | |
| "reward_std": "0.05545", | |
| "kl": "1.039", | |
| "completion_length": "101", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.3e-05", | |
| "grad_norm": "1.376", | |
| "epoch": "0.228" | |
| }, | |
| { | |
| "episode": 115, | |
| "loss": "0.0184", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "0.9662", | |
| "completion_length": "165.5", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "4.289e-05", | |
| "grad_norm": "0.04314", | |
| "epoch": "0.23" | |
| }, | |
| { | |
| "episode": 116, | |
| "loss": "0.1954", | |
| "reward": "-0.03875", | |
| "reward_std": "0.06237", | |
| "kl": "1.071", | |
| "completion_length": "158", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "4.278e-05", | |
| "grad_norm": "1.173", | |
| "epoch": "0.232" | |
| }, | |
| { | |
| "episode": 117, | |
| "loss": "0.02071", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.051", | |
| "completion_length": "130", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.267e-05", | |
| "grad_norm": "0.02952", | |
| "epoch": "0.234" | |
| }, | |
| { | |
| "episode": 118, | |
| "loss": "0.1281", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "1.111", | |
| "completion_length": "128.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.256e-05", | |
| "grad_norm": "1.226", | |
| "epoch": "0.236" | |
| }, | |
| { | |
| "episode": 119, | |
| "loss": "0.1146", | |
| "reward": "-0.01", | |
| "reward_std": "0.05", | |
| "kl": "0.9954", | |
| "completion_length": "124.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.244e-05", | |
| "grad_norm": "1.035", | |
| "epoch": "0.238" | |
| }, | |
| { | |
| "episode": 120, | |
| "loss": "0.09495", | |
| "reward": "-0.0425", | |
| "reward_std": "0.0664", | |
| "kl": "0.6759", | |
| "completion_length": "192.2", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "4.233e-05", | |
| "grad_norm": "0.9413", | |
| "epoch": "0.24" | |
| }, | |
| { | |
| "episode": 121, | |
| "loss": "0.01714", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "0.894", | |
| "completion_length": "164.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.222e-05", | |
| "grad_norm": "0.04506", | |
| "epoch": "0.242" | |
| }, | |
| { | |
| "episode": 122, | |
| "loss": "-0.1289", | |
| "reward": "-0.01375", | |
| "reward_std": "0.0575", | |
| "kl": "1.226", | |
| "completion_length": "102.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.211e-05", | |
| "grad_norm": "1.349", | |
| "epoch": "0.244" | |
| }, | |
| { | |
| "episode": 123, | |
| "loss": "0.02344", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.195", | |
| "completion_length": "115.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.2e-05", | |
| "grad_norm": "0.0458", | |
| "epoch": "0.246" | |
| }, | |
| { | |
| "episode": 124, | |
| "loss": "0.02134", | |
| "reward": "0.015", | |
| "reward_std": "0", | |
| "kl": "1.169", | |
| "completion_length": "125.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.189e-05", | |
| "grad_norm": "0.06143", | |
| "epoch": "0.248" | |
| }, | |
| { | |
| "episode": 125, | |
| "loss": "-0.1984", | |
| "reward": "0.01125", | |
| "reward_std": "0.0075", | |
| "kl": "1.124", | |
| "completion_length": "116.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.178e-05", | |
| "grad_norm": "1.044", | |
| "epoch": "0.25" | |
| }, | |
| { | |
| "episode": 126, | |
| "loss": "0.0192", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.9959", | |
| "completion_length": "133.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.167e-05", | |
| "grad_norm": "0.04122", | |
| "epoch": "0.252" | |
| }, | |
| { | |
| "episode": 127, | |
| "loss": "0.01727", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.977", | |
| "completion_length": "147.5", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "4.156e-05", | |
| "grad_norm": "0.02834", | |
| "epoch": "0.254" | |
| }, | |
| { | |
| "episode": 128, | |
| "loss": "0.009876", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "1.032", | |
| "completion_length": "156.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.144e-05", | |
| "grad_norm": "0.9371", | |
| "epoch": "0.256" | |
| }, | |
| { | |
| "episode": 129, | |
| "loss": "-0.09492", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "1.074", | |
| "completion_length": "148.5", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "4.133e-05", | |
| "grad_norm": "0.845", | |
| "epoch": "0.258" | |
| }, | |
| { | |
| "episode": 130, | |
| "loss": "0.02401", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "1.203", | |
| "completion_length": "104.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.122e-05", | |
| "grad_norm": "0.05182", | |
| "epoch": "0.26" | |
| }, | |
| { | |
| "episode": 131, | |
| "loss": "-0.2198", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.9641", | |
| "completion_length": "162", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "4.111e-05", | |
| "grad_norm": "0.8911", | |
| "epoch": "0.262" | |
| }, | |
| { | |
| "episode": 132, | |
| "loss": "0.08979", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "1.071", | |
| "completion_length": "107.2", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.1e-05", | |
| "grad_norm": "1.38", | |
| "epoch": "0.264" | |
| }, | |
| { | |
| "episode": 133, | |
| "loss": "0.0148", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.8439", | |
| "completion_length": "204", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "4.089e-05", | |
| "grad_norm": "0.03386", | |
| "epoch": "0.266" | |
| }, | |
| { | |
| "episode": 134, | |
| "loss": "0.0148", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.8089", | |
| "completion_length": "192", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "4.078e-05", | |
| "grad_norm": "0.0373", | |
| "epoch": "0.268" | |
| }, | |
| { | |
| "episode": 135, | |
| "loss": "0.2167", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.8866", | |
| "completion_length": "182.2", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "4.067e-05", | |
| "grad_norm": "0.9069", | |
| "epoch": "0.27" | |
| }, | |
| { | |
| "episode": 136, | |
| "loss": "0.02339", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "1.197", | |
| "completion_length": "107.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.056e-05", | |
| "grad_norm": "0.06824", | |
| "epoch": "0.272" | |
| }, | |
| { | |
| "episode": 137, | |
| "loss": "-0.1594", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "1.195", | |
| "completion_length": "150.2", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "4.044e-05", | |
| "grad_norm": "0.8749", | |
| "epoch": "0.274" | |
| }, | |
| { | |
| "episode": 138, | |
| "loss": "0.021", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "1.15", | |
| "completion_length": "121.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "4.033e-05", | |
| "grad_norm": "0.0369", | |
| "epoch": "0.276" | |
| }, | |
| { | |
| "episode": 139, | |
| "loss": "0.0129", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.7527", | |
| "completion_length": "185", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "4.022e-05", | |
| "grad_norm": "0.02335", | |
| "epoch": "0.278" | |
| }, | |
| { | |
| "episode": 140, | |
| "loss": "0.0135", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.7518", | |
| "completion_length": "191.5", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "4.011e-05", | |
| "grad_norm": "0.03046", | |
| "epoch": "0.28" | |
| }, | |
| { | |
| "episode": 141, | |
| "loss": "0.01559", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.8364", | |
| "completion_length": "197", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "4e-05", | |
| "grad_norm": "0.1297", | |
| "epoch": "0.282" | |
| }, | |
| { | |
| "episode": 142, | |
| "loss": "0.02244", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "1.18", | |
| "completion_length": "119.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.989e-05", | |
| "grad_norm": "0.04509", | |
| "epoch": "0.284" | |
| }, | |
| { | |
| "episode": 143, | |
| "loss": "-0.06772", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "1.04", | |
| "completion_length": "113", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.978e-05", | |
| "grad_norm": "1.059", | |
| "epoch": "0.286" | |
| }, | |
| { | |
| "episode": 144, | |
| "loss": "-0.1361", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.8591", | |
| "completion_length": "164", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.967e-05", | |
| "grad_norm": "0.7481", | |
| "epoch": "0.288" | |
| }, | |
| { | |
| "episode": 145, | |
| "loss": "0.009652", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.7233", | |
| "completion_length": "170.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.956e-05", | |
| "grad_norm": "0.8965", | |
| "epoch": "0.29" | |
| }, | |
| { | |
| "episode": 146, | |
| "loss": "0.01383", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.8261", | |
| "completion_length": "172", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.944e-05", | |
| "grad_norm": "0.01843", | |
| "epoch": "0.292" | |
| }, | |
| { | |
| "episode": 147, | |
| "loss": "0.1679", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.7532", | |
| "completion_length": "195.5", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.933e-05", | |
| "grad_norm": "0.8574", | |
| "epoch": "0.294" | |
| }, | |
| { | |
| "episode": 148, | |
| "loss": "0.01479", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.8126", | |
| "completion_length": "161.5", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.922e-05", | |
| "grad_norm": "0.04673", | |
| "epoch": "0.296" | |
| }, | |
| { | |
| "episode": 149, | |
| "loss": "0.09016", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.6905", | |
| "completion_length": "222", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.911e-05", | |
| "grad_norm": "0.9023", | |
| "epoch": "0.298" | |
| }, | |
| { | |
| "episode": 150, | |
| "loss": "0.01703", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.9712", | |
| "completion_length": "154.8", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.9e-05", | |
| "grad_norm": "0.04395", | |
| "epoch": "0.3" | |
| }, | |
| { | |
| "episode": 151, | |
| "loss": "0.01369", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.8077", | |
| "completion_length": "179.5", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.889e-05", | |
| "grad_norm": "0.02968", | |
| "epoch": "0.302" | |
| }, | |
| { | |
| "episode": 152, | |
| "loss": "0.01495", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.7448", | |
| "completion_length": "236.2", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.878e-05", | |
| "grad_norm": "0.2067", | |
| "epoch": "0.304" | |
| }, | |
| { | |
| "episode": 153, | |
| "loss": "0.01446", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.8869", | |
| "completion_length": "174.5", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.867e-05", | |
| "grad_norm": "0.02209", | |
| "epoch": "0.306" | |
| }, | |
| { | |
| "episode": 154, | |
| "loss": "0.01308", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.7685", | |
| "completion_length": "191", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.856e-05", | |
| "grad_norm": "0.02081", | |
| "epoch": "0.308" | |
| }, | |
| { | |
| "episode": 155, | |
| "loss": "0.09928", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.7042", | |
| "completion_length": "218.2", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.844e-05", | |
| "grad_norm": "0.8511", | |
| "epoch": "0.31" | |
| }, | |
| { | |
| "episode": 156, | |
| "loss": "0.1472", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.8815", | |
| "completion_length": "202", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.833e-05", | |
| "grad_norm": "1.008", | |
| "epoch": "0.312" | |
| }, | |
| { | |
| "episode": 157, | |
| "loss": "0.01473", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.7722", | |
| "completion_length": "184.5", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.822e-05", | |
| "grad_norm": "0.1047", | |
| "epoch": "0.314" | |
| }, | |
| { | |
| "episode": 158, | |
| "loss": "0.01633", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.8847", | |
| "completion_length": "166.5", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.811e-05", | |
| "grad_norm": "0.06288", | |
| "epoch": "0.316" | |
| }, | |
| { | |
| "episode": 159, | |
| "loss": "0.01355", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.7771", | |
| "completion_length": "177.2", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.8e-05", | |
| "grad_norm": "0.05157", | |
| "epoch": "0.318" | |
| }, | |
| { | |
| "episode": 160, | |
| "loss": "0.06912", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.592", | |
| "completion_length": "229.5", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.789e-05", | |
| "grad_norm": "0.7783", | |
| "epoch": "0.32" | |
| }, | |
| { | |
| "episode": 161, | |
| "loss": "0.01657", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "1.046", | |
| "completion_length": "135.5", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.778e-05", | |
| "grad_norm": "0.02938", | |
| "epoch": "0.322" | |
| }, | |
| { | |
| "episode": 162, | |
| "loss": "0.01172", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.6083", | |
| "completion_length": "232.5", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.767e-05", | |
| "grad_norm": "0.02841", | |
| "epoch": "0.324" | |
| }, | |
| { | |
| "episode": 163, | |
| "loss": "0.1041", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.9652", | |
| "completion_length": "218.5", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.756e-05", | |
| "grad_norm": "0.9357", | |
| "epoch": "0.326" | |
| }, | |
| { | |
| "episode": 164, | |
| "loss": "0.02038", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "1.034", | |
| "completion_length": "114.8", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.744e-05", | |
| "grad_norm": "0.03502", | |
| "epoch": "0.328" | |
| }, | |
| { | |
| "episode": 165, | |
| "loss": "-0.2373", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.6106", | |
| "completion_length": "219.5", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.733e-05", | |
| "grad_norm": "0.6781", | |
| "epoch": "0.33" | |
| }, | |
| { | |
| "episode": 166, | |
| "loss": "-0.1384", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "1.235", | |
| "completion_length": "123.5", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.722e-05", | |
| "grad_norm": "0.8199", | |
| "epoch": "0.332" | |
| }, | |
| { | |
| "episode": 167, | |
| "loss": "0.1289", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.6907", | |
| "completion_length": "207.8", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.711e-05", | |
| "grad_norm": "0.8933", | |
| "epoch": "0.334" | |
| }, | |
| { | |
| "episode": 168, | |
| "loss": "0.05713", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.5286", | |
| "completion_length": "234", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.7e-05", | |
| "grad_norm": "0.6729", | |
| "epoch": "0.336" | |
| }, | |
| { | |
| "episode": 169, | |
| "loss": "0.01499", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.8495", | |
| "completion_length": "174.2", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.689e-05", | |
| "grad_norm": "0.1389", | |
| "epoch": "0.338" | |
| }, | |
| { | |
| "episode": 170, | |
| "loss": "0.01559", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.8565", | |
| "completion_length": "168.5", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.678e-05", | |
| "grad_norm": "0.04733", | |
| "epoch": "0.34" | |
| }, | |
| { | |
| "episode": 171, | |
| "loss": "-0.09501", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.8351", | |
| "completion_length": "168", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.667e-05", | |
| "grad_norm": "0.84", | |
| "epoch": "0.342" | |
| }, | |
| { | |
| "episode": 172, | |
| "loss": "-0.2282", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.862", | |
| "completion_length": "177", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.656e-05", | |
| "grad_norm": "0.6844", | |
| "epoch": "0.344" | |
| }, | |
| { | |
| "episode": 173, | |
| "loss": "0.01553", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.8731", | |
| "completion_length": "185.5", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.644e-05", | |
| "grad_norm": "0.2459", | |
| "epoch": "0.346" | |
| }, | |
| { | |
| "episode": 174, | |
| "loss": "0.0121", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.68", | |
| "completion_length": "206.8", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.633e-05", | |
| "grad_norm": "0.02169", | |
| "epoch": "0.348" | |
| }, | |
| { | |
| "episode": 175, | |
| "loss": "0.01206", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.6513", | |
| "completion_length": "199", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.622e-05", | |
| "grad_norm": "0.04694", | |
| "epoch": "0.35" | |
| }, | |
| { | |
| "episode": 176, | |
| "loss": "0.2827", | |
| "reward": "-0.05", | |
| "reward_std": "0.05774", | |
| "kl": "0.7719", | |
| "completion_length": "170.5", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.611e-05", | |
| "grad_norm": "0.7261", | |
| "epoch": "0.352" | |
| }, | |
| { | |
| "episode": 177, | |
| "loss": "0.1534", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.7106", | |
| "completion_length": "200", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.6e-05", | |
| "grad_norm": "0.8582", | |
| "epoch": "0.354" | |
| }, | |
| { | |
| "episode": 178, | |
| "loss": "0.01187", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.5987", | |
| "completion_length": "222.5", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.589e-05", | |
| "grad_norm": "0.02377", | |
| "epoch": "0.356" | |
| }, | |
| { | |
| "episode": 179, | |
| "loss": "0.01027", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.5137", | |
| "completion_length": "256", | |
| "clipped_ratio": "1", | |
| "learning_rate": "3.578e-05", | |
| "grad_norm": "0.03761", | |
| "epoch": "0.358" | |
| }, | |
| { | |
| "episode": 180, | |
| "loss": "0.009968", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.5013", | |
| "completion_length": "246", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.567e-05", | |
| "grad_norm": "0.03003", | |
| "epoch": "0.36" | |
| }, | |
| { | |
| "episode": 181, | |
| "loss": "0.0136", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.6799", | |
| "completion_length": "256", | |
| "clipped_ratio": "1", | |
| "learning_rate": "3.556e-05", | |
| "grad_norm": "0.6516", | |
| "epoch": "0.362" | |
| }, | |
| { | |
| "episode": 182, | |
| "loss": "0.0167", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.8716", | |
| "completion_length": "147", | |
| "clipped_ratio": "0", | |
| "learning_rate": "3.544e-05", | |
| "grad_norm": "0.07296", | |
| "epoch": "0.364" | |
| }, | |
| { | |
| "episode": 183, | |
| "loss": "0.0114", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.6002", | |
| "completion_length": "206", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.533e-05", | |
| "grad_norm": "0.03717", | |
| "epoch": "0.366" | |
| }, | |
| { | |
| "episode": 184, | |
| "loss": "0.07569", | |
| "reward": "-0.05", | |
| "reward_std": "0.05774", | |
| "kl": "0.4309", | |
| "completion_length": "237.5", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.522e-05", | |
| "grad_norm": "0.6907", | |
| "epoch": "0.368" | |
| }, | |
| { | |
| "episode": 185, | |
| "loss": "0.01558", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.8331", | |
| "completion_length": "182.2", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.511e-05", | |
| "grad_norm": "0.1109", | |
| "epoch": "0.37" | |
| }, | |
| { | |
| "episode": 186, | |
| "loss": "0.08199", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.6914", | |
| "completion_length": "225", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.5e-05", | |
| "grad_norm": "0.7589", | |
| "epoch": "0.372" | |
| }, | |
| { | |
| "episode": 187, | |
| "loss": "0.01244", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.6683", | |
| "completion_length": "198.5", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.489e-05", | |
| "grad_norm": "0.02421", | |
| "epoch": "0.374" | |
| }, | |
| { | |
| "episode": 188, | |
| "loss": "0.009995", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.5007", | |
| "completion_length": "243", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.478e-05", | |
| "grad_norm": "0.0273", | |
| "epoch": "0.376" | |
| }, | |
| { | |
| "episode": 189, | |
| "loss": "0.07081", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.5754", | |
| "completion_length": "228.5", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.467e-05", | |
| "grad_norm": "0.5827", | |
| "epoch": "0.378" | |
| }, | |
| { | |
| "episode": 190, | |
| "loss": "-0.2955", | |
| "reward": "-0.05", | |
| "reward_std": "0.05774", | |
| "kl": "0.5758", | |
| "completion_length": "189", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.456e-05", | |
| "grad_norm": "0.9176", | |
| "epoch": "0.38" | |
| }, | |
| { | |
| "episode": 191, | |
| "loss": "0.01553", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.7765", | |
| "completion_length": "256", | |
| "clipped_ratio": "1", | |
| "learning_rate": "3.444e-05", | |
| "grad_norm": "0.8604", | |
| "epoch": "0.382" | |
| }, | |
| { | |
| "episode": 192, | |
| "loss": "0.01256", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.6766", | |
| "completion_length": "182.2", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.433e-05", | |
| "grad_norm": "0.01865", | |
| "epoch": "0.384" | |
| }, | |
| { | |
| "episode": 193, | |
| "loss": "-0.06049", | |
| "reward": "-0.05", | |
| "reward_std": "0.05774", | |
| "kl": "0.5448", | |
| "completion_length": "236.5", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.422e-05", | |
| "grad_norm": "0.6642", | |
| "epoch": "0.386" | |
| }, | |
| { | |
| "episode": 194, | |
| "loss": "0.01536", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.8527", | |
| "completion_length": "160.8", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.411e-05", | |
| "grad_norm": "0.03851", | |
| "epoch": "0.388" | |
| }, | |
| { | |
| "episode": 195, | |
| "loss": "0.03194", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "1.676", | |
| "completion_length": "193", | |
| "clipped_ratio": "0.25", | |
| "learning_rate": "3.4e-05", | |
| "grad_norm": "2.288", | |
| "epoch": "0.39" | |
| }, | |
| { | |
| "episode": 196, | |
| "loss": "-0.03057", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.5017", | |
| "completion_length": "249.2", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.389e-05", | |
| "grad_norm": "0.6679", | |
| "epoch": "0.392" | |
| }, | |
| { | |
| "episode": 197, | |
| "loss": "0.01438", | |
| "reward": "-0.05", | |
| "reward_std": "0.05774", | |
| "kl": "0.6339", | |
| "completion_length": "255.5", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.378e-05", | |
| "grad_norm": "0.6842", | |
| "epoch": "0.394" | |
| }, | |
| { | |
| "episode": 198, | |
| "loss": "0.01817", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.5629", | |
| "completion_length": "252.5", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.367e-05", | |
| "grad_norm": "0.699", | |
| "epoch": "0.396" | |
| }, | |
| { | |
| "episode": 199, | |
| "loss": "0.05561", | |
| "reward": "-0.05", | |
| "reward_std": "0.05774", | |
| "kl": "0.5969", | |
| "completion_length": "235.5", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.356e-05", | |
| "grad_norm": "0.662", | |
| "epoch": "0.398" | |
| }, | |
| { | |
| "episode": 200, | |
| "loss": "0.01062", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.5544", | |
| "completion_length": "223", | |
| "clipped_ratio": "0.5", | |
| "learning_rate": "3.344e-05", | |
| "grad_norm": "0.03785", | |
| "epoch": "0.4" | |
| }, | |
| { | |
| "episode": 201, | |
| "loss": "-0.09729", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.4514", | |
| "completion_length": "239", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.333e-05", | |
| "grad_norm": "0.6039", | |
| "epoch": "0.402" | |
| }, | |
| { | |
| "episode": 202, | |
| "loss": "0.01084", | |
| "reward": "0", | |
| "reward_std": "0", | |
| "kl": "0.5435", | |
| "completion_length": "249.8", | |
| "clipped_ratio": "0.75", | |
| "learning_rate": "3.322e-05", | |
| "grad_norm": "0.05489", | |
| "epoch": "0.404" | |
| }, | |
| { | |
| "episode": 203, | |
| "loss": "0.01082", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.5408", | |
| "completion_length": "256", | |
| "clipped_ratio": "1", | |
| "learning_rate": "3.311e-05", | |
| "grad_norm": "0.5738", | |
| "epoch": "0.406" | |
| }, | |
| { | |
| "episode": 204, | |
| "loss": "0.01224", | |
| "reward": "-0.025", | |
| "reward_std": "0.05", | |
| "kl": "0.6122", | |
| "completion_length": "256", | |
| "clipped_ratio": "1", | |
| "learning_rate": "3.3e-05", | |
| "grad_norm": "0.6763", | |
| "epoch": "0.408" | |
| } | |
| ] | |
| } |