| step,v_loss,q_loss,policy_loss,v_mean,q_mean,advantage_mean,exp_advantage_mean | |
| 1000,8.633156539872289e-05,0.015890419483184814,-17.1130428314209,0.07388518750667572,0.0913907140493393,-0.013603377155959606,0.9621621370315552 | |
| 2000,0.00022342226293403655,0.008890161290764809,-18.65302848815918,0.1730019897222519,0.17385894060134888,-0.014234611764550209,0.9636377096176147 | |
| 3000,0.0003153202123939991,0.006112693343311548,-18.849489212036133,0.2734842300415039,0.2571020722389221,-0.02838163636624813,0.9265838861465454 | |
| 4000,0.00036868295865133405,0.0054898555390536785,-19.068748474121094,0.3791458010673523,0.36570316553115845,-0.023557525128126144,0.9403215646743774 | |
| 5000,0.00025687876041047275,0.004053803626447916,-20.802186965942383,0.49114149808883667,0.4801120460033417,-0.020775873214006424,0.945335865020752 | |
| 6000,0.0002028238377533853,0.0025532315485179424,-20.560470581054688,0.5579091310501099,0.5511236786842346,-0.021731793880462646,0.9420556426048279 | |
| 7000,0.0002890181203838438,0.0028655496425926685,-21.101268768310547,0.6067690849304199,0.5996283292770386,-0.015843860805034637,0.9606245756149292 | |
| 8000,0.00012473930837586522,0.0037494099233299494,-21.582630157470703,0.6371811032295227,0.6307183504104614,-0.015300982631742954,0.9584085941314697 | |
| 9000,9.362320270156488e-05,0.011256005614995956,-22.086801528930664,0.695525050163269,0.6839786171913147,-0.013751563616096973,0.9622327089309692 | |
| 10000,7.372274558292702e-05,0.0069471681490540504,-23.057952880859375,0.701438307762146,0.6972486972808838,-0.006967584136873484,0.9811217784881592 | |
| 11000,5.77913669985719e-05,0.005811262875795364,-21.531478881835938,0.715826153755188,0.710740327835083,-0.010211922228336334,0.9714177846908569 | |
| 12000,2.330305323994253e-05,0.006924539804458618,-23.403247833251953,0.7225379943847656,0.7122185230255127,-0.008711772970855236,0.9747204780578613 | |
| 13000,1.91244780580746e-05,0.004158196039497852,-24.227293014526367,0.7178207635879517,0.7157298922538757,-0.0072660124860703945,0.9789519309997559 | |
| 14000,2.7676462195813656e-05,0.0024608615785837173,-23.139917373657227,0.712261974811554,0.7025079727172852,-0.005698599386960268,0.983720064163208 | |
| 15000,1.6675907318131067e-05,0.0048270137049257755,-23.838401794433594,0.7048780918121338,0.6967397928237915,-0.002392102964222431,0.9933193922042847 | |
| 16000,1.717374470899813e-05,0.0076991114765405655,-23.68405532836914,0.704494833946228,0.7028764486312866,-0.006684239022433758,0.9805353879928589 | |
| 17000,1.9749410057556815e-05,0.003989814780652523,-24.07297706604004,0.7002980709075928,0.6924494504928589,-0.00933011993765831,0.9729256629943848 | |
| 18000,3.471372474450618e-05,0.00655061611905694,-24.047462463378906,0.6986175179481506,0.6871231198310852,-0.0072408514097332954,0.979374885559082 | |
| 19000,2.730828600761015e-05,0.007088114507496357,-24.311758041381836,0.6959446668624878,0.6857842206954956,-0.003000223310664296,0.9916227459907532 | |
| 20000,2.1714029571739957e-05,0.0010327689815312624,-24.89046287536621,0.686647891998291,0.6799449324607849,-0.006227823905646801,0.9818724393844604 | |
| 21000,1.923709351103753e-05,0.002896358259022236,-24.746932983398438,0.6843904256820679,0.6809468269348145,-0.004032081458717585,0.9885591864585876 | |
| 22000,2.452788976370357e-05,0.0019037777092307806,-24.367595672607422,0.6834739446640015,0.6751296520233154,-0.010064507834613323,0.9709842205047607 | |
| 23000,2.3824793970561586e-05,0.0068470872938632965,-24.526565551757812,0.6896730661392212,0.6839350461959839,-0.0049414969980716705,0.9858757257461548 | |
| 24000,2.102752659993712e-05,0.00215637288056314,-24.872169494628906,0.6762269139289856,0.6697770953178406,-0.005311812274158001,0.9848089218139648 | |
| 25000,1.2893460734630935e-05,0.004671152215451002,-25.09964942932129,0.6790029406547546,0.6731180548667908,-0.007796351797878742,0.9772374629974365 | |
| 26000,3.001440200023353e-05,0.0044553447514772415,-25.820392608642578,0.6799694299697876,0.6667003631591797,-0.006443562917411327,0.9818300008773804 | |
| 27000,3.056092100450769e-05,0.00720272958278656,-25.036136627197266,0.6696533560752869,0.6662483215332031,-0.003895236412063241,0.989342987537384 | |
| 28000,3.1051044061314315e-05,0.0026594712398946285,-25.216712951660156,0.6740062236785889,0.671353280544281,-0.0035530405584722757,0.9904331564903259 | |
| 29000,1.032192085403949e-05,0.0032755990978330374,-25.593048095703125,0.674153208732605,0.6719672083854675,-0.004421061836183071,0.9870840311050415 | |
| 30000,2.0230429072398692e-05,0.0028659936506301165,-25.8599853515625,0.675439178943634,0.6788569688796997,-0.0044437735341489315,0.9873762130737305 | |
| 31000,3.17005324177444e-05,0.0024345070123672485,-26.05536651611328,0.6785483360290527,0.6742671728134155,-0.0058674560859799385,0.9835294485092163 | |
| 32000,3.0348874133778736e-05,0.006819955073297024,-24.843631744384766,0.6786972284317017,0.6673327088356018,-0.008128872141242027,0.9766704440116882 | |
| 33000,4.808776066056453e-05,0.012050201185047626,-25.563095092773438,0.6724185943603516,0.6668604612350464,-0.007043089717626572,0.9801015853881836 | |
| 34000,4.2582185415085405e-05,0.0017795809544622898,-24.648584365844727,0.6845198273658752,0.6747610569000244,-0.006796265486627817,0.981103777885437 | |
| 35000,3.713354089995846e-05,0.006844840012490749,-26.4482421875,0.6809781789779663,0.6747550964355469,-0.004605089779943228,0.9872903227806091 | |
| 36000,2.2931160856387578e-05,0.006998369470238686,-25.68916130065918,0.6782712340354919,0.6742398738861084,-0.005071967840194702,0.9856425523757935 | |
| 37000,3.0021585189388134e-05,0.009208982810378075,-26.197978973388672,0.6757074594497681,0.6728830337524414,-0.005948767997324467,0.9832687973976135 | |
| 38000,3.9117076084949076e-05,0.0005408144206739962,-25.098886489868164,0.6811916828155518,0.6690225005149841,-0.006390203721821308,0.9821051955223083 | |
| 39000,2.5233166525140405e-05,0.006823300383985043,-26.27112579345703,0.6786589026451111,0.6703064441680908,-0.00786391831934452,0.9775221347808838 | |
| 40000,5.6581979151815176e-05,0.010767117142677307,-26.468229293823242,0.6801621913909912,0.6691429615020752,-0.005272462964057922,0.9862924218177795 | |
| 41000,4.808146331924945e-05,0.002310088835656643,-26.41440200805664,0.6942647695541382,0.6873034238815308,-0.006539342924952507,0.9816314578056335 | |
| 42000,3.9539780118502676e-05,0.0024409256875514984,-26.128372192382812,0.681615948677063,0.6770057082176208,-0.00894300639629364,0.9747663140296936 | |
| 43000,3.1755302188685164e-05,0.0016174130141735077,-25.75682830810547,0.6805109977722168,0.6809408664703369,-0.00495165353640914,0.9862682819366455 | |
| 44000,3.792825009441003e-05,0.006302870810031891,-26.618202209472656,0.6784427762031555,0.6649922728538513,-0.005928444676101208,0.9835434556007385 | |
| 45000,4.099719080841169e-05,0.010189332067966461,-26.381240844726562,0.6749536395072937,0.6773879528045654,-0.005499912425875664,0.9847501516342163 | |
| 46000,3.8136044167913496e-05,0.00333679118193686,-26.452701568603516,0.6828993558883667,0.6704730987548828,-0.007539449259638786,0.978760838508606 | |
| 47000,3.2699768780730665e-05,0.007844277657568455,-25.758146286010742,0.6762375831604004,0.6753689050674438,-0.007920825853943825,0.9772622585296631 | |
| 48000,4.708790947915986e-05,0.0035792530979961157,-26.559282302856445,0.6810895800590515,0.6803090572357178,-0.004941944964230061,0.9866853952407837 | |
| 49000,4.1881379729602486e-05,0.001161919441074133,-26.247440338134766,0.6800355911254883,0.6777771711349487,-0.006865913979709148,0.9809058904647827 | |
| 50000,3.066545832552947e-05,0.0032787222880870104,-26.585365295410156,0.6752890348434448,0.6644101142883301,-0.005832739174365997,0.9836618900299072 | |
| 51000,7.048835868772585e-06,0.0020110318437218666,-25.934309005737305,0.6758707761764526,0.6685066223144531,-0.007339830510318279,0.9783182740211487 | |
| 52000,1.620029070181772e-05,0.0025953464210033417,-27.39383316040039,0.672835648059845,0.6679995656013489,-0.002929943148046732,0.9917386770248413 | |
| 53000,3.769915929296985e-05,0.0031190640293061733,-25.899755477905273,0.6645402908325195,0.6553307771682739,-0.008330282755196095,0.976494550704956 | |
| 54000,1.6480016711284406e-05,0.0017218291759490967,-26.750865936279297,0.6589409112930298,0.6568036079406738,-0.0031633100006729364,0.9908485412597656 | |
| 55000,3.8655321986880153e-05,0.002428494393825531,-27.065013885498047,0.6606956720352173,0.655286431312561,-0.0020556210074573755,0.9951444864273071 | |
| 56000,1.2303480616537854e-05,0.001647316850721836,-26.819110870361328,0.6642846465110779,0.6604283452033997,-0.004564017988741398,0.9867464303970337 | |
| 57000,4.355218698037788e-05,0.002616706071421504,-27.304141998291016,0.662507176399231,0.6596034169197083,-0.00417429581284523,0.988905131816864 | |
| 58000,2.9108910894137807e-05,0.004978956189006567,-26.76839828491211,0.6655115485191345,0.6602908372879028,-0.006595100276172161,0.9814106225967407 | |
| 59000,2.4943896278273314e-05,0.007811486721038818,-26.851871490478516,0.658052921295166,0.6612030267715454,-0.0027196162845939398,0.9923555254936218 | |
| 60000,2.6589445042191073e-05,0.0012291574385017157,-27.195335388183594,0.6573009490966797,0.6478146910667419,-0.0031281171832233667,0.9914597868919373 | |
| 61000,3.781884515774436e-05,0.0006807289319112897,-26.560279846191406,0.66215980052948,0.6576732397079468,-0.008606025949120522,0.9757674932479858 | |
| 62000,3.5344848583918065e-05,0.007952027022838593,-27.09160614013672,0.6586782932281494,0.6535213589668274,-0.0071608275175094604,0.9799836874008179 | |
| 63000,3.464055771473795e-05,0.010840530507266521,-27.003475189208984,0.6560536623001099,0.6466917395591736,-0.007446694653481245,0.9789988994598389 | |
| 64000,3.9236001612152904e-05,0.0016599304508417845,-26.125595092773438,0.6569247245788574,0.6540800333023071,-0.0026249606162309647,0.9934149384498596 | |
| 65000,4.034869198221713e-05,0.006343608722090721,-26.807863235473633,0.6563618183135986,0.6472693681716919,-0.004783772863447666,0.9870704412460327 | |
| 66000,3.6838835512753576e-05,0.006468208972364664,-27.423999786376953,0.654080867767334,0.6494276523590088,-0.00800769217312336,0.9775445461273193 | |
| 67000,3.9939121052157134e-05,0.0062063317745924,-26.424152374267578,0.6546080112457275,0.6505107879638672,-0.004112246446311474,0.9888710975646973 | |
| 68000,2.9374608857324347e-05,0.0028726779855787754,-26.849634170532227,0.6469296216964722,0.6483762860298157,-0.005041126161813736,0.9857789278030396 | |
| 69000,1.9566154151107185e-05,0.0066520352847874165,-27.306751251220703,0.6548199653625488,0.6441365480422974,-0.0043816156685352325,0.9875500202178955 | |
| 70000,2.1118759832461365e-05,0.0013170663733035326,-27.991741180419922,0.6450580954551697,0.6440883874893188,-0.0032825320959091187,0.9908612966537476 | |
| 71000,5.025139398640022e-05,0.00625806488096714,-27.112871170043945,0.651551365852356,0.6470623016357422,-0.0034519562032073736,0.9912365674972534 | |
| 72000,2.829503864631988e-05,0.0024456605315208435,-26.368144989013672,0.6521545052528381,0.6424663066864014,-0.009035173803567886,0.9739041924476624 | |
| 73000,3.529671812430024e-05,0.0008579347631894052,-26.497997283935547,0.6512706875801086,0.6456409692764282,-0.005865306593477726,0.9837328791618347 | |
| 74000,6.0824371757917106e-05,0.00417448952794075,-28.428619384765625,0.6531693935394287,0.6497136354446411,-0.004164134152233601,0.98954838514328 | |
| 75000,3.715656930580735e-05,0.005738687701523304,-27.210186004638672,0.6455581188201904,0.6303784847259521,-0.007865890860557556,0.977941632270813 | |
| 76000,2.3443153622793034e-05,0.0032165779266506433,-25.80697250366211,0.659044623374939,0.6488752961158752,-0.0071230484172701836,0.9795742034912109 | |
| 77000,2.5314100639661774e-05,0.004329728428274393,-27.869922637939453,0.6502360105514526,0.6442248821258545,-0.005447769537568092,0.9846412539482117 | |
| 78000,3.104194183833897e-05,0.004126978572458029,-27.172258377075195,0.6488205790519714,0.6446129679679871,-0.005098063498735428,0.9858541488647461 | |
| 79000,1.7062660845112987e-05,0.004034895915538073,-28.040241241455078,0.6511364579200745,0.653035581111908,-0.005350543651729822,0.9845170974731445 | |
| 80000,3.642105730250478e-05,0.0023903108667582273,-27.023893356323242,0.6425094604492188,0.6420444250106812,-0.0016369062941521406,0.9962606430053711 | |
| 81000,2.4611443222966045e-05,0.0010714665986597538,-27.000789642333984,0.6459441781044006,0.6401053071022034,-0.007348035462200642,0.9789760112762451 | |
| 82000,3.3153482945635915e-05,0.00222736201249063,-27.377277374267578,0.6421973705291748,0.6466044783592224,-0.002466588281095028,0.9931188821792603 | |
| 83000,3.869382635457441e-05,0.003922243136912584,-27.834348678588867,0.6508054733276367,0.6488324403762817,-0.005412166938185692,0.9851295351982117 | |
| 84000,5.030693864682689e-05,0.00566294277086854,-27.959030151367188,0.6465016603469849,0.639745831489563,-0.004487460013478994,0.9883763790130615 | |
| 85000,2.927093009930104e-05,0.0013682672288268805,-26.966632843017578,0.6520128846168518,0.6495124101638794,-0.005779010243713856,0.9836788177490234 | |
| 86000,2.9128757887519896e-05,0.009402984753251076,-27.250411987304688,0.6438755393028259,0.6361543536186218,-0.0069296276196837425,0.98047935962677 | |
| 87000,5.136815889272839e-05,0.0015798760578036308,-27.10993766784668,0.6526371836662292,0.6411746144294739,-0.006555532105267048,0.9820915460586548 | |
| 88000,3.5731514799408615e-05,0.0003340569091960788,-27.353830337524414,0.6426182389259338,0.63825523853302,-0.005608444567769766,0.9844815731048584 | |
| 89000,1.774285192368552e-05,0.0018508539069443941,-26.62392234802246,0.6369621157646179,0.6339382529258728,-0.006478981114923954,0.9813953638076782 | |
| 90000,2.232634142274037e-05,0.0024223015643656254,-27.503925323486328,0.6365375518798828,0.6300281882286072,-0.0070667569525539875,0.9797694683074951 | |
| 91000,8.070836338447407e-05,0.002379580168053508,-26.868328094482422,0.6362221240997314,0.6282573938369751,-0.006179952062666416,0.9840031862258911 | |
| 92000,2.952972863567993e-05,0.001950854784809053,-28.00855255126953,0.6439036130905151,0.6396648287773132,-0.005069437902420759,0.9858957529067993 | |
| 93000,2.4904114980017766e-05,0.003191376104950905,-27.411190032958984,0.63435959815979,0.6254229545593262,-0.006492051761597395,0.9814631938934326 | |
| 94000,5.690989928552881e-05,0.0019737754482775927,-27.296226501464844,0.6341214179992676,0.6334408521652222,-0.006165827624499798,0.983433723449707 | |
| 95000,3.979518078267574e-05,0.002752769272774458,-27.75511932373047,0.6336755156517029,0.6283588409423828,-0.003818800672888756,0.9898561239242554 | |
| 96000,4.960062506143004e-05,0.009014890529215336,-27.756690979003906,0.6328295469284058,0.6303460597991943,-0.007676229812204838,0.9784045815467834 | |
| 97000,5.540142592508346e-05,0.004548730328679085,-27.483566284179688,0.637954831123352,0.6368046998977661,-0.006071236915886402,0.9838674068450928 | |
| 98000,3.653082239907235e-05,0.001282983459532261,-27.655887603759766,0.6389430165290833,0.635525107383728,-0.0060473354533314705,0.9831870794296265 | |
| 99000,5.1891689508920535e-05,0.005756565369665623,-27.46810531616211,0.6350830793380737,0.6379168033599854,-0.0060617346316576,0.9831979274749756 | |
| 100000,4.6143824874889106e-05,0.005222284235060215,-27.365333557128906,0.6431466341018677,0.6322906017303467,-0.010959780775010586,0.9687907695770264 | |