step,v_loss,q_loss,policy_loss,v_mean,q_mean,advantage_mean,exp_advantage_mean
1000,0.0001488937414251268,0.017123185098171234,-17.457326889038086,0.04721087962388992,0.06451725959777832,-0.00599301652982831,0.9837942123413086
2000,0.0005286661325953901,0.01033392921090126,-18.9333438873291,0.09802320599555969,0.11099211871623993,-0.006110389716923237,0.9875072240829468
3000,0.0010939082130789757,0.006473910994827747,-19.31658172607422,0.1788613498210907,0.15808287262916565,-0.01295169722288847,0.9724383354187012
4000,0.0012577481102198362,0.0070578427985310555,-19.839832305908203,0.21844390034675598,0.21006625890731812,-0.01451408863067627,0.9698702692985535
5000,0.0006946508074179292,0.006777857430279255,-21.192907333374023,0.30034518241882324,0.2972208559513092,-0.013719955459237099,0.9666890501976013
6000,0.0008185493643395603,0.00454492773860693,-21.101402282714844,0.3440312147140503,0.3355218172073364,-0.014879625290632248,0.9639429450035095
7000,0.0011498236563056707,0.0043181260116398335,-22.2924747467041,0.3763347268104553,0.37749457359313965,-0.0027895004022866488,1.003147840499878
8000,0.0005304793594405055,0.006958195939660072,-21.830453872680664,0.3898789882659912,0.3875240087509155,-0.009434308856725693,0.9775826930999756
9000,0.0005791351431980729,0.009224031120538712,-22.123802185058594,0.4506993889808655,0.4588368535041809,-0.008604421280324459,0.979835033416748
10000,0.000552843848709017,0.007071847096085548,-23.160566329956055,0.4691406488418579,0.46988046169281006,-0.007200233638286591,0.9843333959579468
11000,0.0006002543959766626,0.005418932065367699,-21.51403045654297,0.5051866769790649,0.5040819644927979,-0.009896766394376755,0.9763743877410889
12000,0.00021477842528838664,0.006183978170156479,-23.75772476196289,0.5187397599220276,0.5109261274337769,-0.005618714727461338,0.9855198860168457
13000,0.00026278692530468106,0.004638044163584709,-23.86483383178711,0.5331025123596191,0.5372599363327026,-0.005588066764175892,0.9861727952957153
14000,0.00040789341437630355,0.004485384561121464,-23.18863868713379,0.542675256729126,0.5440471768379211,-0.005035771988332272,0.9896504282951355
15000,0.0004344815097283572,0.005826424807310104,-24.25287628173828,0.5674669742584229,0.555858850479126,-0.003824046580120921,0.9928783178329468
16000,0.00030007955501787364,0.008056105114519596,-23.966318130493164,0.5731939673423767,0.5757041573524475,-0.005145515780895948,0.9877653121948242
17000,0.0002703202480915934,0.005122414790093899,-24.526195526123047,0.5817747116088867,0.5819088816642761,-0.0036303987726569176,0.9921929240226746
18000,0.00039316178299486637,0.00689063873142004,-23.439794540405273,0.5910090804100037,0.5759234428405762,-0.010704418644309044,0.9726033210754395
19000,0.00018615866429172456,0.00721375085413456,-25.051410675048828,0.5937175154685974,0.5937061309814453,0.0011550185736268759,1.0055806636810303
20000,0.00025072688004001975,0.0015508097130805254,-25.218236923217773,0.5873339176177979,0.5861520767211914,0.00019771966617554426,1.0030486583709717
21000,0.00022000273747835308,0.0030086319893598557,-23.37228012084961,0.5993434190750122,0.6004638671875,-0.002670956775546074,0.9945750832557678
22000,0.00017362178186886013,0.00239961757324636,-24.678878784179688,0.6005529165267944,0.596611499786377,-0.0016725929453969002,0.9968942403793335
23000,0.00013084686361253262,0.006569467484951019,-24.938079833984375,0.6206008195877075,0.6142706871032715,-0.0033857692033052444,0.9911432266235352
24000,0.00018223526421934366,0.002369912341237068,-24.460426330566406,0.6045263409614563,0.5985832810401917,-0.005711064673960209,0.985088586807251
25000,0.00010868997196666896,0.004901471547782421,-25.510717391967773,0.6051712036132812,0.6084377765655518,-3.5031349398195744e-05,1.001131534576416
26000,0.00019300405983813107,0.00422974769026041,-26.030649185180664,0.6115922927856445,0.6020850539207458,-0.006489139050245285,0.9829984903335571
27000,0.00020204705651849508,0.006951279006898403,-25.439002990722656,0.6009725332260132,0.5989433526992798,0.0013640759279951453,1.0060769319534302
28000,0.00021440471755340695,0.0036399043165147305,-25.49695587158203,0.6086745858192444,0.6052184104919434,0.0013740079011768103,1.0065057277679443
29000,8.494644134771079e-05,0.004319016356021166,-25.657848358154297,0.6075325608253479,0.6071163415908813,-0.0016160481609404087,0.9961751103401184
30000,7.651586929569021e-05,0.004498209338635206,-25.5654354095459,0.6022244095802307,0.6025192737579346,-0.0028532766737043858,0.9923875331878662
31000,0.00014085255679674447,0.0028083024080842733,-25.80255889892578,0.600712776184082,0.6001958250999451,-0.0006916482234373689,0.9995920062065125
32000,0.00012024045281577855,0.006574017461389303,-24.54662322998047,0.5963567495346069,0.5882075428962708,-0.005151441786438227,0.9860668182373047
33000,0.00015378290845546871,0.011917476542294025,-25.39267921447754,0.5959603190422058,0.5814091563224792,-0.004795705899596214,0.9874920845031738
34000,0.00019843160407617688,0.0028308317996561527,-25.03889274597168,0.6032512187957764,0.6092047095298767,0.0007185174617916346,1.004421591758728
35000,0.00013722555013373494,0.006110752001404762,-25.946321487426758,0.5967638492584229,0.5925816297531128,-0.004547523334622383,0.9881558418273926
36000,0.00011616927804425359,0.006658279802650213,-25.817729949951172,0.5974419116973877,0.589691162109375,-0.008166152983903885,0.9768060445785522
37000,0.00015622015052940696,0.00927338469773531,-26.523395538330078,0.5916243195533752,0.5916246175765991,-0.000850978191010654,0.9992008209228516
38000,0.00016840218449942768,0.0005011010216549039,-25.401657104492188,0.5957861542701721,0.5887603759765625,-0.00331634609028697,0.9920378923416138
39000,0.00010988415306201205,0.007036756724119186,-26.540027618408203,0.587198793888092,0.5874839425086975,-0.003404047805815935,0.9908913969993591
40000,0.00019101548241451383,0.010661423206329346,-25.81806182861328,0.5920298099517822,0.5750237703323364,-0.005834830924868584,0.9848833084106445
41000,0.00013355040573514998,0.002639091107994318,-25.92420196533203,0.5998861789703369,0.592593789100647,-0.005590290296822786,0.984661340713501
42000,0.00013216228398960084,0.0030659311451017857,-26.353906631469727,0.5818030834197998,0.5826287865638733,-0.0015514675760641694,0.9969161748886108
43000,0.00012512158718891442,0.0013485478702932596,-25.845565795898438,0.5800286531448364,0.5821515321731567,-0.005123673472553492,0.9859508275985718
44000,0.00012281336239539087,0.009433717466890812,-26.624422073364258,0.5814752578735352,0.5757251977920532,-0.0027976958081126213,0.9930979609489441
45000,0.00010371934331487864,0.009669829159975052,-27.00676727294922,0.5739758014678955,0.5833988189697266,0.0028173085302114487,1.0097479820251465
46000,9.72836569417268e-05,0.005585405509918928,-26.44808006286621,0.5774369239807129,0.5724466443061829,-0.0031470314133912325,0.9917936325073242
47000,9.05219349078834e-05,0.008765907026827335,-26.231090545654297,0.5737698078155518,0.5855615735054016,-5.775375757366419e-05,1.000643253326416
48000,0.00014932529302313924,0.0039146654307842255,-25.868698120117188,0.5787293314933777,0.5797147750854492,-0.00457192724570632,0.9880678653717041
49000,0.00015518236614298075,0.0016166161512956023,-25.75746726989746,0.5819497108459473,0.5806148052215576,-0.00766854640096426,0.9790775775909424
50000,9.765935828909278e-05,0.005000343546271324,-26.063396453857422,0.5749824047088623,0.5687891244888306,-0.004873913247138262,0.9865258932113647
51000,6.426209438359365e-05,0.0021728933788836002,-25.74820327758789,0.5731598138809204,0.5830702781677246,-0.0007245639571920037,0.9981404542922974
52000,5.935573426540941e-05,0.003119529690593481,-26.813621520996094,0.5822153091430664,0.5772654414176941,-0.004683001898229122,0.9867243766784668
53000,0.00013016824959777296,0.003621558891609311,-26.15018081665039,0.5665186047554016,0.5619409084320068,-0.003845440223813057,0.9900767207145691
54000,5.861584941158071e-05,0.0019813221879303455,-26.52945327758789,0.5710108280181885,0.5725727081298828,-0.00014177733100950718,1.000206708908081
55000,0.00012967182556167245,0.0027217124588787556,-26.197147369384766,0.5730605125427246,0.5763099193572998,-0.00575302867218852,0.9843593835830688
56000,5.904484714847058e-05,0.002086964203044772,-26.71909523010254,0.5752797722816467,0.5784851312637329,-0.0036513577215373516,0.9895596504211426
57000,0.0001286962942685932,0.0025332202203571796,-27.35590171813965,0.5737621784210205,0.5751775503158569,0.0008270799880847335,1.004018783569336
58000,7.815567369107157e-05,0.006133121903985739,-26.280715942382812,0.5805566906929016,0.5763545632362366,-0.004147685598582029,0.9885240793228149
59000,7.395993452519178e-05,0.008265103213489056,-26.150333404541016,0.5733118057250977,0.5674525499343872,-0.0022185463458299637,0.9940261244773865
60000,8.750478446017951e-05,0.001516152173280716,-26.9849853515625,0.5730513334274292,0.5679193735122681,-0.004655994474887848,0.9871608018875122
61000,0.00011709539830917493,0.0008902038680389524,-25.748294830322266,0.5814056992530823,0.5742424726486206,-0.006016991101205349,0.9833992123603821
62000,0.0001369673991575837,0.007829621434211731,-26.747182846069336,0.5807332992553711,0.5711119174957275,-0.00680540269240737,0.9812262058258057
63000,0.00011835618352051824,0.01048852689564228,-26.834095001220703,0.5697199702262878,0.5688411593437195,-0.002711495151743293,0.9930967688560486
64000,0.00010618756641633809,0.0011887461878359318,-25.574569702148438,0.5720807313919067,0.5710684061050415,-0.0034998124465346336,0.99074387550354
65000,0.00010482417565071955,0.005848865956068039,-26.042343139648438,0.5716762542724609,0.5632674098014832,-0.004522524308413267,0.9877628087997437
66000,8.895703649614006e-05,0.006089819595217705,-27.129566192626953,0.560055136680603,0.5623401403427124,-0.0003869016654789448,0.9999047517776489
67000,0.00011330326378811151,0.006693074479699135,-23.250038146972656,0.5589816570281982,0.5644457340240479,-0.0018776217475533485,0.9955991506576538
68000,9.079788287635893e-05,0.0031706918962299824,-26.634340286254883,0.5578062534332275,0.5601974129676819,-0.002269890159368515,0.9943193197250366
69000,5.73778452235274e-05,0.006508987862616777,-26.516582489013672,0.5657087564468384,0.5613890886306763,-0.007151088677346706,0.9794530272483826
70000,8.407401764998212e-05,0.0019406850915402174,-27.36440658569336,0.5493730902671814,0.5476880669593811,-0.003952967002987862,0.9890477657318115
71000,0.00016143931134138256,0.006198926828801632,-26.58662223815918,0.5561314821243286,0.5490660667419434,-0.007141285575926304,0.9803811311721802
72000,8.3572042058222e-05,0.0032507595606148243,-26.001232147216797,0.555461049079895,0.549469530582428,-0.005260828882455826,0.9850512146949768
73000,0.00010661175474524498,0.0007440145127475262,-26.81865882873535,0.5565646886825562,0.5539497137069702,-0.0019408157095313072,0.9954901933670044
74000,0.00024070873041637242,0.0052931467071175575,-27.776180267333984,0.5686681866645813,0.5621130466461182,-0.007694175001233816,0.9798183441162109
75000,0.00015474247629754245,0.005048967897891998,-27.402301788330078,0.5500936508178711,0.5413309335708618,-0.002668903209269047,0.9936968088150024
76000,7.424000796163455e-05,0.0038600116968154907,-25.61629295349121,0.5703386068344116,0.5637786388397217,-0.002474918495863676,0.9934521913528442
77000,8.965870074462146e-05,0.006210260558873415,-27.60029411315918,0.5523503422737122,0.5520658493041992,-0.004888276569545269,0.9864134788513184
78000,0.00010770530207082629,0.004852974787354469,-27.586360931396484,0.5570819973945618,0.5587167143821716,-0.0017596060642972589,0.9959943294525146
79000,0.00010430492693558335,0.0048827421851456165,-27.83783721923828,0.5605653524398804,0.5653701424598694,-0.0019765314646065235,0.9948995113372803
80000,0.00010558339999988675,0.00285049295052886,-26.190105438232422,0.5531935691833496,0.5483263731002808,-0.002991686575114727,0.9923228025436401
81000,8.737208554521203e-05,0.0008890702156350017,-26.73209571838379,0.5547089576721191,0.5558770895004272,-0.0003758469829335809,0.9996782541275024
82000,8.134762174449861e-05,0.004000999964773655,-26.78078842163086,0.5528160929679871,0.5524387955665588,-0.0018687330884858966,0.9953027963638306
83000,0.00012431899085640907,0.005905299913138151,-27.33001136779785,0.5564279556274414,0.556323766708374,-0.00594191811978817,0.9836968183517456
84000,0.00020311625848989934,0.005271616391837597,-27.606876373291016,0.5467767715454102,0.545536994934082,-0.0037131914868950844,0.9911004304885864
85000,6.36972690699622e-05,0.001846288563683629,-26.49146270751953,0.554013729095459,0.5509346127510071,-0.003998124040663242,0.9887819886207581
86000,7.593004556838423e-05,0.008715381845831871,-27.499427795410156,0.5444061756134033,0.5431278944015503,-0.002713325899094343,0.9928195476531982
87000,0.00011334389273542911,0.0015394220827147365,-26.958168029785156,0.5473346710205078,0.5436162948608398,-0.0051136803813278675,0.9861065149307251
88000,8.364733366761357e-05,0.0004284453170839697,-26.754413604736328,0.5363893508911133,0.533919632434845,-0.003050466999411583,0.9918617010116577
89000,8.446662104688585e-05,0.002174799796193838,-26.83147430419922,0.5341291427612305,0.5379350185394287,-0.003239447483792901,0.9912080764770508
90000,7.296711555682123e-05,0.0037364636082202196,-27.3741397857666,0.5361378192901611,0.5344648957252502,-0.001979462569579482,0.9947989583015442
91000,0.0001922717347042635,0.002574441023170948,-26.53437614440918,0.5284481048583984,0.5287431478500366,-0.004611331969499588,0.9885634779930115
92000,8.725952648092061e-05,0.0029895692132413387,-27.269290924072266,0.5517081618309021,0.5533859729766846,-0.00309176673181355,0.9917852878570557
93000,7.919182098703459e-05,0.0047914437018334866,-27.147789001464844,0.5381056070327759,0.5327804088592529,-0.004360011778771877,0.9878666996955872
94000,0.00018320753588341177,0.002686972264200449,-26.622901916503906,0.5366100072860718,0.5350701212882996,-0.004095324315130711,0.9899418354034424
95000,0.00014682550681754947,0.0029815668240189552,-27.30376625061035,0.5415458679199219,0.535621702671051,-0.0025409297086298466,0.9939871430397034
96000,0.00012204082304378971,0.010050460696220398,-28.051822662353516,0.5348919034004211,0.537726879119873,-0.0016675336519256234,0.9962784051895142
97000,0.00019438120943959802,0.0029506103601306677,-26.26782989501953,0.5408000946044922,0.5382518172264099,-0.004446289502084255,0.9889114499092102
98000,0.00012340216198936105,0.0007030363194644451,-27.668563842773438,0.5397628545761108,0.5373809337615967,-0.0035567376762628555,0.9906792640686035
99000,0.00011772227298934013,0.006932809017598629,-27.143587112426758,0.5412322282791138,0.5380535125732422,-0.000751726794987917,0.9989335536956787
100000,0.000119170275866054,0.004066753666847944,-27.553874969482422,0.5407698154449463,0.5363407135009766,-0.0040032402612268925,0.9894227981567383