{"step": 100, "update": 50, "epoch": 0, "loss": 1.0466632970422507, "answer_loss": 0.9093208994437009, "evidence_loss": 0.2746847915649414, "lr": 1.2561576354679805e-05, "elapsed": 154.3350830078125}
{"step": 200, "update": 100, "epoch": 0, "loss": 0.7416581735294312, "answer_loss": 0.6910765176918358, "evidence_loss": 0.10116331100463867, "lr": 2.4876847290640397e-05, "elapsed": 257.4809904098511}
{"step": 300, "update": 150, "epoch": 0, "loss": 0.6601098832546267, "answer_loss": 0.6098437773936893, "evidence_loss": 0.10053220888599754, "lr": 3.719211822660099e-05, "elapsed": 371.6660888195038}
{"step": 400, "update": 200, "epoch": 0, "loss": 0.7968255157326348, "answer_loss": 0.7370119733747561, "evidence_loss": 0.11962707860395312, "lr": 4.950738916256158e-05, "elapsed": 487.11128091812134}
{"step": 500, "update": 250, "epoch": 0, "loss": 0.5190051701292395, "answer_loss": 0.5003233546391129, "evidence_loss": 0.03736363284289837, "lr": 4.999372686422346e-05, "elapsed": 602.2875165939331}
{"step": 600, "update": 300, "epoch": 0, "loss": 0.7238669737521559, "answer_loss": 0.7120701452810317, "evidence_loss": 0.02359365880023688, "lr": 4.997328388910923e-05, "elapsed": 721.729451417923}
{"step": 700, "update": 350, "epoch": 0, "loss": 0.5110658522252924, "answer_loss": 0.501890649611596, "evidence_loss": 0.018350402312353253, "lr": 4.993865712608182e-05, "elapsed": 826.3627226352692}
{"step": 800, "update": 400, "epoch": 0, "loss": 0.707598408163758, "answer_loss": 0.6938729958058684, "evidence_loss": 0.027450826987624168, "lr": 4.9889866241704306e-05, "elapsed": 933.359840631485}
{"step": 900, "update": 450, "epoch": 0, "loss": 0.5563830675184727, "answer_loss": 0.5391353450086899, "evidence_loss": 0.03449544358138155, "lr": 4.982693894717237e-05, "elapsed": 1040.5720059871674}
{"step": 1000, "update": 500, "epoch": 0, "loss": 0.5940256051276811, "answer_loss": 0.5801869906159118, "evidence_loss": 0.02767722815130867, "lr": 4.97499109825754e-05, "elapsed": 1145.7536494731903}
{"step": 1100, "update": 550, "epoch": 0, "loss": 0.5344747617281973, "answer_loss": 0.513588702082634, "evidence_loss": 0.04177212506803585, "lr": 4.965882609659774e-05, "elapsed": 1275.479078054428}
{"step": 1200, "update": 600, "epoch": 0, "loss": 0.5738527677254751, "answer_loss": 0.572868535448797, "evidence_loss": 0.0019684605054499115, "lr": 4.955373602167117e-05, "elapsed": 1361.5137095451355}
{"step": 1300, "update": 650, "epoch": 0, "loss": 0.6228927255561575, "answer_loss": 0.6063712534727529, "evidence_loss": 0.033042942728825436, "lr": 4.943470044459301e-05, "elapsed": 1467.5046110153198}
{"step": 1400, "update": 700, "epoch": 0, "loss": 0.5342604766273871, "answer_loss": 0.5312385475146584, "evidence_loss": 0.0060438554310348995, "lr": 4.930178697262657e-05, "elapsed": 1567.4875328540802}
{"step": 1500, "update": 750, "epoch": 0, "loss": 0.48719982599723155, "answer_loss": 0.47769132474437354, "evidence_loss": 0.019017004827378513, "lr": 4.9155071095103e-05, "elapsed": 1676.7254557609558}
{"step": 1600, "update": 800, "epoch": 0, "loss": 0.5404521024005953, "answer_loss": 0.528679445082089, "evidence_loss": 0.02354531695116606, "lr": 4.899463614054647e-05, "elapsed": 1792.2951774597168}
{"step": 1700, "update": 850, "epoch": 0, "loss": 0.5006524209631606, "answer_loss": 0.5006182217225432, "evidence_loss": 6.839833200501744e-05, "lr": 4.882057322934704e-05, "elapsed": 1886.2667684555054}
{"step": 1800, "update": 900, "epoch": 0, "loss": 0.5758555604936555, "answer_loss": 0.539689148021862, "evidence_loss": 0.0723328263185147, "lr": 4.863298122200809e-05, "elapsed": 2002.5272870063782}
{"step": 1900, "update": 950, "epoch": 0, "loss": 0.49814286177512257, "answer_loss": 0.4793539487721864, "evidence_loss": 0.03757782667705669, "lr": 4.843196666299762e-05, "elapsed": 2123.8212616443634}
{"step": 2000, "update": 1000, "epoch": 0, "loss": 0.5743265111767687, "answer_loss": 0.5593323490326293, "evidence_loss": 0.02998832721196475, "lr": 4.821764372023544e-05, "elapsed": 2229.363822221756}
{"step": 2100, "update": 1050, "epoch": 0, "loss": 0.4775368913402781, "answer_loss": 0.4539963689164142, "evidence_loss": 0.04708105071846186, "lr": 4.799013412025052e-05, "elapsed": 2349.06672334671}
{"step": 2200, "update": 1100, "epoch": 0, "loss": 0.5363413573545404, "answer_loss": 0.5363412640371825, "evidence_loss": 1.952083284395556e-07, "lr": 4.7749567079045424e-05, "elapsed": 2445.444249868393}
{"step": 2300, "update": 1150, "epoch": 0, "loss": 0.4746944914071355, "answer_loss": 0.4746943751501385, "evidence_loss": 2.3482379049477586e-07, "lr": 4.74960792287069e-05, "elapsed": 2552.2553861141205}
{"step": 2400, "update": 1200, "epoch": 0, "loss": 0.5232260901574045, "answer_loss": 0.5228329078108073, "evidence_loss": 0.0007863650698963553, "lr": 4.722981453980462e-05, "elapsed": 2657.0607981681824}
{"step": 2500, "update": 1250, "epoch": 0, "loss": 0.5513794144941494, "answer_loss": 0.533010982836131, "evidence_loss": 0.0367368643773321, "lr": 4.695092423962183e-05, "elapsed": 2767.573856115341}
{"step": 2600, "update": 1300, "epoch": 0, "loss": 0.5735468521388247, "answer_loss": 0.5634960023453459, "evidence_loss": 0.020101699072139424, "lr": 4.6659566726264524e-05, "elapsed": 2877.4249551296234}
{"step": 2700, "update": 1350, "epoch": 0, "loss": 0.42455267066659874, "answer_loss": 0.40903420170943716, "evidence_loss": 0.031036934792302347, "lr": 4.635590747869793e-05, "elapsed": 2994.5280821323395}
{"step": 2800, "update": 1400, "epoch": 0, "loss": 0.48469938746362456, "answer_loss": 0.48457985530549197, "evidence_loss": 0.00023906831981600261, "lr": 4.604011896276135e-05, "elapsed": 3103.987872838974}
{"step": 2900, "update": 1450, "epoch": 0, "loss": 0.5344389405541006, "answer_loss": 0.530350395317364, "evidence_loss": 0.008177093630389045, "lr": 4.571238053321464e-05, "elapsed": 3224.394252538681}
{"step": 3000, "update": 1500, "epoch": 0, "loss": 0.5819819614558946, "answer_loss": 0.5376607599260751, "evidence_loss": 0.08864240093117996, "lr": 4.5372878331872246e-05, "elapsed": 3350.2840735912323}
{"step": 3100, "update": 1550, "epoch": 0, "loss": 0.4642395470815245, "answer_loss": 0.46400510797160677, "evidence_loss": 0.00046887970986347227, "lr": 4.502180518188236e-05, "elapsed": 3472.488645553589}
{"step": 3200, "update": 1600, "epoch": 0, "loss": 0.5830743358284235, "answer_loss": 0.5706681804778054, "evidence_loss": 0.02481231747850984, "lr": 4.4659360478211355e-05, "elapsed": 3590.0721051692963}
{"step": 3300, "update": 1650, "epoch": 0, "loss": 0.5431933886697515, "answer_loss": 0.5277707322640345, "evidence_loss": 0.03084531450773291, "lr": 4.428575007439575e-05, "elapsed": 3691.8125829696655}
{"step": 3400, "update": 1700, "epoch": 0, "loss": 0.5290522611211054, "answer_loss": 0.5236124264192767, "evidence_loss": 0.010879670253632314, "lr": 4.390118616562595e-05, "elapsed": 3807.3116776943207}
{"step": 3500, "update": 1750, "epoch": 0, "loss": 0.48495657499530354, "answer_loss": 0.4821242574881762, "evidence_loss": 0.005664636454030691, "lr": 4.35058871682282e-05, "elapsed": 3912.8884415626526}
{"step": 3600, "update": 1800, "epoch": 0, "loss": 0.4668302389397286, "answer_loss": 0.46674671555054376, "evidence_loss": 0.00016704751175979028, "lr": 4.3100077595613146e-05, "elapsed": 4032.690903902054}
{"step": 3700, "update": 1850, "epoch": 0, "loss": 0.5461908026458695, "answer_loss": 0.5188855572417378, "evidence_loss": 0.054610492039893, "lr": 4.2683987930761596e-05, "elapsed": 4135.709332227707}
{"step": 3800, "update": 1900, "epoch": 0, "loss": 0.46654491690685973, "answer_loss": 0.4664767611282878, "evidence_loss": 0.0001363158306709411, "lr": 4.225785449531973e-05, "elapsed": 4234.659541130066}
{"step": 3900, "update": 1950, "epoch": 0, "loss": 0.42730378168082095, "answer_loss": 0.4273031175887445, "evidence_loss": 1.3314065834246768e-06, "lr": 4.182191931537822e-05, "elapsed": 4346.67816734314}
{"step": 4000, "update": 2000, "epoch": 0, "loss": 0.5631801752815955, "answer_loss": 0.5631604929215973, "evidence_loss": 3.936643363275705e-05, "lr": 4.137642998401147e-05, "elapsed": 4453.173367977142}
{"step": 4100, "update": 2050, "epoch": 0, "loss": 0.47845682796556505, "answer_loss": 0.45774981057271363, "evidence_loss": 0.041414034293519364, "lr": 4.0921639520655043e-05, "elapsed": 4547.15971159935}
{"step": 4200, "update": 2100, "epoch": 0, "loss": 0.5083158849657048, "answer_loss": 0.47591323538741565, "evidence_loss": 0.06480530041098546, "lr": 4.045780622740113e-05, "elapsed": 4666.14689540863}
{"step": 4300, "update": 2150, "epoch": 0, "loss": 0.492032068799017, "answer_loss": 0.46687227254151364, "evidence_loss": 0.050319592165840844, "lr": 3.998519354229365e-05, "elapsed": 4782.390250205994}
{"step": 4100, "update": 2050, "epoch": 0, "loss": 0.4950320903561078, "answer_loss": 0.4731320877978578, "evidence_loss": 0.04380001190301136, "lr": 4.0921639520655043e-05, "elapsed": 93.81770753860474}
{"step": 4200, "update": 2100, "epoch": 0, "loss": 0.5207801886694505, "answer_loss": 0.48821464135311543, "evidence_loss": 0.06513109242844621, "lr": 4.045780622740113e-05, "elapsed": 212.7313916683197}
{"step": 4300, "update": 2150, "epoch": 0, "loss": 0.491773782215314, "answer_loss": 0.4684547888091765, "evidence_loss": 0.04663798563404666, "lr": 3.998519354229365e-05, "elapsed": 328.43280816078186}
{"step": 4400, "update": 2200, "epoch": 0, "loss": 0.4974437666492304, "answer_loss": 0.46550733276759276, "evidence_loss": 0.06387287114079772, "lr": 3.9504069889706476e-05, "elapsed": 430.45074915885925}
{"step": 4500, "update": 2250, "epoch": 0, "loss": 0.5081180370456423, "answer_loss": 0.5022541567738518, "evidence_loss": 0.011727759505515536, "lr": 3.901470852788942e-05, "elapsed": 530.8654174804688}
{"step": 4600, "update": 2300, "epoch": 0, "loss": 0.4814008385059424, "answer_loss": 0.4805939413566375, "evidence_loss": 0.0016137976962623667, "lr": 3.8517387393769e-05, "elapsed": 641.8180623054504}
{"step": 4700, "update": 2350, "epoch": 0, "loss": 0.5245338723540772, "answer_loss": 0.5204225685761776, "evidence_loss": 0.008222610500095881, "lr": 3.801238894509176e-05, "elapsed": 753.5041408538818}
{"step": 4800, "update": 2400, "epoch": 0, "loss": 0.3832768932898762, "answer_loss": 0.37741137452132534, "evidence_loss": 0.01173103900319043, "lr": 3.7500000000000003e-05, "elapsed": 861.6717767715454}
{"step": 4900, "update": 2450, "epoch": 0, "loss": 0.5356292374519398, "answer_loss": 0.5253163638827391, "evidence_loss": 0.0206257526912173, "lr": 3.6980511574131034e-05, "elapsed": 979.6903760433197}
{"step": 5000, "update": 2500, "epoch": 0, "loss": 0.520200138433138, "answer_loss": 0.5201998082955833, "evidence_loss": 6.633352515539314e-07, "lr": 3.6454218715332324e-05, "elapsed": 1098.0372767448425}
{"step": 5100, "update": 2550, "epoch": 0, "loss": 0.45275376982346643, "answer_loss": 0.452752989862347, "evidence_loss": 1.5633659917235755e-06, "lr": 3.5921420336086606e-05, "elapsed": 1204.1972362995148}
{"step": 5200, "update": 2600, "epoch": 0, "loss": 0.5308002051792573, "answer_loss": 0.5307994494203012, "evidence_loss": 1.517018775767376e-06, "lr": 3.538241904374201e-05, "elapsed": 1316.0419623851776}
{"step": 5300, "update": 2650, "epoch": 0, "loss": 0.4377178295957856, "answer_loss": 0.4377175888605416, "evidence_loss": 4.800104222102419e-07, "lr": 3.4837520968643644e-05, "elapsed": 1429.3595159053802}
{"step": 5400, "update": 2700, "epoch": 0, "loss": 0.439339493070147, "answer_loss": 0.439101385162212, "evidence_loss": 0.0004762143806309371, "lr": 3.42870355902643e-05, "elapsed": 1560.911366224289}
{"step": 5500, "update": 2750, "epoch": 0, "loss": 0.46321893111336976, "answer_loss": 0.44960031494498254, "evidence_loss": 0.027237233082020085, "lr": 3.373127556143291e-05, "elapsed": 1670.3537316322327}
{"step": 5600, "update": 2800, "epoch": 0, "loss": 0.5332666349224746, "answer_loss": 0.5150763697223738, "evidence_loss": 0.03638052615694885, "lr": 3.317055653076081e-05, "elapsed": 1776.308336019516}
{"step": 5700, "update": 2850, "epoch": 0, "loss": 0.5509440461546182, "answer_loss": 0.5439056775625795, "evidence_loss": 0.014076729759511686, "lr": 3.260519696336636e-05, "elapsed": 1882.9945969581604}
{"step": 5800, "update": 2900, "epoch": 0, "loss": 0.3639013960887678, "answer_loss": 0.36380102701368744, "evidence_loss": 0.0002007373871879281, "lr": 3.2035517960000046e-05, "elapsed": 1991.515107870102}
{"step": 5900, "update": 2950, "epoch": 0, "loss": 0.5100994391855784, "answer_loss": 0.4940486961381976, "evidence_loss": 0.032101496025152304, "lr": 3.146184307467255e-05, "elapsed": 2096.060045480728}
{"step": 6000, "update": 3000, "epoch": 0, "loss": 0.48002396800206043, "answer_loss": 0.48002270987955853, "evidence_loss": 2.5178889558929996e-06, "lr": 3.088449813088953e-05, "elapsed": 2205.22669005394}
{"step": 6100, "update": 3050, "epoch": 0, "loss": 0.37645776741206644, "answer_loss": 0.3764565281197429, "evidence_loss": 2.4761888549562626e-06, "lr": 3.0303811036597396e-05, "elapsed": 2311.468476295471}
{"step": 6200, "update": 3100, "epoch": 0, "loss": 0.45311510397470556, "answer_loss": 0.4516035089973593, "evidence_loss": 0.0030231872583493667, "lr": 2.9720111597945237e-05, "elapsed": 2417.656094312668}
{"step": 6300, "update": 3150, "epoch": 0, "loss": 0.4857915040678927, "answer_loss": 0.48259961118747013, "evidence_loss": 0.006383783599609565, "lr": 2.9133731331968607e-05, "elapsed": 2534.446584224701}
{"step": 6400, "update": 3200, "epoch": 0, "loss": 0.44749429628136567, "answer_loss": 0.43921083775814623, "evidence_loss": 0.01656691852699405, "lr": 2.854500327830162e-05, "elapsed": 2637.2639179229736}
{"step": 6500, "update": 3250, "epoch": 0, "loss": 0.4559358202840667, "answer_loss": 0.4494398599502165, "evidence_loss": 0.0129919221403469, "lr": 2.795426181002427e-05, "elapsed": 2757.463133096695}
{"step": 6600, "update": 3300, "epoch": 0, "loss": 0.5957308709388599, "answer_loss": 0.5911648769211024, "evidence_loss": 0.00913198922353331, "lr": 2.7361842443752427e-05, "elapsed": 2893.4360072612762}
{"step": 6700, "update": 3350, "epoch": 0, "loss": 0.4670684744877508, "answer_loss": 0.4642935997445602, "evidence_loss": 0.005549743893116101, "lr": 2.676808164907833e-05, "elapsed": 3005.8959035873413}
{"step": 6800, "update": 3400, "epoch": 0, "loss": 0.4658524791576201, "answer_loss": 0.46540585774695503, "evidence_loss": 0.0008932487317431992, "lr": 2.6173316657469816e-05, "elapsed": 3110.553952217102}
{"step": 6900, "update": 3450, "epoch": 0, "loss": 0.47041740787768505, "answer_loss": 0.47037568698084214, "evidence_loss": 8.344542271885303e-05, "lr": 2.557788527073688e-05, "elapsed": 3246.5727021694183}
{"step": 7000, "update": 3500, "epoch": 0, "loss": 0.4430269662756473, "answer_loss": 0.4333769900747575, "evidence_loss": 0.019299950632438865, "lr": 2.4982125669174254e-05, "elapsed": 3357.106659889221}
{"step": 7100, "update": 3550, "epoch": 0, "loss": 0.53851259538671, "answer_loss": 0.5313033811445348, "evidence_loss": 0.014418437511594391, "lr": 2.4386376219489085e-05, "elapsed": 3474.9814581871033}
{"step": 7200, "update": 3600, "epoch": 0, "loss": 0.47839031941490245, "answer_loss": 0.47835108564584516, "evidence_loss": 7.846973450837514e-05, "lr": 2.3790975282622648e-05, "elapsed": 3595.1623787879944}
{"step": 7300, "update": 3650, "epoch": 0, "loss": 0.43931732105556875, "answer_loss": 0.4079691893584095, "evidence_loss": 0.06269626595791976, "lr": 2.3196261021575427e-05, "elapsed": 3720.810885667801}
{"step": 7400, "update": 3700, "epoch": 0, "loss": 0.4365746343682986, "answer_loss": 0.4256263518787455, "evidence_loss": 0.0218965731957978, "lr": 2.2602571209344562e-05, "elapsed": 3834.521643638611}
{"step": 7500, "update": 3750, "epoch": 0, "loss": 0.4500409300159663, "answer_loss": 0.422346715615131, "evidence_loss": 0.055388428679780194, "lr": 2.2010243037082813e-05, "elapsed": 3940.470145702362}
{"step": 7600, "update": 3800, "epoch": 0, "loss": 0.4599592524394393, "answer_loss": 0.45990150287747383, "evidence_loss": 0.0001155024600024035, "lr": 2.1419612922588012e-05, "elapsed": 4059.6920573711395}
{"step": 7700, "update": 3850, "epoch": 0, "loss": 0.48828520355047655, "answer_loss": 0.48828473751433193, "evidence_loss": 9.283837837870123e-07, "lr": 2.0831016319231703e-05, "elapsed": 4151.278962612152}
{"step": 7800, "update": 3900, "epoch": 0, "loss": 0.45586519574339035, "answer_loss": 0.4558648662210908, "evidence_loss": 6.68125817355758e-07, "lr": 2.024478752543556e-05, "elapsed": 4266.632747888565}
{"step": 7900, "update": 3950, "epoch": 0, "loss": 0.48138918802374975, "answer_loss": 0.4782767493254505, "evidence_loss": 0.006224881735236352, "lr": 1.9661259494803795e-05, "elapsed": 4369.711484909058}
{"step": 8000, "update": 4000, "epoch": 0, "loss": 0.5006899367499864, "answer_loss": 0.5003588348929771, "evidence_loss": 0.0006622073367475423, "lr": 1.9080763647019328e-05, "elapsed": 4471.893168449402}
{"step": 8100, "update": 4050, "epoch": 0, "loss": 0.4319746668660082, "answer_loss": 0.4280566773447208, "evidence_loss": 0.007835977174427172, "lr": 1.850362967961117e-05, "elapsed": 4577.780429840088}
{"step": 8200, "update": 4100, "epoch": 0, "loss": 0.4673368737427518, "answer_loss": 0.46149286096915604, "evidence_loss": 0.011688025794315991, "lr": 1.7930185380699906e-05, "elapsed": 4687.738959312439}
{"step": 8300, "update": 4150, "epoch": 0, "loss": 0.4473612277809298, "answer_loss": 0.4473604839979089, "evidence_loss": 1.4891167899122592e-06, "lr": 1.73607564428277e-05, "elapsed": 4798.392721891403}
{"step": 8400, "update": 4200, "epoch": 0, "loss": 0.46840745707973835, "answer_loss": 0.46764759002951906, "evidence_loss": 0.001519733650734949, "lr": 1.6795666277978432e-05, "elapsed": 4900.386907577515}
{"step": 8500, "update": 4250, "epoch": 0, "loss": 0.45989898687432285, "answer_loss": 0.4355156029490172, "evidence_loss": 0.04876676511669871, "lr": 1.6235235833893107e-05, "elapsed": 5008.159985780716}
{"step": 8600, "update": 4300, "epoch": 0, "loss": 0.4570891419821419, "answer_loss": 0.4568088326486759, "evidence_loss": 0.0005606203279592137, "lr": 1.567978341178492e-05, "elapsed": 5109.400854349136}
{"step": 8700, "update": 4350, "epoch": 0, "loss": 0.48315355408005417, "answer_loss": 0.4775139547570143, "evidence_loss": 0.01127920375422434, "lr": 1.5129624485557331e-05, "elapsed": 5215.7930335998535}
{"step": 8800, "update": 4400, "epoch": 0, "loss": 0.4190238815994235, "answer_loss": 0.4189879939588718, "evidence_loss": 7.177452474024904e-05, "lr": 1.4585071522628044e-05, "elapsed": 5321.05663394928}
{"step": 8900, "update": 4450, "epoch": 0, "loss": 0.4748892635566881, "answer_loss": 0.4730017786214012, "evidence_loss": 0.0037749671035356867, "lr": 1.4046433806460445e-05, "elapsed": 5432.626954078674}
{"step": 9000, "update": 4500, "epoch": 0, "loss": 0.4353832123952452, "answer_loss": 0.4353829197387677, "evidence_loss": 5.880843872319019e-07, "lr": 1.3514017260903495e-05, "elapsed": 5535.756450653076}
{"step": 9100, "update": 4550, "epoch": 0, "loss": 0.39594513215357435, "answer_loss": 0.3925364017987158, "evidence_loss": 0.006817455651212754, "lr": 1.2988124276439632e-05, "elapsed": 5648.987658023834}
{"step": 9200, "update": 4600, "epoch": 0, "loss": 0.38374086744734087, "answer_loss": 0.3753158838243689, "evidence_loss": 0.016849961100750762, "lr": 1.2469053538439599e-05, "elapsed": 5755.371156454086}
{"step": 9300, "update": 4650, "epoch": 0, "loss": 0.47940689367242156, "answer_loss": 0.4711623780394439, "evidence_loss": 0.016489021650122594, "lr": 1.1957099857521489e-05, "elapsed": 5870.985414505005}
{"step": 9400, "update": 4700, "epoch": 0, "loss": 0.4299167326278985, "answer_loss": 0.42968212099513037, "evidence_loss": 0.0004692193736286754, "lr": 1.1452554002110647e-05, "elapsed": 5981.759161233902}
{"step": 9500, "update": 4750, "epoch": 0, "loss": 0.4164541179564549, "answer_loss": 0.4056672558869468, "evidence_loss": 0.021573721798752103, "lr": 1.0955702533295189e-05, "elapsed": 6094.58864569664}
{"step": 9600, "update": 4800, "epoch": 0, "loss": 0.45831689791113606, "answer_loss": 0.4583165735419607, "evidence_loss": 6.530154345796291e-07, "lr": 1.0466827642071293e-05, "elapsed": 6202.668437004089}
{"step": 9700, "update": 4850, "epoch": 0, "loss": 0.4710958126705373, "answer_loss": 0.4710936766676605, "evidence_loss": 4.27520848830909e-06, "lr": 9.986206989070355e-06, "elapsed": 6322.259366512299}
{"step": 9800, "update": 4900, "epoch": 0, "loss": 0.40285806550004055, "answer_loss": 0.3935272289440036, "evidence_loss": 0.018661671176353138, "lr": 9.514113546859365e-06, "elapsed": 6437.950893163681}
{"step": 9900, "update": 4950, "epoch": 0, "loss": 0.4606573904794641, "answer_loss": 0.4605170532001648, "evidence_loss": 0.0002806783346022712, "lr": 9.050815444903748e-06, "elapsed": 6561.037968873978}
{"step": 10000, "update": 5000, "epoch": 0, "loss": 0.48335507344920187, "answer_loss": 0.48319887116551397, "evidence_loss": 0.000312406980083324, "lr": 8.596575817281035e-06, "elapsed": 6677.962034702301}
{"step": 10100, "update": 5050, "epoch": 0, "loss": 0.4223300266102888, "answer_loss": 0.4126726179290563, "evidence_loss": 0.0193148145511077, "lr": 8.151652653231584e-06, "elapsed": 6796.059580564499}
{"step": 10200, "update": 5100, "epoch": 0, "loss": 0.43294058976927774, "answer_loss": 0.43288003307650796, "evidence_loss": 0.0001211141562089324, "lr": 7.716298650631428e-06, "elapsed": 6908.10114979744}
{"step": 10300, "update": 5150, "epoch": 0, "loss": 0.48472219418210444, "answer_loss": 0.48419279763125816, "evidence_loss": 0.001058796302504561, "lr": 7.2907610724704006e-06, "elapsed": 7023.115696668625}
{"step": 10400, "update": 5200, "epoch": 0, "loss": 0.46715727597591467, "answer_loss": 0.4556182489270577, "evidence_loss": 0.023078054624129436, "lr": 6.87528160641698e-06, "elapsed": 7139.109053373337}
{"step": 10500, "update": 5250, "epoch": 0, "loss": 0.4871681500226259, "answer_loss": 0.4812744154036045, "evidence_loss": 0.011787464185908902, "lr": 6.4700962275497854e-06, "elapsed": 7250.013733863831}
{"step": 10600, "update": 5300, "epoch": 0, "loss": 0.43944938538712447, "answer_loss": 0.43879598681931387, "evidence_loss": 0.001306796344651957, "lr": 6.075435064333479e-06, "elapsed": 7359.430778264999}
{"step": 10700, "update": 5350, "epoch": 0, "loss": 0.37098298177617833, "answer_loss": 0.37096054804947926, "evidence_loss": 4.486462179386308e-05, "lr": 5.6915222679154165e-06, "elapsed": 7463.829654216766}
{"step": 10800, "update": 5400, "epoch": 0, "loss": 0.35547495214559605, "answer_loss": 0.35547234656638466, "evidence_loss": 5.2090198801124645e-06, "lr": 5.3185758848170535e-06, "elapsed": 7575.687963962555}
{"step": 10900, "update": 5450, "epoch": 0, "loss": 0.4352072715136455, "answer_loss": 0.43379306112299676, "evidence_loss": 0.0028284223711148113, "lr": 4.956807733092617e-06, "elapsed": 7683.106428861618}
{"step": 11000, "update": 5500, "epoch": 0, "loss": 0.39414913783228256, "answer_loss": 0.3941442339553032, "evidence_loss": 9.807227767169025e-06, "lr": 4.606423282025213e-06, "elapsed": 7782.823413610458}
{"step": 11100, "update": 5550, "epoch": 0, "loss": 0.43862153622700134, "answer_loss": 0.43857135064637986, "evidence_loss": 0.00010037092405127623, "lr": 4.2676215354288565e-06, "elapsed": 7881.112507343292}
{"step": 11200, "update": 5600, "epoch": 0, "loss": 0.40736669033183714, "answer_loss": 0.40225677982729396, "evidence_loss": 0.010219817206588004, "lr": 3.940594918622512e-06, "elapsed": 7995.154633760452}
{"step": 11300, "update": 5650, "epoch": 0, "loss": 0.49603209488268474, "answer_loss": 0.4827433528547408, "evidence_loss": 0.026577481231875026, "lr": 3.6255291691405652e-06, "elapsed": 8103.613470315933}
{"step": 11400, "update": 5700, "epoch": 0, "loss": 0.46784877389494794, "answer_loss": 0.4678220012877137, "evidence_loss": 5.354465382424678e-05, "lr": 3.3226032312415612e-06, "elapsed": 8202.745464324951}
{"step": 11500, "update": 5750, "epoch": 0, "loss": 0.5164436005242169, "answer_loss": 0.4999234300921671, "evidence_loss": 0.03304033864309986, "lr": 3.031989154275361e-06, "elapsed": 8335.71618938446}
{"step": 11600, "update": 5800, "epoch": 0, "loss": 0.45249480980506634, "answer_loss": 0.4464464603882516, "evidence_loss": 0.01209669656710048, "lr": 2.753851994966189e-06, "elapsed": 8436.460706472397}
{"step": 11700, "update": 5850, "epoch": 0, "loss": 0.45089312612020876, "answer_loss": 0.45075863058737015, "evidence_loss": 0.0002689919732142698, "lr": 2.4883497236673353e-06, "elapsed": 8541.418382883072}
{"step": 11800, "update": 5900, "epoch": 0, "loss": 0.4452268042968353, "answer_loss": 0.44424556556536116, "evidence_loss": 0.0019624787477732753, "lr": 2.235633134640497e-06, "elapsed": 8648.319731712341}
{"step": 11900, "update": 5950, "epoch": 0, "loss": 0.4153186084679328, "answer_loss": 0.41129743786295875, "evidence_loss": 0.008042340509709901, "lr": 1.9958457604109584e-06, "elapsed": 8752.258842468262}
{"step": 12000, "update": 6000, "epoch": 0, "loss": 0.42718260430934607, "answer_loss": 0.42684502567804883, "evidence_loss": 0.0006751604532109922, "lr": 1.7691237902470236e-06, "elapsed": 8851.503152132034}
{"step": 12100, "update": 6050, "epoch": 0, "loss": 0.47669057034196155, "answer_loss": 0.45829702616814755, "evidence_loss": 0.036787092402428244, "lr": 1.5555959928102188e-06, "elapsed": 8965.566213130951}
{"step": 12200, "update": 6100, "epoch": 0, "loss": 0.4425456356949871, "answer_loss": 0.4425402174919145, "evidence_loss": 1.0831030594999902e-05, "lr": 1.355383643020003e-06, "elapsed": 9064.531921386719}
{"step": 12300, "update": 6150, "epoch": 0, "loss": 0.44247193973104004, "answer_loss": 0.43773227051162394, "evidence_loss": 0.009479338799953893, "lr": 1.1686004531746757e-06, "elapsed": 9169.78522515297}
{"step": 12400, "update": 6200, "epoch": 0, "loss": 0.405267085271189, "answer_loss": 0.40471490265685134, "evidence_loss": 0.0011043679019212506, "lr": 9.953525083674947e-07, "elapsed": 9274.789207458496}
{"step": 12500, "update": 6250, "epoch": 0, "loss": 0.45154568844736787, "answer_loss": 0.44815644546149996, "evidence_loss": 0.006778485635009019, "lr": 8.357382062347819e-07, "elapsed": 9369.492041826248}
{"step": 12600, "update": 6300, "epoch": 0, "loss": 0.4115633034714847, "answer_loss": 0.4115076944499742, "evidence_loss": 0.00011121496921987273, "lr": 6.898482010701035e-07, "elapsed": 9493.970655679703}
{"step": 12700, "update": 6350, "epoch": 0, "loss": 0.46379801131319254, "answer_loss": 0.4637708245939575, "evidence_loss": 5.4374564861063846e-05, "lr": 5.577653523364479e-07, "elapsed": 9608.941450834274}
{"step": 12800, "update": 6400, "epoch": 0, "loss": 0.35798254798399287, "answer_loss": 0.35261169414268806, "evidence_loss": 0.010741709093840654, "lr": 4.3956467760545573e-07, "elapsed": 9717.863548517227}
{"step": 12900, "update": 6450, "epoch": 0, "loss": 0.4216876322880853, "answer_loss": 0.421679410393117, "evidence_loss": 1.6444868206235695e-05, "lr": 3.3531330995058287e-07, "elapsed": 9828.310291528702}
{"step": 13000, "update": 6500, "epoch": 0, "loss": 0.4379342442762572, "answer_loss": 0.43724123114021496, "evidence_loss": 0.0013860227765007949, "lr": 2.450704598182779e-07, "elapsed": 9935.698092222214}
{"step": 13100, "update": 6550, "epoch": 0, "loss": 0.43254806531302165, "answer_loss": 0.43075663063908, "evidence_loss": 0.0035828758762909275, "lr": 1.6888738139892724e-07, "elapsed": 10034.31571650505}
{"step": 13200, "update": 6600, "epoch": 0, "loss": 0.37071704588306603, "answer_loss": 0.37065767009044065, "evidence_loss": 0.00011874930562044028, "lr": 1.0680734351659061e-07, "elapsed": 10144.50322151184}
{"step": 13300, "update": 6650, "epoch": 0, "loss": 0.4850760158908088, "answer_loss": 0.48505614486406556, "evidence_loss": 3.974013048264169e-05, "lr": 5.886560505412031e-08, "elapsed": 10258.301747083664}
{"step": 13400, "update": 6700, "epoch": 0, "loss": 0.48753603306831794, "answer_loss": 0.48752045711735265, "evidence_loss": 3.115377083759085e-05, "lr": 2.5089394927563235e-08, "elapsed": 10363.920886039734}
{"step": 13500, "update": 6750, "epoch": 0, "loss": 0.47188873108942064, "answer_loss": 0.4717700101213995, "evidence_loss": 0.00023744011210965254, "lr": 5.497896621281528e-09, "elapsed": 10465.92549085617}
{"step": 13588, "dev": {"count": 5513, "accuracy": 0.8345728278614185, "brier": 0.24521264721151587, "brier_soft": 0.22135841773342682, "nll": 0.46613226108284944, "nll_soft": 0.48607260387495255, "ece": 0.04855302549823719, "mean_confidence": 0.8630526785971036, "confident_error_rate": 0.036277888626881916, "score_mae": 0.471010453508593, "evidence_auc": 0.9999887610142061}, "dev_by_source": {"ag_news": {"count": 100, "accuracy": 0.91, "brier": 0.14675387157720124, "brier_soft": 0.14675387157720124, "nll": 0.2912302939848356, "nll_soft": 0.2912302939848356, "ece": 0.06685722156976967, "mean_confidence": 0.941055656674617, "confident_error_rate": 0.05}, "amazon_polarity": {"count": 100, "accuracy": 0.95, "brier": 0.06195212131140546, "brier_soft": 0.06195212131140546, "nll": 0.10053970676292343, "nll_soft": 0.10053970676292343, "ece": 0.03339375320732605, "mean_confidence": 0.9740179407588133, "confident_error_rate": 0.01}, "amazon_stars": {"count": 100, "accuracy": 0.67, "brier": 0.47443850133431553, "brier_soft": 0.47443850133431553, "nll": 0.8165046029997326, "nll_soft": 0.8165046029997326, "ece": 0.13569209388350084, "mean_confidence": 0.5784805667887004, "confident_error_rate": 0.0, "score_mae": 0.4510158879717489}, "anli": {"count": 100, "accuracy": 0.74, "brier": 0.39964040280539254, "brier_soft": 0.39964040280539254, "nll": 0.7264094626418607, "nll_soft": 0.7264094626418607, "ece": 0.15583201403785235, "mean_confidence": 0.8675408405401412, "confident_error_rate": 0.08}, "arc_challenge": {"count": 100, "accuracy": 0.98, "brier": 0.0338466710496252, "brier_soft": 0.0338466710496252, "nll": 0.06223066351037851, "nll_soft": 0.06223066351037851, "ece": 0.013031628765155607, "mean_confidence": 0.9885343841116019, "confident_error_rate": 0.01}, "arc_easy": {"count": 100, "accuracy": 0.98, "brier": 0.02898907365294522, "brier_soft": 0.02898907365294522, "nll": 0.049804731538844765, "nll_soft": 0.049804731538844765, "ece": 0.0203574919070956, "mean_confidence": 0.9893880594062854, "confident_error_rate": 0.01}, "banking77": {"count": 100, "accuracy": 0.83, "brier": 0.28368283342030337, "brier_soft": 0.28368283342030337, "nll": 0.757150864558469, "nll_soft": 0.757150864558469, "ece": 0.10648182762315579, "mean_confidence": 0.8818236425754715, "confident_error_rate": 0.07}, "boolq": {"count": 125, "accuracy": 0.72, "brier": 0.4320747195048023, "brier_soft": 0.4320747195048023, "nll": 0.6419028511365726, "nll_soft": 0.6419028511365726, "ece": 0.22197986221059174, "mean_confidence": 0.9277224964550744, "confident_error_rate": 0.08, "evidence_auc": 1.0}, "cb": {"count": 28, "accuracy": 0.9642857142857143, "brier": 0.08009602886981086, "brier_soft": 0.08009602886981086, "nll": 0.1383170388054993, "nll_soft": 0.1383170388054993, "ece": 0.0539652842052121, "mean_confidence": 0.9543658374733184, "confident_error_rate": 0.03571428571428571}, "civil_comments_severity": {"count": 100, "accuracy": 0.86, "brier": 0.22068328925164493, "brier_soft": 0.1529327563945061, "nll": 0.4278688922326199, "nll_soft": 0.5868835533574466, "ece": 0.11533257179634501, "mean_confidence": 0.7616463548019397, "confident_error_rate": 0.01, "score_mae": 0.30176027449639936}, "civil_comments_toxic": {"count": 100, "accuracy": 0.96, "brier": 0.08335265266916769, "brier_soft": 0.035325646018993, "nll": 0.15999763596847744, "nll_soft": 0.273538273309447, "ece": 0.07025124166293703, "mean_confidence": 0.8897487583370629, "confident_error_rate": 0.0}, "clinc150": {"count": 100, "accuracy": 0.88, "brier": 0.1914855487934617, "brier_soft": 0.1914855487934617, "nll": 0.5818385912529987, "nll_soft": 0.5818385912529987, "ece": 0.07644133988403692, "mean_confidence": 0.9504764250442571, "confident_error_rate": 0.05}, "cola": {"count": 100, "accuracy": 0.82, "brier": 0.23936409929410002, "brier_soft": 0.23936409929410002, "nll": 0.3789224756239914, "nll_soft": 0.3789224756239914, "ece": 0.11088711147702807, "mean_confidence": 0.8847813277631773, "confident_error_rate": 0.05}, "commonsense_qa": {"count": 100, "accuracy": 0.91, "brier": 0.15279201504656054, "brier_soft": 0.15279201504656054, "nll": 0.38053510799344925, "nll_soft": 0.38053510799344925, "ece": 0.055310114097351794, "mean_confidence": 0.9592039264777928, "confident_error_rate": 0.05}, "cosmos_qa": {"count": 120, "accuracy": 0.7166666666666667, "brier": 0.46689377122498227, "brier_soft": 0.46689377122498227, "nll": 1.0493689192831135, "nll_soft": 1.0493689192831135, "ece": 0.1842691185339397, "mean_confidence": 0.8969067516393356, "confident_error_rate": 0.13333333333333333, "evidence_auc": 0.9996875}, "dbpedia": {"count": 100, "accuracy": 1.0, "brier": 0.003369142699290202, "brier_soft": 0.003369142699290202, "nll": 0.0062849907399302775, "nll_soft": 0.0062849907399302775, "ece": 0.005018755002197648, "mean_confidence": 0.9949812449978023, "confident_error_rate": 0.0}, "dynasent": {"count": 100, "accuracy": 0.74, "brier": 0.3740545426955645, "brier_soft": 0.26081181440449636, "nll": 0.6428067650335194, "nll_soft": 0.7148067644257405, "ece": 0.15895209598817536, "mean_confidence": 0.6365691849311554, "confident_error_rate": 0.0, "score_mae": 0.42895381500154295}, "enron_spam": {"count": 100, "accuracy": 0.99, "brier": 0.020026272903107403, "brier_soft": 0.020026272903107403, "nll": 0.03562782185316014, "nll_soft": 0.03562782185316014, "ece": 0.019098380748180314, "mean_confidence": 0.9891996384225809, "confident_error_rate": 0.01}, "go_emotions": {"count": 100, "accuracy": 0.61, "brier": 0.5741589899965046, "brier_soft": 0.21588675035041727, "nll": 1.4307033727614344, "nll_soft": 1.91358506717411, "ece": 0.20796112646172915, "mean_confidence": 0.4163597675175832, "confident_error_rate": 0.0}, "hellaswag": {"count": 100, "accuracy": 0.97, "brier": 0.04486725792629876, "brier_soft": 0.04486725792629876, "nll": 0.0745400472799033, "nll_soft": 0.0745400472799033, "ece": 0.03637399457536667, "mean_confidence": 0.9683134231198267, "confident_error_rate": 0.0}, "imdb": {"count": 100, "accuracy": 0.98, "brier": 0.04568085346299085, "brier_soft": 0.04568085346299085, "nll": 0.10292772526304866, "nll_soft": 0.10292772526304866, "ece": 0.028451957696295264, "mean_confidence": 0.9728709039056583, "confident_error_rate": 0.01}, "ledgar": {"count": 100, "accuracy": 0.77, "brier": 0.3097771675840078, "brier_soft": 0.3097771675840078, "nll": 0.7872336976006159, "nll_soft": 0.7872336976006159, "ece": 0.1081009333709956, "mean_confidence": 0.8781009333709956, "confident_error_rate": 0.05}, "liar": {"count": 100, "accuracy": 0.4, "brier": 0.7189634047428461, "brier_soft": 0.7189634047428461, "nll": 1.436837871988536, "nll_soft": 1.436837871988536, "ece": 0.06865019845085027, "mean_confidence": 0.3381281030587656, "confident_error_rate": 0.0, "score_mae": 0.9441482667310059}, "measuring_hate_speech": {"count": 100, "accuracy": 0.84, "brier": 0.25256207725930363, "brier_soft": 0.13580843340392565, "nll": 0.4850187830942446, "nll_soft": 0.6520812834706986, "ece": 0.1169304401504889, "mean_confidence": 0.7261185069578411, "confident_error_rate": 0.0, "score_mae": 0.49203690882532636}, "medqa": {"count": 100, "accuracy": 0.89, "brier": 0.1535992569978112, "brier_soft": 0.1535992569978112, "nll": 0.2625025810302494, "nll_soft": 0.2625025810302494, "ece": 0.07181873065356893, "mean_confidence": 0.9229456023995973, "confident_error_rate": 0.02}, "mnli": {"count": 100, "accuracy": 0.97, "brier": 0.05573973735423007, "brier_soft": 0.05573973735423007, "nll": 0.1098216209157549, "nll_soft": 0.1098216209157549, "ece": 0.06763109111270017, "mean_confidence": 0.9361391609146414, "confident_error_rate": 0.01}, "mrpc": {"count": 100, "accuracy": 0.9, "brier": 0.1668976595442053, "brier_soft": 0.1668976595442053, "nll": 0.26565708797200693, "nll_soft": 0.26565708797200693, "ece": 0.04940636901408266, "mean_confidence": 0.9098964097140114, "confident_error_rate": 0.02}, "multirc": {"count": 125, "accuracy": 0.768, "brier": 0.41389630382118325, "brier_soft": 0.41389630382118325, "nll": 0.7069573124860199, "nll_soft": 0.7069573124860199, "ece": 0.20462945216784828, "mean_confidence": 0.9358565258357632, "confident_error_rate": 0.152, "evidence_auc": 0.9997333333333334}, "openbookqa": {"count": 100, "accuracy": 0.98, "brier": 0.03658790275205525, "brier_soft": 0.03658790275205525, "nll": 0.0717812851783545, "nll_soft": 0.0717812851783545, "ece": 0.020032410400518365, "mean_confidence": 0.9769089288843115, "confident_error_rate": 0.0}, "piqa": {"count": 100, "accuracy": 0.91, "brier": 0.12530473877108808, "brier_soft": 0.12530473877108808, "nll": 0.23578200111355627, "nll_soft": 0.23578200111355627, "ece": 0.05405603140631721, "mean_confidence": 0.9513254109878784, "confident_error_rate": 0.03}, "policy_application": {"count": 150, "accuracy": 0.9466666666666667, "brier": 0.07846287806722692, "brier_soft": 0.07846287806722692, "nll": 0.13571846060017997, "nll_soft": 0.13571846060017997, "ece": 0.046881348646941226, "mean_confidence": 0.9855395932885163, "confident_error_rate": 0.03333333333333333, "evidence_auc": 1.0}, "qqp": {"count": 100, "accuracy": 0.92, "brier": 0.1322711368798063, "brier_soft": 0.1322711368798063, "nll": 0.23146400397635053, "nll_soft": 0.23146400397635053, "ece": 0.039428147172002034, "mean_confidence": 0.9206436247298535, "confident_error_rate": 0.04}, "race": {"count": 125, "accuracy": 0.792, "brier": 0.27270894946390484, "brier_soft": 0.27270894946390484, "nll": 0.5369188120492517, "nll_soft": 0.5369188120492517, "ece": 0.09056234813224388, "mean_confidence": 0.8570818045914835, "confident_error_rate": 0.04, "evidence_auc": 1.0}, "rotten_tomatoes": {"count": 100, "accuracy": 0.98, "brier": 0.034796008989876, "brier_soft": 0.034796008989876, "nll": 0.07214729080189161, "nll_soft": 0.07214729080189161, "ece": 0.016962863770244453, "mean_confidence": 0.9715926379457167, "confident_error_rate": 0.01}, "rte": {"count": 100, "accuracy": 0.93, "brier": 0.1188184574788011, "brier_soft": 0.1188184574788011, "nll": 0.2052937800968413, "nll_soft": 0.2052937800968413, "ece": 0.04706417933618942, "mean_confidence": 0.9770641793361895, "confident_error_rate": 0.03}, "scotus": {"count": 100, "accuracy": 0.61, "brier": 0.6325394621335682, "brier_soft": 0.6325394621335682, "nll": 1.6277428172003265, "nll_soft": 1.6277428172003265, "ece": 0.25235176671516, "mean_confidence": 0.8520386087095022, "confident_error_rate": 0.13}, "shp": {"count": 100, "accuracy": 0.63, "brier": 0.46616116956928333, "brier_soft": 0.10030172097143464, "nll": 0.6589587286619418, "nll_soft": 0.683599739803445, "ece": 0.10805766937270722, "mean_confidence": 0.5400307704442183, "confident_error_rate": 0.0}, "sms_spam": {"count": 100, "accuracy": 0.99, "brier": 0.012794386799121387, "brier_soft": 0.012794386799121387, "nll": 0.01940308157677538, "nll_soft": 0.01940308157677538, "ece": 0.011367616270430043, "mean_confidence": 0.9945961193100143, "confident_error_rate": 0.0}, "snli": {"count": 100, "accuracy": 0.92, "brier": 0.1289873154832004, "brier_soft": 0.1289873154832004, "nll": 0.23811747993428342, "nll_soft": 0.23811747993428342, "ece": 0.06210566404106858, "mean_confidence": 0.9526758552214877, "confident_error_rate": 0.05}, "social_iqa": {"count": 100, "accuracy": 0.83, "brier": 0.26525446472548825, "brier_soft": 0.26525446472548825, "nll": 0.5398700586730137, "nll_soft": 0.5398700586730137, "ece": 0.09454909352171337, "mean_confidence": 0.9155281247457637, "confident_error_rate": 0.06}, "sst2": {"count": 100, "accuracy": 0.98, "brier": 0.040267340327601445, "brier_soft": 0.040267340327601445, "nll": 0.11672427288562982, "nll_soft": 0.11672427288562982, "ece": 0.01171578851442261, "mean_confidence": 0.9896982162000899, "confident_error_rate": 0.02}, "sst5": {"count": 100, "accuracy": 0.67, "brier": 0.4781693492848172, "brier_soft": 0.4781693492848172, "nll": 0.829577053110301, "nll_soft": 0.829577053110301, "ece": 0.14951808706248798, "mean_confidence": 0.554350865405928, "confident_error_rate": 0.0, "score_mae": 0.4378708779667829}, "stance_abortion": {"count": 60, "accuracy": 0.65, "brier": 0.5461631051578539, "brier_soft": 0.5461631051578539, "nll": 0.9158908383826021, "nll_soft": 0.9158908383826021, "ece": 0.24628237403169523, "mean_confidence": 0.870929660288412, "confident_error_rate": 0.15}, "stance_atheism": {"count": 60, "accuracy": 0.8, "brier": 0.31998745599311396, "brier_soft": 0.31998745599311396, "nll": 0.5000077747926376, "nll_soft": 0.5000077747926376, "ece": 0.14067486996116663, "mean_confidence": 0.8896575243678433, "confident_error_rate": 0.03333333333333333}, "stance_feminist": {"count": 60, "accuracy": 0.7833333333333333, "brier": 0.30915546723565146, "brier_soft": 0.30915546723565146, "nll": 0.5200861811211221, "nll_soft": 0.5200861811211221, "ece": 0.09158266666502275, "mean_confidence": 0.8748122325072478, "confident_error_rate": 0.05}, "stance_hillary": {"count": 60, "accuracy": 0.7166666666666667, "brier": 0.3835906907389575, "brier_soft": 0.3835906907389575, "nll": 0.6306440958758969, "nll_soft": 0.6306440958758969, "ece": 0.12429301714888963, "mean_confidence": 0.8135901747968072, "confident_error_rate": 0.05}, "stsb": {"count": 100, "accuracy": 0.65, "brier": 0.4467653588401074, "brier_soft": 0.20158728761075526, "nll": 0.7971364231767178, "nll_soft": 0.8773070175167166, "ece": 0.12983348156631902, "mean_confidence": 0.6330772411677799, "confident_error_rate": 0.0, "score_mae": 0.4435737806984108}, "subjectivity": {"count": 100, "accuracy": 0.99, "brier": 0.021267046464386992, "brier_soft": 0.021267046464386992, "nll": 0.04746715932690836, "nll_soft": 0.04746715932690836, "ece": 0.03920972397831717, "mean_confidence": 0.9638173734240708, "confident_error_rate": 0.0}, "swag": {"count": 100, "accuracy": 0.82, "brier": 0.25416755318707396, "brier_soft": 0.25416755318707396, "nll": 0.4673240743680174, "nll_soft": 0.4673240743680174, "ece": 0.08878300453136384, "mean_confidence": 0.9073698161052748, "confident_error_rate": 0.05}, "trec": {"count": 100, "accuracy": 0.97, "brier": 0.042537389155980695, "brier_soft": 0.042537389155980695, "nll": 0.11417283998567929, "nll_soft": 0.11417283998567929, "ece": 0.02627913171850392, "mean_confidence": 0.9776115812099822, "confident_error_rate": 0.01}, "tweet_hate": {"count": 100, "accuracy": 0.5, "brier": 0.6857332540214516, "brier_soft": 0.6857332540214516, "nll": 0.981097446252855, "nll_soft": 0.981097446252855, "ece": 0.33351503217194933, "mean_confidence": 0.833515032171949, "confident_error_rate": 0.11}, "tweet_sentiment": {"count": 100, "accuracy": 0.74, "brier": 0.38173636485792045, "brier_soft": 0.38173636485792045, "nll": 0.648052632172017, "nll_soft": 0.648052632172017, "ece": 0.11421575179540841, "mean_confidence": 0.6257842482045916, "confident_error_rate": 0.0, "score_mae": 0.35166887086126225}, "vitaminc": {"count": 100, "accuracy": 0.85, "brier": 0.23076445057087602, "brier_soft": 0.23076445057087602, "nll": 0.4375848144910949, "nll_soft": 0.4375848144910949, "ece": 0.09955280097416959, "mean_confidence": 0.91871404204557, "confident_error_rate": 0.06}, "wic": {"count": 100, "accuracy": 0.78, "brier": 0.3413490724528473, "brier_soft": 0.3413490724528473, "nll": 0.5457579942139619, "nll_soft": 0.5457579942139619, "ece": 0.16134131925967396, "mean_confidence": 0.9073427942234525, "confident_error_rate": 0.1}, "winogrande": {"count": 100, "accuracy": 0.91, "brier": 0.16856567514663653, "brier_soft": 0.16856567514663653, "nll": 0.3547808080133929, "nll_soft": 0.3547808080133929, "ece": 0.07623049848240572, "mean_confidence": 0.945613134694146, "confident_error_rate": 0.06}, "yelp": {"count": 100, "accuracy": 0.73, "brier": 0.40297617138952174, "brier_soft": 0.40297617138952174, "nll": 0.685889895087886, "nll_soft": 0.685889895087886, "ece": 0.1806122500426958, "mean_confidence": 0.622435088128629, "confident_error_rate": 0.0, "score_mae": 0.38806539902485776}}}
{"step": 13588, "dev": {"count": 5513, "accuracy": 0.8342100489751496, "brier": 0.24499939603673204, "brier_soft": 0.22122350305548508, "nll": 0.4655553015844398, "nll_soft": 0.48562642001788464, "ece": 0.04843755672557183, "mean_confidence": 0.8631851509248467, "confident_error_rate": 0.0359151097406131, "score_mae": 0.4705968118889738, "evidence_auc": 0.9999878244320566}, "dev_by_source": {"ag_news": {"count": 100, "accuracy": 0.91, "brier": 0.1465319479149763, "brier_soft": 0.1465319479149763, "nll": 0.2904767555416907, "nll_soft": 0.2904767555416907, "ece": 0.06696793567024037, "mean_confidence": 0.9411689492273619, "confident_error_rate": 0.05}, "amazon_polarity": {"count": 100, "accuracy": 0.95, "brier": 0.06128869305634376, "brier_soft": 0.06128869305634376, "nll": 0.10096164621475562, "nll_soft": 0.10096164621475562, "ece": 0.023046591891905697, "mean_confidence": 0.9730465918919056, "confident_error_rate": 0.01}, "amazon_stars": {"count": 100, "accuracy": 0.69, "brier": 0.4731412386648261, "brier_soft": 0.4731412386648261, "nll": 0.8151410149736901, "nll_soft": 0.8151410149736901, "ece": 0.1366735053126959, "mean_confidence": 0.579111282045835, "confident_error_rate": 0.0, "score_mae": 0.4509802678028108}, "anli": {"count": 100, "accuracy": 0.74, "brier": 0.3976344462707522, "brier_soft": 0.3976344462707522, "nll": 0.7224163582147134, "nll_soft": 0.7224163582147134, "ece": 0.14788199940722557, "mean_confidence": 0.8672915745774106, "confident_error_rate": 0.08}, "arc_challenge": {"count": 100, "accuracy": 0.98, "brier": 0.03366315261301923, "brier_soft": 0.03366315261301923, "nll": 0.060837047229714225, "nll_soft": 0.060837047229714225, "ece": 0.011432103687330048, "mean_confidence": 0.9886713474416573, "confident_error_rate": 0.01}, "arc_easy": {"count": 100, "accuracy": 0.98, "brier": 0.029996236913866735, "brier_soft": 0.029996236913866735, "nll": 0.05253505337779216, "nll_soft": 0.05253505337779216, "ece": 0.018028940214597153, "mean_confidence": 0.990060141873229, "confident_error_rate": 0.01}, "banking77": {"count": 100, "accuracy": 0.83, "brier": 0.28099173269454264, "brier_soft": 0.28099173269454264, "nll": 0.7554653567994057, "nll_soft": 0.7554653567994057, "ece": 0.11505295474931794, "mean_confidence": 0.8828400651254894, "confident_error_rate": 0.07}, "boolq": {"count": 125, "accuracy": 0.72, "brier": 0.43268232797354933, "brier_soft": 0.43268232797354933, "nll": 0.6422496879755143, "nll_soft": 0.6422496879755143, "ece": 0.22190986014776565, "mean_confidence": 0.9281403357012442, "confident_error_rate": 0.072, "evidence_auc": 1.0}, "cb": {"count": 28, "accuracy": 0.9642857142857143, "brier": 0.07835667610715101, "brier_soft": 0.07835667610715101, "nll": 0.13341829300103206, "nll_soft": 0.13341829300103206, "ece": 0.05279191576259608, "mean_confidence": 0.954691373765687, "confident_error_rate": 0.03571428571428571}, "civil_comments_severity": {"count": 100, "accuracy": 0.85, "brier": 0.21990075062806394, "brier_soft": 0.15256546144647498, "nll": 0.42595271409351193, "nll_soft": 0.5859219055739261, "ece": 0.1029050524606589, "mean_confidence": 0.7619406731562297, "confident_error_rate": 0.0, "score_mae": 0.3013470975763519}, "civil_comments_toxic": {"count": 100, "accuracy": 0.96, "brier": 0.08184155550658603, "brier_soft": 0.035637691017917736, "nll": 0.15796842110857123, "nll_soft": 0.2738468412771658, "ece": 0.07203162905008202, "mean_confidence": 0.8907618087052214, "confident_error_rate": 0.0}, "clinc150": {"count": 100, "accuracy": 0.88, "brier": 0.19125913762163368, "brier_soft": 0.19125913762163368, "nll": 0.5776643056115763, "nll_soft": 0.5776643056115763, "ece": 0.09283844266870002, "mean_confidence": 0.9496720060766602, "confident_error_rate": 0.06}, "cola": {"count": 100, "accuracy": 0.82, "brier": 0.23386052759583184, "brier_soft": 0.23386052759583184, "nll": 0.37037031376428564, "nll_soft": 0.37037031376428564, "ece": 0.09766561804689824, "mean_confidence": 0.8833423357882064, "confident_error_rate": 0.03}, "commonsense_qa": {"count": 100, "accuracy": 0.91, "brier": 0.15377198586333088, "brier_soft": 0.15377198586333088, "nll": 0.3819044445089102, "nll_soft": 0.3819044445089102, "ece": 0.05144685490055159, "mean_confidence": 0.9596385183475031, "confident_error_rate": 0.05}, "cosmos_qa": {"count": 120, "accuracy": 0.7166666666666667, "brier": 0.4672354484316241, "brier_soft": 0.4672354484316241, "nll": 1.0482225583041105, "nll_soft": 1.0482225583041105, "ece": 0.19858953692704095, "mean_confidence": 0.8962658679831853, "confident_error_rate": 0.13333333333333333, "evidence_auc": 0.999375}, "dbpedia": {"count": 100, "accuracy": 1.0, "brier": 0.0033691323280231667, "brier_soft": 0.0033691323280231667, "nll": 0.00628770684289355, "nll_soft": 0.00628770684289355, "ece": 0.0050215623881869585, "mean_confidence": 0.994978437611813, "confident_error_rate": 0.0}, "dynasent": {"count": 100, "accuracy": 0.73, "brier": 0.37323258779066765, "brier_soft": 0.2605018455661368, "nll": 0.6422479980426224, "nll_soft": 0.7148729964174031, "ece": 0.15595018764547483, "mean_confidence": 0.6386131537774967, "confident_error_rate": 0.0, "score_mae": 0.42700063346629696}, "enron_spam": {"count": 100, "accuracy": 0.99, "brier": 0.020538910131957966, "brier_soft": 0.020538910131957966, "nll": 0.037125773454637925, "nll_soft": 0.037125773454637925, "ece": 0.018135869575440167, "mean_confidence": 0.9889960329597918, "confident_error_rate": 0.01}, "go_emotions": {"count": 100, "accuracy": 0.6, "brier": 0.5706133279962924, "brier_soft": 0.2146223457949257, "nll": 1.4243747447457458, "nll_soft": 1.9114536152236135, "ece": 0.2153529377728328, "mean_confidence": 0.4180754880267198, "confident_error_rate": 0.0}, "hellaswag": {"count": 100, "accuracy": 0.97, "brier": 0.04398586313752212, "brier_soft": 0.04398586313752212, "nll": 0.07282392796652562, "nll_soft": 0.07282392796652562, "ece": 0.03281997526145711, "mean_confidence": 0.967702411105094, "confident_error_rate": 0.0}, "imdb": {"count": 100, "accuracy": 0.98, "brier": 0.04503053337307919, "brier_soft": 0.04503053337307919, "nll": 0.10202723595434093, "nll_soft": 0.10202723595434093, "ece": 0.02799987491297936, "mean_confidence": 0.9735376597268748, "confident_error_rate": 0.01}, "ledgar": {"count": 100, "accuracy": 0.78, "brier": 0.31022259775593414, "brier_soft": 0.31022259775593414, "nll": 0.7916681164712028, "nll_soft": 0.7916681164712028, "ece": 0.10685236902591677, "mean_confidence": 0.8791648234039952, "confident_error_rate": 0.05}, "liar": {"count": 100, "accuracy": 0.39, "brier": 0.7168465726369471, "brier_soft": 0.7168465726369471, "nll": 1.430973024742663, "nll_soft": 1.430973024742663, "ece": 0.07922166544125636, "mean_confidence": 0.33914869030092576, "confident_error_rate": 0.0, "score_mae": 0.9383143759928696}, "measuring_hate_speech": {"count": 100, "accuracy": 0.84, "brier": 0.2523259794626622, "brier_soft": 0.1350772440937763, "nll": 0.48511890737634117, "nll_soft": 0.6516605742694233, "ece": 0.1276134336558651, "mean_confidence": 0.7243118123610279, "confident_error_rate": 0.0, "score_mae": 0.49301893575302613}, "medqa": {"count": 100, "accuracy": 0.88, "brier": 0.15509559657024174, "brier_soft": 0.15509559657024174, "nll": 0.2630871286801779, "nll_soft": 0.2630871286801779, "ece": 0.07094760936712856, "mean_confidence": 0.9227796946829281, "confident_error_rate": 0.02}, "mnli": {"count": 100, "accuracy": 0.97, "brier": 0.05622277315175722, "brier_soft": 0.05622277315175722, "nll": 0.11140887517203295, "nll_soft": 0.11140887517203295, "ece": 0.06498924366805267, "mean_confidence": 0.9358105562109016, "confident_error_rate": 0.01}, "mrpc": {"count": 100, "accuracy": 0.9, "brier": 0.16885290914097817, "brier_soft": 0.16885290914097817, "nll": 0.26678509569101083, "nll_soft": 0.26678509569101083, "ece": 0.056738491245727135, "mean_confidence": 0.9094846375493769, "confident_error_rate": 0.02}, "multirc": {"count": 125, "accuracy": 0.768, "brier": 0.4125322538820213, "brier_soft": 0.4125322538820213, "nll": 0.702692751642681, "nll_soft": 0.702692751642681, "ece": 0.2040484715924632, "mean_confidence": 0.9360141123782398, "confident_error_rate": 0.152, "evidence_auc": 0.9997333333333334}, "openbookqa": {"count": 100, "accuracy": 0.98, "brier": 0.036469910012703805, "brier_soft": 0.036469910012703805, "nll": 0.07196625626831986, "nll_soft": 0.07196625626831986, "ece": 0.02228237729925685, "mean_confidence": 0.977701842636366, "confident_error_rate": 0.0}, "piqa": {"count": 100, "accuracy": 0.91, "brier": 0.12451330680810697, "brier_soft": 0.12451330680810697, "nll": 0.23446545298444263, "nll_soft": 0.23446545298444263, "ece": 0.06753078707740053, "mean_confidence": 0.9522836212325586, "confident_error_rate": 0.03}, "policy_application": {"count": 150, "accuracy": 0.9466666666666667, "brier": 0.0777876053884264, "brier_soft": 0.0777876053884264, "nll": 0.13521520720550428, "nll_soft": 0.13521520720550428, "ece": 0.04668170693287413, "mean_confidence": 0.985035205551867, "confident_error_rate": 0.03333333333333333, "evidence_auc": 1.0}, "qqp": {"count": 100, "accuracy": 0.92, "brier": 0.13043954738084676, "brier_soft": 0.13043954738084676, "nll": 0.22957888778440388, "nll_soft": 0.22957888778440388, "ece": 0.03919742572811998, "mean_confidence": 0.9198058774243847, "confident_error_rate": 0.04}, "race": {"count": 125, "accuracy": 0.792, "brier": 0.273025071208486, "brier_soft": 0.273025071208486, "nll": 0.5401223012436509, "nll_soft": 0.5401223012436509, "ece": 0.09241534466772917, "mean_confidence": 0.8570207713461646, "confident_error_rate": 0.04, "evidence_auc": 1.0}, "rotten_tomatoes": {"count": 100, "accuracy": 0.98, "brier": 0.035290060552569, "brier_soft": 0.035290060552569, "nll": 0.07276457767313435, "nll_soft": 0.07276457767313435, "ece": 0.017489686473322937, "mean_confidence": 0.9711417797012452, "confident_error_rate": 0.01}, "rte": {"count": 100, "accuracy": 0.93, "brier": 0.11679704186673533, "brier_soft": 0.11679704186673533, "nll": 0.20138234100382452, "nll_soft": 0.20138234100382452, "ece": 0.04667671248922903, "mean_confidence": 0.976676712489229, "confident_error_rate": 0.03}, "scotus": {"count": 100, "accuracy": 0.61, "brier": 0.6341498051249195, "brier_soft": 0.6341498051249195, "nll": 1.6277115865464133, "nll_soft": 1.6277115865464133, "ece": 0.25533107042012765, "mean_confidence": 0.8527349247542704, "confident_error_rate": 0.13}, "shp": {"count": 100, "accuracy": 0.65, "brier": 0.46443770760873426, "brier_soft": 0.09936027473515505, "nll": 0.6572154778957793, "nll_soft": 0.6826727776739735, "ece": 0.1103068097721516, "mean_confidence": 0.5396931902278483, "confident_error_rate": 0.0}, "sms_spam": {"count": 100, "accuracy": 0.99, "brier": 0.01279388922890178, "brier_soft": 0.01279388922890178, "nll": 0.019404364213720597, "nll_soft": 0.019404364213720597, "ece": 0.011369026874576384, "mean_confidence": 0.994594708705868, "confident_error_rate": 0.0}, "snli": {"count": 100, "accuracy": 0.92, "brier": 0.1303060757325426, "brier_soft": 0.1303060757325426, "nll": 0.240589231743456, "nll_soft": 0.240589231743456, "ece": 0.04670265643035693, "mean_confidence": 0.9518659140843424, "confident_error_rate": 0.05}, "social_iqa": {"count": 100, "accuracy": 0.84, "brier": 0.2641777007342555, "brier_soft": 0.2641777007342555, "nll": 0.5338294339120516, "nll_soft": 0.5338294339120516, "ece": 0.09934345700863399, "mean_confidence": 0.9154886411744081, "confident_error_rate": 0.06}, "sst2": {"count": 100, "accuracy": 0.98, "brier": 0.04030628615298806, "brier_soft": 0.04030628615298806, "nll": 0.11747284260512826, "nll_soft": 0.11747284260512826, "ece": 0.012949483786838481, "mean_confidence": 0.9895809775228284, "confident_error_rate": 0.02}, "sst5": {"count": 100, "accuracy": 0.66, "brier": 0.47858939401943057, "brier_soft": 0.47858939401943057, "nll": 0.8312380624917151, "nll_soft": 0.8312380624917151, "ece": 0.14479568835917578, "mean_confidence": 0.5538169983846644, "confident_error_rate": 0.0, "score_mae": 0.4382710876978508}, "stance_abortion": {"count": 60, "accuracy": 0.65, "brier": 0.5483838340126799, "brier_soft": 0.5483838340126799, "nll": 0.916795129103024, "nll_soft": 0.916795129103024, "ece": 0.2470612492514964, "mean_confidence": 0.8717795686329541, "confident_error_rate": 0.15}, "stance_atheism": {"count": 60, "accuracy": 0.7833333333333333, "brier": 0.31917383626334284, "brier_soft": 0.31917383626334284, "nll": 0.49926359046026203, "nll_soft": 0.49926359046026203, "ece": 0.15642293053711553, "mean_confidence": 0.889754045572878, "confident_error_rate": 0.03333333333333333}, "stance_feminist": {"count": 60, "accuracy": 0.7833333333333333, "brier": 0.3049816554075578, "brier_soft": 0.3049816554075578, "nll": 0.5154249440169435, "nll_soft": 0.5154249440169435, "ece": 0.10461663391963241, "mean_confidence": 0.8737570622243047, "confident_error_rate": 0.05}, "stance_hillary": {"count": 60, "accuracy": 0.7166666666666667, "brier": 0.3812783742364739, "brier_soft": 0.3812783742364739, "nll": 0.6242170771095513, "nll_soft": 0.6242170771095513, "ece": 0.11280688269490617, "mean_confidence": 0.8143798914819186, "confident_error_rate": 0.05}, "stsb": {"count": 100, "accuracy": 0.65, "brier": 0.45170767908012466, "brier_soft": 0.20552974536260238, "nll": 0.8029710252702897, "nll_soft": 0.8819413373331731, "ece": 0.10983419549350473, "mean_confidence": 0.6357630237891139, "confident_error_rate": 0.0, "score_mae": 0.44494913340968734}, "subjectivity": {"count": 100, "accuracy": 0.99, "brier": 0.02143104125551823, "brier_soft": 0.02143104125551823, "nll": 0.047739931072467476, "nll_soft": 0.047739931072467476, "ece": 0.039431940261527555, "mean_confidence": 0.9635951571408603, "confident_error_rate": 0.0}, "swag": {"count": 100, "accuracy": 0.82, "brier": 0.2537048639515909, "brier_soft": 0.2537048639515909, "nll": 0.4672971887646461, "nll_soft": 0.4672971887646461, "ece": 0.08815823139371007, "mean_confidence": 0.9081582313937099, "confident_error_rate": 0.05}, "trec": {"count": 100, "accuracy": 0.97, "brier": 0.04378629853100501, "brier_soft": 0.04378629853100501, "nll": 0.11549823864485118, "nll_soft": 0.11549823864485118, "ece": 0.025471181892042873, "mean_confidence": 0.9775477901155262, "confident_error_rate": 0.01}, "tweet_hate": {"count": 100, "accuracy": 0.5, "brier": 0.6883996904854307, "brier_soft": 0.6883996904854307, "nll": 0.9833215742341621, "nll_soft": 0.9833215742341621, "ece": 0.3459218953511739, "mean_confidence": 0.8344252768252641, "confident_error_rate": 0.11}, "tweet_sentiment": {"count": 100, "accuracy": 0.74, "brier": 0.3831706610576699, "brier_soft": 0.3831706610576699, "nll": 0.6497370609790863, "nll_soft": 0.6497370609790863, "ece": 0.13208543978054657, "mean_confidence": 0.6252989072735136, "confident_error_rate": 0.0, "score_mae": 0.3541000877573742}, "vitaminc": {"count": 100, "accuracy": 0.84, "brier": 0.23269831680007946, "brier_soft": 0.23269831680007946, "nll": 0.4394585245066372, "nll_soft": 0.4394585245066372, "ece": 0.09072915887713841, "mean_confidence": 0.9197737336036776, "confident_error_rate": 0.06}, "wic": {"count": 100, "accuracy": 0.78, "brier": 0.34073352692700504, "brier_soft": 0.34073352692700504, "nll": 0.5452054678565941, "nll_soft": 0.5452054678565941, "ece": 0.13825743108455368, "mean_confidence": 0.9071892135345369, "confident_error_rate": 0.1}, "winogrande": {"count": 100, "accuracy": 0.91, "brier": 0.17067644004760918, "brier_soft": 0.17067644004760918, "nll": 0.35855105926065917, "nll_soft": 0.35855105926065917, "ece": 0.08624812639081393, "mean_confidence": 0.9450851377589773, "confident_error_rate": 0.07}, "yelp": {"count": 100, "accuracy": 0.73, "brier": 0.402625269533739, "brier_soft": 0.402625269533739, "nll": 0.6852448507549156, "nll_soft": 0.6852448507549156, "ece": 0.16558757141209074, "mean_confidence": 0.6233702191486455, "confident_error_rate": 0.0, "score_mae": 0.3873896875444972}}}