{
  "results": {
    "crows_pairs_english_religion": {
      "likelihood_difference": 5.7657657657657655,
      "likelihood_difference_stderr": 0.7051443071103357,
      "pct_stereotype": 0.5135135135135135,
      "pct_stereotype_stderr": 0.04765571461988585
    },
    "hendrycksTest-jurisprudence": {
      "acc": 0.16666666666666666,
      "acc_stderr": 0.036028141763926456,
      "acc_norm": 0.26851851851851855,
      "acc_norm_stderr": 0.04284467968052191
    },
    "hendrycksTest-high_school_mathematics": {
      "acc": 0.22592592592592592,
      "acc_stderr": 0.02549753263960955,
      "acc_norm": 0.2740740740740741,
      "acc_norm_stderr": 0.027195934804085626
    },
    "hendrycksTest-high_school_government_and_politics": {
      "acc": 0.20725388601036268,
      "acc_stderr": 0.02925282329180363,
      "acc_norm": 0.25906735751295334,
      "acc_norm_stderr": 0.031618779179354094
    },
    "hendrycksTest-human_sexuality": {
      "acc": 0.2366412213740458,
      "acc_stderr": 0.03727673575596918,
      "acc_norm": 0.2824427480916031,
      "acc_norm_stderr": 0.03948406125768362
    },
    "crows_pairs_english_socioeconomic": {
      "likelihood_difference": 5.946875,
      "likelihood_difference_stderr": 0.5419079991233648,
      "pct_stereotype": 0.5,
      "pct_stereotype_stderr": 0.036369648372665396
    },
    "hendrycksTest-professional_medicine": {
      "acc": 0.22794117647058823,
      "acc_stderr": 0.025483081468029804,
      "acc_norm": 0.2647058823529412,
      "acc_norm_stderr": 0.026799562024887685
    },
    "hendrycksTest-elementary_mathematics": {
      "acc": 0.22486772486772486,
      "acc_stderr": 0.021502096078229147,
      "acc_norm": 0.23544973544973544,
      "acc_norm_stderr": 0.021851509822031708
    },
    "hendrycksTest-high_school_psychology": {
      "acc": 0.24770642201834864,
      "acc_stderr": 0.018508143602547798,
      "acc_norm": 0.25504587155963304,
      "acc_norm_stderr": 0.018688500856535843
    },
    "crows_pairs_french_disability": {
      "likelihood_difference": 14.919507575757576,
      "likelihood_difference_stderr": 1.4664292625655948,
      "pct_stereotype": 0.3787878787878788,
      "pct_stereotype_stderr": 0.06016741025240241
    },
    "hendrycksTest-high_school_geography": {
      "acc": 0.2474747474747475,
      "acc_stderr": 0.030746300742124488,
      "acc_norm": 0.25757575757575757,
      "acc_norm_stderr": 0.03115626951964684
    },
    "hendrycksTest-college_computer_science": {
      "acc": 0.27,
      "acc_stderr": 0.044619604333847394,
      "acc_norm": 0.18,
      "acc_norm_stderr": 0.03861229196653695
    },
    "hendrycksTest-us_foreign_policy": {
      "acc": 0.2,
      "acc_stderr": 0.04020151261036846,
      "acc_norm": 0.28,
      "acc_norm_stderr": 0.04512608598542128
    },
    "hendrycksTest-computer_security": {
      "acc": 0.23,
      "acc_stderr": 0.042295258468165065,
      "acc_norm": 0.29,
      "acc_norm_stderr": 0.045604802157206845
    },
    "hendrycksTest-abstract_algebra": {
      "acc": 0.27,
      "acc_stderr": 0.044619604333847394,
      "acc_norm": 0.2,
      "acc_norm_stderr": 0.04020151261036845
    },
    "hendrycksTest-formal_logic": {
      "acc": 0.2698412698412698,
      "acc_stderr": 0.03970158273235172,
      "acc_norm": 0.2619047619047619,
      "acc_norm_stderr": 0.039325376803928724
    },
    "hendrycksTest-moral_disputes": {
      "acc": 0.2138728323699422,
      "acc_stderr": 0.022075709251757177,
      "acc_norm": 0.20809248554913296,
      "acc_norm_stderr": 0.0218552552634218
    },
    "hendrycksTest-professional_law": {
      "acc": 0.2333767926988266,
      "acc_stderr": 0.010803108481179097,
      "acc_norm": 0.23728813559322035,
      "acc_norm_stderr": 0.01086543669078027
    },
    "hendrycksTest-prehistory": {
      "acc": 0.2654320987654321,
      "acc_stderr": 0.024569223600460852,
      "acc_norm": 0.22839506172839505,
      "acc_norm_stderr": 0.023358211840626267
    },
    "winogrande": {
      "acc": 0.48224151539068666,
      "acc_stderr": 0.014043619596174964
    },
    "hendrycksTest-college_mathematics": {
      "acc": 0.18,
      "acc_stderr": 0.038612291966536955,
      "acc_norm": 0.18,
      "acc_norm_stderr": 0.03861229196653696
    },
    "hendrycksTest-miscellaneous": {
      "acc": 0.2681992337164751,
      "acc_stderr": 0.015842430835269435,
      "acc_norm": 0.26436781609195403,
      "acc_norm_stderr": 0.015769984840690518
    },
    "hendrycksTest-high_school_statistics": {
      "acc": 0.2361111111111111,
      "acc_stderr": 0.028963702570791037,
      "acc_norm": 0.2824074074074074,
      "acc_norm_stderr": 0.030701372111510934
    },
    "hendrycksTest-business_ethics": {
      "acc": 0.25,
      "acc_stderr": 0.04351941398892446,
      "acc_norm": 0.26,
      "acc_norm_stderr": 0.04408440022768079
    },
    "sciq": {
      "acc": 0.205,
      "acc_stderr": 0.012772554096113121,
      "acc_norm": 0.205,
      "acc_norm_stderr": 0.012772554096113123
    },
    "hendrycksTest-marketing": {
      "acc": 0.23076923076923078,
      "acc_stderr": 0.027601921381417607,
      "acc_norm": 0.2564102564102564,
      "acc_norm_stderr": 0.028605953702004243
    },
    "hendrycksTest-electrical_engineering": {
      "acc": 0.2,
      "acc_stderr": 0.0333333333333333,
      "acc_norm": 0.2,
      "acc_norm_stderr": 0.03333333333333331
    },
    "hendrycksTest-global_facts": {
      "acc": 0.29,
      "acc_stderr": 0.045604802157206845,
      "acc_norm": 0.26,
      "acc_norm_stderr": 0.044084400227680794
    },
    "crows_pairs_english_physical_appearance": {
      "likelihood_difference": 6.262586805555555,
      "likelihood_difference_stderr": 0.8386766732679737,
      "pct_stereotype": 0.5694444444444444,
      "pct_stereotype_stderr": 0.05876396677084613
    },
    "crows_pairs_french_nationality": {
      "likelihood_difference": 9.763833992094861,
      "likelihood_difference_stderr": 0.5454943873296275,
      "pct_stereotype": 0.5533596837944664,
      "pct_stereotype_stderr": 0.03131716554414947
    },
    "hendrycksTest-college_chemistry": {
      "acc": 0.21,
      "acc_stderr": 0.04093601807403325,
      "acc_norm": 0.25,
      "acc_norm_stderr": 0.04351941398892446
    },
    "crows_pairs_english_autre": {
      "likelihood_difference": 5.792613636363637,
      "likelihood_difference_stderr": 3.094109462379434,
      "pct_stereotype": 0.45454545454545453,
      "pct_stereotype_stderr": 0.15745916432444335
    },
    "crows_pairs_french_age": {
      "likelihood_difference": 5.9534722222222225,
      "likelihood_difference_stderr": 0.7502305104509913,
      "pct_stereotype": 0.6666666666666666,
      "pct_stereotype_stderr": 0.04996877926639073
    },
    "crows_pairs_english_disability": {
      "likelihood_difference": 7.8625,
      "likelihood_difference_stderr": 1.2300665742321708,
      "pct_stereotype": 0.6,
      "pct_stereotype_stderr": 0.06123724356957946
    },
    "crows_pairs_french_race_color": {
      "likelihood_difference": 9.584782608695653,
      "likelihood_difference_stderr": 0.3782157420364811,
      "pct_stereotype": 0.6978260869565217,
      "pct_stereotype_stderr": 0.021433630616234406
    },
    "hendrycksTest-high_school_macroeconomics": {
      "acc": 0.2230769230769231,
      "acc_stderr": 0.021107730127243988,
      "acc_norm": 0.23846153846153847,
      "acc_norm_stderr": 0.021606294494647734
    },
    "hendrycksTest-high_school_world_history": {
      "acc": 0.18565400843881857,
      "acc_stderr": 0.025310495376944856,
      "acc_norm": 0.20253164556962025,
      "acc_norm_stderr": 0.02616056824660147
    },
    "crows_pairs_english_sexual_orientation": {
      "likelihood_difference": 5.498655913978495,
      "likelihood_difference_stderr": 0.7803814003203932,
      "pct_stereotype": 0.45161290322580644,
      "pct_stereotype_stderr": 0.051883930752016603
    },
    "hendrycksTest-college_medicine": {
      "acc": 0.20809248554913296,
      "acc_stderr": 0.030952890217749884,
      "acc_norm": 0.24277456647398843,
      "acc_norm_stderr": 0.0326926380614177
    },
    "hendrycksTest-high_school_european_history": {
      "acc": 0.18787878787878787,
      "acc_stderr": 0.03050193405942914,
      "acc_norm": 0.20606060606060606,
      "acc_norm_stderr": 0.031584153240477086
    },
    "hendrycksTest-philosophy": {
      "acc": 0.22186495176848875,
      "acc_stderr": 0.02359885829286305,
      "acc_norm": 0.2861736334405145,
      "acc_norm_stderr": 0.025670259242188943
    },
    "crows_pairs_french_physical_appearance": {
      "likelihood_difference": 10.22482638888889,
      "likelihood_difference_stderr": 1.3720668559778915,
      "pct_stereotype": 0.4861111111111111,
      "pct_stereotype_stderr": 0.05931618532716555
    },
    "crows_pairs_english": {
      "likelihood_difference": 5.680251192605843,
      "likelihood_difference_stderr": 0.1962056921179939,
      "pct_stereotype": 0.4883720930232558,
      "pct_stereotype_stderr": 0.01220999609506964
    },
    "hendrycksTest-high_school_chemistry": {
      "acc": 0.2019704433497537,
      "acc_stderr": 0.02824735012218027,
      "acc_norm": 0.21182266009852216,
      "acc_norm_stderr": 0.028748983689941075
    },
    "hendrycksTest-medical_genetics": {
      "acc": 0.25,
      "acc_stderr": 0.04351941398892446,
      "acc_norm": 0.2,
      "acc_norm_stderr": 0.040201512610368445
    },
    "hendrycksTest-high_school_us_history": {
      "acc": 0.17647058823529413,
      "acc_stderr": 0.026756401538078955,
      "acc_norm": 0.2549019607843137,
      "acc_norm_stderr": 0.030587591351604257
    },
    "hendrycksTest-high_school_biology": {
      "acc": 0.20967741935483872,
      "acc_stderr": 0.02315787934908352,
      "acc_norm": 0.22258064516129034,
      "acc_norm_stderr": 0.023664216671642518
    },
    "piqa": {
      "acc": 0.5266594124047879,
      "acc_stderr": 0.011649229994347386,
      "acc_norm": 0.5233949945593036,
      "acc_norm_stderr": 0.011653047155927796
    },
    "hendrycksTest-human_aging": {
      "acc": 0.27802690582959644,
      "acc_stderr": 0.03006958487449403,
      "acc_norm": 0.27802690582959644,
      "acc_norm_stderr": 0.03006958487449403
    },
    "hendrycksTest-econometrics": {
      "acc": 0.22807017543859648,
      "acc_stderr": 0.03947152782669415,
      "acc_norm": 0.3157894736842105,
      "acc_norm_stderr": 0.04372748290278008
    },
    "hendrycksTest-sociology": {
      "acc": 0.26865671641791045,
      "acc_stderr": 0.031343283582089536,
      "acc_norm": 0.263681592039801,
      "acc_norm_stderr": 0.03115715086935557
    },
    "hendrycksTest-astronomy": {
      "acc": 0.18421052631578946,
      "acc_stderr": 0.0315469804508223,
      "acc_norm": 0.23026315789473684,
      "acc_norm_stderr": 0.03426059424403165
    },
    "crows_pairs_english_age": {
      "likelihood_difference": 3.64010989010989,
      "likelihood_difference_stderr": 0.6494737736897132,
      "pct_stereotype": 0.6043956043956044,
      "pct_stereotype_stderr": 0.05154303032773001
    },
    "logiqa": {
      "acc": 0.2227342549923195,
      "acc_stderr": 0.016320054046165128,
      "acc_norm": 0.2519201228878648,
      "acc_norm_stderr": 0.017027415657021122
    },
    "crows_pairs_french": {
      "likelihood_difference": 10.133110651460942,
      "likelihood_difference_stderr": 0.23305057763604026,
      "pct_stereotype": 0.5915324985092427,
      "pct_stereotype_stderr": 0.012006904380989914
    },
    "hendrycksTest-high_school_microeconomics": {
      "acc": 0.18487394957983194,
      "acc_stderr": 0.025215992877954205,
      "acc_norm": 0.2647058823529412,
      "acc_norm_stderr": 0.028657491285071973
    },
    "crows_pairs_french_sexual_orientation": {
      "likelihood_difference": 14.914835164835164,
      "likelihood_difference_stderr": 1.0458321095115632,
      "pct_stereotype": 0.7912087912087912,
      "pct_stereotype_stderr": 0.04284305206509431
    },
    "hendrycksTest-public_relations": {
      "acc": 0.2545454545454545,
      "acc_stderr": 0.041723430387053825,
      "acc_norm": 0.2,
      "acc_norm_stderr": 0.03831305140884603
    },
    "hendrycksTest-clinical_knowledge": {
      "acc": 0.18490566037735848,
      "acc_stderr": 0.02389335183446432,
      "acc_norm": 0.29056603773584905,
      "acc_norm_stderr": 0.027943219989337142
    },
    "crows_pairs_french_gender": {
      "likelihood_difference": 7.616043613707165,
      "likelihood_difference_stderr": 0.453671327786389,
      "pct_stereotype": 0.5482866043613707,
      "pct_stereotype_stderr": 0.027820204204815787
    },
    "arc_challenge": {
      "acc": 0.2090443686006826,
      "acc_stderr": 0.011882746987406457,
      "acc_norm": 0.24829351535836178,
      "acc_norm_stderr": 0.01262491286808977
    },
    "hendrycksTest-college_physics": {
      "acc": 0.21568627450980393,
      "acc_stderr": 0.04092563958237656,
      "acc_norm": 0.22549019607843138,
      "acc_norm_stderr": 0.04158307533083286
    },
    "hendrycksTest-world_religions": {
      "acc": 0.19883040935672514,
      "acc_stderr": 0.030611116557432535,
      "acc_norm": 0.25146198830409355,
      "acc_norm_stderr": 0.033275044238468436
    },
    "crows_pairs_french_socioeconomic": {
      "likelihood_difference": 12.976283482142858,
      "likelihood_difference_stderr": 0.8478950106991682,
      "pct_stereotype": 0.4387755102040816,
      "pct_stereotype_stderr": 0.03553629865790393
    },
    "crows_pairs_french_autre": {
      "likelihood_difference": 7.144230769230769,
      "likelihood_difference_stderr": 2.005306269861725,
      "pct_stereotype": 0.3076923076923077,
      "pct_stereotype_stderr": 0.13323467750529824
    },
    "hendrycksTest-logical_fallacies": {
      "acc": 0.15337423312883436,
      "acc_stderr": 0.02831160144143859,
      "acc_norm": 0.22699386503067484,
      "acc_norm_stderr": 0.03291099578615769
    },
    "crows_pairs_english_gender": {
      "likelihood_difference": 4.92646484375,
      "likelihood_difference_stderr": 0.4975177732945778,
      "pct_stereotype": 0.503125,
      "pct_stereotype_stderr": 0.027994078772422815
    },
    "hendrycksTest-international_law": {
      "acc": 0.10743801652892562,
      "acc_stderr": 0.028268812192540637,
      "acc_norm": 0.23140495867768596,
      "acc_norm_stderr": 0.038498560987940904
    },
    "crows_pairs_english_nationality": {
      "likelihood_difference": 6.327112268518518,
      "likelihood_difference_stderr": 0.5322781195091362,
      "pct_stereotype": 0.4444444444444444,
      "pct_stereotype_stderr": 0.03388857118502325
    },
    "hendrycksTest-virology": {
      "acc": 0.16265060240963855,
      "acc_stderr": 0.0287302378926138,
      "acc_norm": 0.20481927710843373,
      "acc_norm_stderr": 0.03141784291663926
    },
    "hendrycksTest-college_biology": {
      "acc": 0.2569444444444444,
      "acc_stderr": 0.03653946969442099,
      "acc_norm": 0.2708333333333333,
      "acc_norm_stderr": 0.03716177437566017
    },
    "hendrycksTest-machine_learning": {
      "acc": 0.25,
      "acc_stderr": 0.04109974682633932,
      "acc_norm": 0.30357142857142855,
      "acc_norm_stderr": 0.04364226155841044
    },
    "hendrycksTest-high_school_physics": {
      "acc": 0.1986754966887417,
      "acc_stderr": 0.03257847384436777,
      "acc_norm": 0.31788079470198677,
      "acc_norm_stderr": 0.038020397601079024
    },
    "hendrycksTest-high_school_computer_science": {
      "acc": 0.23,
      "acc_stderr": 0.042295258468165065,
      "acc_norm": 0.34,
      "acc_norm_stderr": 0.047609522856952344
    },
    "hendrycksTest-professional_accounting": {
      "acc": 0.29432624113475175,
      "acc_stderr": 0.027187127011503803,
      "acc_norm": 0.2695035460992908,
      "acc_norm_stderr": 0.026469036818590634
    },
    "crows_pairs_french_religion": {
      "likelihood_difference": 12.339673913043478,
      "likelihood_difference_stderr": 0.968029517405933,
      "pct_stereotype": 0.6347826086956522,
      "pct_stereotype_stderr": 0.04509577025262067
    },
    "lambada_openai": {
      "ppl": 3526086.6936456356,
      "ppl_stderr": 340136.02871194785,
      "acc": 0.0,
      "acc_stderr": 0.0
    },
    "hendrycksTest-conceptual_physics": {
      "acc": 0.2851063829787234,
      "acc_stderr": 0.029513196625539355,
      "acc_norm": 0.2553191489361702,
      "acc_norm_stderr": 0.02850485647051418
    },
    "hendrycksTest-anatomy": {
      "acc": 0.23703703703703705,
      "acc_stderr": 0.03673731683969506,
      "acc_norm": 0.2962962962962963,
      "acc_norm_stderr": 0.03944624162501117
    },
    "arc_easy": {
      "acc": 0.26262626262626265,
      "acc_stderr": 0.009029861776763754,
      "acc_norm": 0.2516835016835017,
      "acc_norm_stderr": 0.008905088235948768
    },
    "hendrycksTest-security_studies": {
      "acc": 0.3183673469387755,
      "acc_stderr": 0.02982253379398205,
      "acc_norm": 0.19591836734693877,
      "acc_norm_stderr": 0.025409301953225678
    },
    "wsc": {
      "acc": 0.6346153846153846,
      "acc_stderr": 0.0474473339327792
    },
    "hendrycksTest-professional_psychology": {
      "acc": 0.2173202614379085,
      "acc_stderr": 0.016684820929148622,
      "acc_norm": 0.23039215686274508,
      "acc_norm_stderr": 0.017035229258034044
    },
    "crows_pairs_english_race_color": {
      "likelihood_difference": 5.796136811023622,
      "likelihood_difference_stderr": 0.35545568796836857,
      "pct_stereotype": 0.44881889763779526,
      "pct_stereotype_stderr": 0.02208913692163594
    },
    "hendrycksTest-management": {
      "acc": 0.21359223300970873,
      "acc_stderr": 0.04058042015646033,
      "acc_norm": 0.27184466019417475,
      "acc_norm_stderr": 0.044052680241409216
    },
    "hendrycksTest-nutrition": {
      "acc": 0.1830065359477124,
      "acc_stderr": 0.02214076751288095,
      "acc_norm": 0.25163398692810457,
      "acc_norm_stderr": 0.0248480182638752
    },
    "hendrycksTest-moral_scenarios": {
      "acc": 0.23798882681564246,
      "acc_stderr": 0.014242630070574915,
      "acc_norm": 0.27262569832402234,
      "acc_norm_stderr": 0.014893391735249588
    }
  },
  "versions": {
    "crows_pairs_english_religion": 0,
    "hendrycksTest-jurisprudence": 0,
    "hendrycksTest-high_school_mathematics": 0,
    "hendrycksTest-high_school_government_and_politics": 0,
    "hendrycksTest-human_sexuality": 0,
    "crows_pairs_english_socioeconomic": 0,
    "hendrycksTest-professional_medicine": 0,
    "hendrycksTest-elementary_mathematics": 0,
    "hendrycksTest-high_school_psychology": 0,
    "crows_pairs_french_disability": 0,
    "hendrycksTest-high_school_geography": 0,
    "hendrycksTest-college_computer_science": 0,
    "hendrycksTest-us_foreign_policy": 0,
    "hendrycksTest-computer_security": 0,
    "hendrycksTest-abstract_algebra": 0,
    "hendrycksTest-formal_logic": 0,
    "hendrycksTest-moral_disputes": 0,
    "hendrycksTest-professional_law": 0,
    "hendrycksTest-prehistory": 0,
    "winogrande": 0,
    "hendrycksTest-college_mathematics": 0,
    "hendrycksTest-miscellaneous": 0,
    "hendrycksTest-high_school_statistics": 0,
    "hendrycksTest-business_ethics": 0,
    "sciq": 0,
    "hendrycksTest-marketing": 0,
    "hendrycksTest-electrical_engineering": 0,
    "hendrycksTest-global_facts": 0,
    "crows_pairs_english_physical_appearance": 0,
    "crows_pairs_french_nationality": 0,
    "hendrycksTest-college_chemistry": 0,
    "crows_pairs_english_autre": 0,
    "crows_pairs_french_age": 0,
    "crows_pairs_english_disability": 0,
    "crows_pairs_french_race_color": 0,
    "hendrycksTest-high_school_macroeconomics": 0,
    "hendrycksTest-high_school_world_history": 0,
    "crows_pairs_english_sexual_orientation": 0,
    "hendrycksTest-college_medicine": 0,
    "hendrycksTest-high_school_european_history": 0,
    "hendrycksTest-philosophy": 0,
    "crows_pairs_french_physical_appearance": 0,
    "crows_pairs_english": 0,
    "hendrycksTest-high_school_chemistry": 0,
    "hendrycksTest-medical_genetics": 0,
    "hendrycksTest-high_school_us_history": 0,
    "hendrycksTest-high_school_biology": 0,
    "piqa": 0,
    "hendrycksTest-human_aging": 0,
    "hendrycksTest-econometrics": 0,
    "hendrycksTest-sociology": 0,
    "hendrycksTest-astronomy": 0,
    "crows_pairs_english_age": 0,
    "logiqa": 0,
    "crows_pairs_french": 0,
    "hendrycksTest-high_school_microeconomics": 0,
    "crows_pairs_french_sexual_orientation": 0,
    "hendrycksTest-public_relations": 0,
    "hendrycksTest-clinical_knowledge": 0,
    "crows_pairs_french_gender": 0,
    "arc_challenge": 0,
    "hendrycksTest-college_physics": 0,
    "hendrycksTest-world_religions": 0,
    "crows_pairs_french_socioeconomic": 0,
    "crows_pairs_french_autre": 0,
    "hendrycksTest-logical_fallacies": 0,
    "crows_pairs_english_gender": 0,
    "hendrycksTest-international_law": 0,
    "crows_pairs_english_nationality": 0,
    "hendrycksTest-virology": 0,
    "hendrycksTest-college_biology": 0,
    "hendrycksTest-machine_learning": 0,
    "hendrycksTest-high_school_physics": 0,
    "hendrycksTest-high_school_computer_science": 0,
    "hendrycksTest-professional_accounting": 0,
    "crows_pairs_french_religion": 0,
    "lambada_openai": 0,
    "hendrycksTest-conceptual_physics": 0,
    "hendrycksTest-anatomy": 0,
    "arc_easy": 0,
    "hendrycksTest-security_studies": 0,
    "wsc": 0,
    "hendrycksTest-professional_psychology": 0,
    "crows_pairs_english_race_color": 0,
    "hendrycksTest-management": 0,
    "hendrycksTest-nutrition": 0,
    "hendrycksTest-moral_scenarios": 0
  },
  "config": {
    "model": "hf-causal",
    "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step16",
    "num_fewshot": 0,
    "batch_size": 32,
    "device": null,
    "no_cache": true,
    "limit": null,
    "bootstrap_iters": 100000,
    "description_dict": {}
  }
}