{
  "results": {
    "hendrycksTest-college_mathematics": {
      "acc": 0.18,
      "acc_stderr": 0.03861229196653695,
      "acc_norm": 0.17,
      "acc_norm_stderr": 0.03775251680686371
    },
    "crows_pairs_english_physical_appearance": {
      "likelihood_difference": 6.254340277777778,
      "likelihood_difference_stderr": 0.84909722861177,
      "pct_stereotype": 0.5416666666666666,
      "pct_stereotype_stderr": 0.05913268547421811
    },
    "hendrycksTest-high_school_psychology": {
      "acc": 0.25504587155963304,
      "acc_stderr": 0.018688500856535853,
      "acc_norm": 0.24587155963302754,
      "acc_norm_stderr": 0.018461940968708457
    },
    "hendrycksTest-security_studies": {
      "acc": 0.3142857142857143,
      "acc_stderr": 0.02971932942241746,
      "acc_norm": 0.19183673469387755,
      "acc_norm_stderr": 0.025206963154225378
    },
    "hendrycksTest-professional_psychology": {
      "acc": 0.22058823529411764,
      "acc_stderr": 0.01677467236546854,
      "acc_norm": 0.23366013071895425,
      "acc_norm_stderr": 0.017119158496044503
    },
    "hendrycksTest-clinical_knowledge": {
      "acc": 0.18867924528301888,
      "acc_stderr": 0.024079995130062207,
      "acc_norm": 0.2792452830188679,
      "acc_norm_stderr": 0.027611163402399715
    },
    "arc_easy": {
      "acc": 0.2718855218855219,
      "acc_stderr": 0.009129795867310499,
      "acc_norm": 0.2516835016835017,
      "acc_norm_stderr": 0.008905088235948768
    },
    "hendrycksTest-public_relations": {
      "acc": 0.2545454545454545,
      "acc_stderr": 0.041723430387053825,
      "acc_norm": 0.18181818181818182,
      "acc_norm_stderr": 0.03694284335337798
    },
    "hendrycksTest-human_aging": {
      "acc": 0.28699551569506726,
      "acc_stderr": 0.030360379710291936,
      "acc_norm": 0.273542600896861,
      "acc_norm_stderr": 0.029918586707798817
    },
    "hendrycksTest-machine_learning": {
      "acc": 0.3125,
      "acc_stderr": 0.043994650575715215,
      "acc_norm": 0.30357142857142855,
      "acc_norm_stderr": 0.04364226155841044
    },
    "hendrycksTest-high_school_government_and_politics": {
      "acc": 0.21761658031088082,
      "acc_stderr": 0.029778663037752964,
      "acc_norm": 0.24352331606217617,
      "acc_norm_stderr": 0.030975436386845436
    },
    "crows_pairs_french_race_color": {
      "likelihood_difference": 8.875135869565218,
      "likelihood_difference_stderr": 0.37474334360732336,
      "pct_stereotype": 0.6434782608695652,
      "pct_stereotype_stderr": 0.022356489247084357
    },
    "wsc": {
      "acc": 0.5961538461538461,
      "acc_stderr": 0.048346889526540184
    },
    "piqa": {
      "acc": 0.5255712731229597,
      "acc_stderr": 0.011650557844573573,
      "acc_norm": 0.5217627856365615,
      "acc_norm_stderr": 0.011654768618560074
    },
    "hendrycksTest-professional_law": {
      "acc": 0.23272490221642764,
      "acc_stderr": 0.010792595553888486,
      "acc_norm": 0.24315514993481094,
      "acc_norm_stderr": 0.010956556654417358
    },
    "sciq": {
      "acc": 0.194,
      "acc_stderr": 0.012510816141264368,
      "acc_norm": 0.216,
      "acc_norm_stderr": 0.013019735539307789
    },
    "logiqa": {
      "acc": 0.22887864823348694,
      "acc_stderr": 0.01647810727631327,
      "acc_norm": 0.24731182795698925,
      "acc_norm_stderr": 0.016922842446712393
    },
    "hendrycksTest-high_school_macroeconomics": {
      "acc": 0.21794871794871795,
      "acc_stderr": 0.020932445774463196,
      "acc_norm": 0.24358974358974358,
      "acc_norm_stderr": 0.02176373368417392
    },
    "hendrycksTest-formal_logic": {
      "acc": 0.24603174603174602,
      "acc_stderr": 0.03852273364924318,
      "acc_norm": 0.23809523809523808,
      "acc_norm_stderr": 0.03809523809523809
    },
    "hendrycksTest-miscellaneous": {
      "acc": 0.2503192848020434,
      "acc_stderr": 0.01549108895149458,
      "acc_norm": 0.26436781609195403,
      "acc_norm_stderr": 0.015769984840690518
    },
    "crows_pairs_french": {
      "likelihood_difference": 10.008921250745379,
      "likelihood_difference_stderr": 0.23452315861799014,
      "pct_stereotype": 0.569469290399523,
      "pct_stereotype_stderr": 0.012094842430543595
    },
    "hendrycksTest-medical_genetics": {
      "acc": 0.23,
      "acc_stderr": 0.04229525846816505,
      "acc_norm": 0.24,
      "acc_norm_stderr": 0.04292346959909283
    },
    "hendrycksTest-world_religions": {
      "acc": 0.19298245614035087,
      "acc_stderr": 0.03026745755489847,
      "acc_norm": 0.23391812865497075,
      "acc_norm_stderr": 0.03246721765117825
    },
    "crows_pairs_english": {
      "likelihood_difference": 5.816375968992248,
      "likelihood_difference_stderr": 0.20056629856250444,
      "pct_stereotype": 0.4561717352415027,
      "pct_stereotype_stderr": 0.012166287275376293
    },
    "hendrycksTest-elementary_mathematics": {
      "acc": 0.2275132275132275,
      "acc_stderr": 0.021591269407823785,
      "acc_norm": 0.23544973544973544,
      "acc_norm_stderr": 0.021851509822031715
    },
    "arc_challenge": {
      "acc": 0.21416382252559726,
      "acc_stderr": 0.011988383205966508,
      "acc_norm": 0.24658703071672355,
      "acc_norm_stderr": 0.01259572626879014
    },
    "crows_pairs_english_age": {
      "likelihood_difference": 3.8372252747252746,
      "likelihood_difference_stderr": 0.6727209631618413,
      "pct_stereotype": 0.6593406593406593,
      "pct_stereotype_stderr": 0.049956709512768704
    },
    "hendrycksTest-high_school_statistics": {
      "acc": 0.25462962962962965,
      "acc_stderr": 0.029711275860005354,
      "acc_norm": 0.2916666666666667,
      "acc_norm_stderr": 0.030998666304560534
    },
    "crows_pairs_french_socioeconomic": {
      "likelihood_difference": 13.08430325255102,
      "likelihood_difference_stderr": 0.865918639747448,
      "pct_stereotype": 0.40816326530612246,
      "pct_stereotype_stderr": 0.03519659177561531
    },
    "hendrycksTest-high_school_geography": {
      "acc": 0.23737373737373738,
      "acc_stderr": 0.03031371053819888,
      "acc_norm": 0.2727272727272727,
      "acc_norm_stderr": 0.03173071239071724
    },
    "hendrycksTest-high_school_us_history": {
      "acc": 0.18627450980392157,
      "acc_stderr": 0.027325470966716326,
      "acc_norm": 0.2647058823529412,
      "acc_norm_stderr": 0.030964517926923393
    },
    "hendrycksTest-philosophy": {
      "acc": 0.22508038585209003,
      "acc_stderr": 0.023720088516179027,
      "acc_norm": 0.2958199356913183,
      "acc_norm_stderr": 0.025922371788818784
    },
    "hendrycksTest-professional_medicine": {
      "acc": 0.24632352941176472,
      "acc_stderr": 0.02617343857052,
      "acc_norm": 0.27941176470588236,
      "acc_norm_stderr": 0.02725720260611495
    },
    "crows_pairs_english_religion": {
      "likelihood_difference": 5.912443693693693,
      "likelihood_difference_stderr": 0.7158104771703969,
      "pct_stereotype": 0.5225225225225225,
      "pct_stereotype_stderr": 0.04762473917649626
    },
    "hendrycksTest-astronomy": {
      "acc": 0.18421052631578946,
      "acc_stderr": 0.031546980450822305,
      "acc_norm": 0.23684210526315788,
      "acc_norm_stderr": 0.03459777606810534
    },
    "hendrycksTest-international_law": {
      "acc": 0.08264462809917356,
      "acc_stderr": 0.02513538235660422,
      "acc_norm": 0.2066115702479339,
      "acc_norm_stderr": 0.03695980128098824
    },
    "hendrycksTest-high_school_physics": {
      "acc": 0.18543046357615894,
      "acc_stderr": 0.031732843842942865,
      "acc_norm": 0.31125827814569534,
      "acc_norm_stderr": 0.03780445850526733
    },
    "hendrycksTest-management": {
      "acc": 0.2524271844660194,
      "acc_stderr": 0.04301250399690879,
      "acc_norm": 0.2912621359223301,
      "acc_norm_stderr": 0.044986763205729245
    },
    "hendrycksTest-prehistory": {
      "acc": 0.23148148148148148,
      "acc_stderr": 0.02346842983245115,
      "acc_norm": 0.20679012345679013,
      "acc_norm_stderr": 0.022535006705942818
    },
    "hendrycksTest-anatomy": {
      "acc": 0.21481481481481482,
      "acc_stderr": 0.03547854198560823,
      "acc_norm": 0.2962962962962963,
      "acc_norm_stderr": 0.03944624162501117
    },
    "hendrycksTest-high_school_european_history": {
      "acc": 0.17575757575757575,
      "acc_stderr": 0.02972094300622445,
      "acc_norm": 0.18181818181818182,
      "acc_norm_stderr": 0.030117688929503585
    },
    "crows_pairs_french_physical_appearance": {
      "likelihood_difference": 10.157118055555555,
      "likelihood_difference_stderr": 1.3610391208223753,
      "pct_stereotype": 0.5138888888888888,
      "pct_stereotype_stderr": 0.05931618532716555
    },
    "crows_pairs_english_race_color": {
      "likelihood_difference": 5.949495570866142,
      "likelihood_difference_stderr": 0.36479478018512124,
      "pct_stereotype": 0.36023622047244097,
      "pct_stereotype_stderr": 0.02132060545173939
    },
    "hendrycksTest-high_school_world_history": {
      "acc": 0.17721518987341772,
      "acc_stderr": 0.024856364184503217,
      "acc_norm": 0.21518987341772153,
      "acc_norm_stderr": 0.02675082699467616
    },
    "crows_pairs_french_gender": {
      "likelihood_difference": 7.704244548286605,
      "likelihood_difference_stderr": 0.45459318084053474,
      "pct_stereotype": 0.5295950155763239,
      "pct_stereotype_stderr": 0.027901844420051173
    },
    "hendrycksTest-high_school_microeconomics": {
      "acc": 0.18907563025210083,
      "acc_stderr": 0.02543511943810536,
      "acc_norm": 0.2815126050420168,
      "acc_norm_stderr": 0.029213549414372163
    },
    "hendrycksTest-jurisprudence": {
      "acc": 0.1574074074074074,
      "acc_stderr": 0.03520703990517965,
      "acc_norm": 0.25925925925925924,
      "acc_norm_stderr": 0.042365112580946315
    },
    "hendrycksTest-nutrition": {
      "acc": 0.17973856209150327,
      "acc_stderr": 0.021986032182064148,
      "acc_norm": 0.238562091503268,
      "acc_norm_stderr": 0.024404394928087877
    },
    "hendrycksTest-abstract_algebra": {
      "acc": 0.26,
      "acc_stderr": 0.04408440022768078,
      "acc_norm": 0.23,
      "acc_norm_stderr": 0.04229525846816506
    },
    "hendrycksTest-high_school_computer_science": {
      "acc": 0.24,
      "acc_stderr": 0.042923469599092816,
      "acc_norm": 0.33,
      "acc_norm_stderr": 0.04725815626252606
    },
    "crows_pairs_french_disability": {
      "likelihood_difference": 14.821022727272727,
      "likelihood_difference_stderr": 1.4754485302166234,
      "pct_stereotype": 0.3484848484848485,
      "pct_stereotype_stderr": 0.0591013677911929
    },
    "hendrycksTest-sociology": {
      "acc": 0.25870646766169153,
      "acc_stderr": 0.030965903123573044,
      "acc_norm": 0.2736318407960199,
      "acc_norm_stderr": 0.03152439186555402
    },
    "hendrycksTest-college_physics": {
      "acc": 0.23529411764705882,
      "acc_stderr": 0.04220773659171452,
      "acc_norm": 0.22549019607843138,
      "acc_norm_stderr": 0.04158307533083286
    },
    "hendrycksTest-high_school_chemistry": {
      "acc": 0.2315270935960591,
      "acc_stderr": 0.029678333141444437,
      "acc_norm": 0.22660098522167488,
      "acc_norm_stderr": 0.029454863835292975
    },
    "winogrande": {
      "acc": 0.4988161010260458,
      "acc_stderr": 0.014052446290529012
    },
    "hendrycksTest-college_medicine": {
      "acc": 0.20809248554913296,
      "acc_stderr": 0.03095289021774988,
      "acc_norm": 0.2658959537572254,
      "acc_norm_stderr": 0.03368762932259431
    },
    "hendrycksTest-global_facts": {
      "acc": 0.26,
      "acc_stderr": 0.04408440022768078,
      "acc_norm": 0.27,
      "acc_norm_stderr": 0.044619604333847394
    },
    "hendrycksTest-moral_disputes": {
      "acc": 0.23410404624277456,
      "acc_stderr": 0.022797110278071138,
      "acc_norm": 0.21098265895953758,
      "acc_norm_stderr": 0.021966309947043117
    },
    "crows_pairs_english_disability": {
      "likelihood_difference": 7.943269230769231,
      "likelihood_difference_stderr": 1.246261479003386,
      "pct_stereotype": 0.5384615384615384,
      "pct_stereotype_stderr": 0.06231481440776789
    },
    "hendrycksTest-moral_scenarios": {
      "acc": 0.23798882681564246,
      "acc_stderr": 0.014242630070574915,
      "acc_norm": 0.27262569832402234,
      "acc_norm_stderr": 0.014893391735249588
    },
    "crows_pairs_english_socioeconomic": {
      "likelihood_difference": 6.05641447368421,
      "likelihood_difference_stderr": 0.5511819524754867,
      "pct_stereotype": 0.46842105263157896,
      "pct_stereotype_stderr": 0.03629703808831611
    },
    "crows_pairs_english_nationality": {
      "likelihood_difference": 6.445746527777778,
      "likelihood_difference_stderr": 0.5390301887250009,
      "pct_stereotype": 0.44907407407407407,
      "pct_stereotype_stderr": 0.03392238405321617
    },
    "crows_pairs_english_autre": {
      "likelihood_difference": 6.295454545454546,
      "likelihood_difference_stderr": 3.1838670351509126,
      "pct_stereotype": 0.45454545454545453,
      "pct_stereotype_stderr": 0.15745916432444335
    },
    "hendrycksTest-college_chemistry": {
      "acc": 0.17,
      "acc_stderr": 0.0377525168068637,
      "acc_norm": 0.19,
      "acc_norm_stderr": 0.03942772444036623
    },
    "crows_pairs_english_gender": {
      "likelihood_difference": 5.0119140625,
      "likelihood_difference_stderr": 0.5134294888398998,
      "pct_stereotype": 0.49375,
      "pct_stereotype_stderr": 0.027992438382232313
    },
    "hendrycksTest-logical_fallacies": {
      "acc": 0.1656441717791411,
      "acc_stderr": 0.029208296231259104,
      "acc_norm": 0.2392638036809816,
      "acc_norm_stderr": 0.033519538795212696
    },
    "crows_pairs_french_autre": {
      "likelihood_difference": 6.980769230769231,
      "likelihood_difference_stderr": 1.9247068413265722,
      "pct_stereotype": 0.38461538461538464,
      "pct_stereotype_stderr": 0.1404416814115811
    },
    "hendrycksTest-virology": {
      "acc": 0.1927710843373494,
      "acc_stderr": 0.030709824050565274,
      "acc_norm": 0.22289156626506024,
      "acc_norm_stderr": 0.03240004825594688
    },
    "hendrycksTest-high_school_biology": {
      "acc": 0.20967741935483872,
      "acc_stderr": 0.02315787934908352,
      "acc_norm": 0.22258064516129034,
      "acc_norm_stderr": 0.02366421667164251
    },
    "crows_pairs_french_sexual_orientation": {
      "likelihood_difference": 14.890796703296703,
      "likelihood_difference_stderr": 1.0225529329727918,
      "pct_stereotype": 0.8021978021978022,
      "pct_stereotype_stderr": 0.04198895203196222
    },
    "hendrycksTest-human_sexuality": {
      "acc": 0.2595419847328244,
      "acc_stderr": 0.03844876139785271,
      "acc_norm": 0.2595419847328244,
      "acc_norm_stderr": 0.03844876139785271
    },
    "crows_pairs_french_age": {
      "likelihood_difference": 5.868055555555555,
      "likelihood_difference_stderr": 0.7717131357071738,
      "pct_stereotype": 0.6444444444444445,
      "pct_stereotype_stderr": 0.05074011803597719
    },
    "lambada_openai": {
      "ppl": 3684074.4512471026,
      "ppl_stderr": 360109.5301043941,
      "acc": 0.0,
      "acc_stderr": 0.0
    },
    "hendrycksTest-high_school_mathematics": {
      "acc": 0.24074074074074073,
      "acc_stderr": 0.026067159222275794,
      "acc_norm": 0.2814814814814815,
      "acc_norm_stderr": 0.027420019350945263
    },
    "hendrycksTest-business_ethics": {
      "acc": 0.23,
      "acc_stderr": 0.04229525846816505,
      "acc_norm": 0.25,
      "acc_norm_stderr": 0.04351941398892446
    },
    "hendrycksTest-marketing": {
      "acc": 0.23504273504273504,
      "acc_stderr": 0.027778835904935437,
      "acc_norm": 0.2564102564102564,
      "acc_norm_stderr": 0.028605953702004243
    },
    "hendrycksTest-us_foreign_policy": {
      "acc": 0.22,
      "acc_stderr": 0.04163331998932269,
      "acc_norm": 0.24,
      "acc_norm_stderr": 0.04292346959909283
    },
    "hendrycksTest-econometrics": {
      "acc": 0.2543859649122807,
      "acc_stderr": 0.04096985139843669,
      "acc_norm": 0.30701754385964913,
      "acc_norm_stderr": 0.0433913832257986
    },
    "hendrycksTest-conceptual_physics": {
      "acc": 0.2553191489361702,
      "acc_stderr": 0.0285048564705142,
      "acc_norm": 0.25957446808510637,
      "acc_norm_stderr": 0.02865917937429232
    },
    "crows_pairs_english_sexual_orientation": {
      "likelihood_difference": 5.844758064516129,
      "likelihood_difference_stderr": 0.7849858741881288,
      "pct_stereotype": 0.44086021505376344,
      "pct_stereotype_stderr": 0.051762678118979284
    },
    "crows_pairs_french_nationality": {
      "likelihood_difference": 10.08498023715415,
      "likelihood_difference_stderr": 0.5574722426656213,
      "pct_stereotype": 0.5652173913043478,
      "pct_stereotype_stderr": 0.03122795678881643
    },
    "crows_pairs_french_religion": {
      "likelihood_difference": 12.433695652173913,
      "likelihood_difference_stderr": 0.9697744506204069,
      "pct_stereotype": 0.6086956521739131,
      "pct_stereotype_stderr": 0.045709346351117126
    },
    "hendrycksTest-college_biology": {
      "acc": 0.2638888888888889,
      "acc_stderr": 0.03685651095897532,
      "acc_norm": 0.2638888888888889,
      "acc_norm_stderr": 0.03685651095897532
    },
    "hendrycksTest-computer_security": {
      "acc": 0.25,
      "acc_stderr": 0.04351941398892446,
      "acc_norm": 0.3,
      "acc_norm_stderr": 0.046056618647183814
    },
    "hendrycksTest-electrical_engineering": {
      "acc": 0.19310344827586207,
      "acc_stderr": 0.03289445522127401,
      "acc_norm": 0.19310344827586207,
      "acc_norm_stderr": 0.03289445522127401
    },
    "hendrycksTest-college_computer_science": {
      "acc": 0.26,
      "acc_stderr": 0.04408440022768079,
      "acc_norm": 0.19,
      "acc_norm_stderr": 0.03942772444036624
    },
    "hendrycksTest-professional_accounting": {
      "acc": 0.2765957446808511,
      "acc_stderr": 0.026684564340460997,
      "acc_norm": 0.2624113475177305,
      "acc_norm_stderr": 0.026244920349843007
    }
  },
  "versions": {
    "hendrycksTest-college_mathematics": 0,
    "crows_pairs_english_physical_appearance": 0,
    "hendrycksTest-high_school_psychology": 0,
    "hendrycksTest-security_studies": 0,
    "hendrycksTest-professional_psychology": 0,
    "hendrycksTest-clinical_knowledge": 0,
    "arc_easy": 0,
    "hendrycksTest-public_relations": 0,
    "hendrycksTest-human_aging": 0,
    "hendrycksTest-machine_learning": 0,
    "hendrycksTest-high_school_government_and_politics": 0,
    "crows_pairs_french_race_color": 0,
    "wsc": 0,
    "piqa": 0,
    "hendrycksTest-professional_law": 0,
    "sciq": 0,
    "logiqa": 0,
    "hendrycksTest-high_school_macroeconomics": 0,
    "hendrycksTest-formal_logic": 0,
    "hendrycksTest-miscellaneous": 0,
    "crows_pairs_french": 0,
    "hendrycksTest-medical_genetics": 0,
    "hendrycksTest-world_religions": 0,
    "crows_pairs_english": 0,
    "hendrycksTest-elementary_mathematics": 0,
    "arc_challenge": 0,
    "crows_pairs_english_age": 0,
    "hendrycksTest-high_school_statistics": 0,
    "crows_pairs_french_socioeconomic": 0,
    "hendrycksTest-high_school_geography": 0,
    "hendrycksTest-high_school_us_history": 0,
    "hendrycksTest-philosophy": 0,
    "hendrycksTest-professional_medicine": 0,
    "crows_pairs_english_religion": 0,
    "hendrycksTest-astronomy": 0,
    "hendrycksTest-international_law": 0,
    "hendrycksTest-high_school_physics": 0,
    "hendrycksTest-management": 0,
    "hendrycksTest-prehistory": 0,
    "hendrycksTest-anatomy": 0,
    "hendrycksTest-high_school_european_history": 0,
    "crows_pairs_french_physical_appearance": 0,
    "crows_pairs_english_race_color": 0,
    "hendrycksTest-high_school_world_history": 0,
    "crows_pairs_french_gender": 0,
    "hendrycksTest-high_school_microeconomics": 0,
    "hendrycksTest-jurisprudence": 0,
    "hendrycksTest-nutrition": 0,
    "hendrycksTest-abstract_algebra": 0,
    "hendrycksTest-high_school_computer_science": 0,
    "crows_pairs_french_disability": 0,
    "hendrycksTest-sociology": 0,
    "hendrycksTest-college_physics": 0,
    "hendrycksTest-high_school_chemistry": 0,
    "winogrande": 0,
    "hendrycksTest-college_medicine": 0,
    "hendrycksTest-global_facts": 0,
    "hendrycksTest-moral_disputes": 0,
    "crows_pairs_english_disability": 0,
    "hendrycksTest-moral_scenarios": 0,
    "crows_pairs_english_socioeconomic": 0,
    "crows_pairs_english_nationality": 0,
    "crows_pairs_english_autre": 0,
    "hendrycksTest-college_chemistry": 0,
    "crows_pairs_english_gender": 0,
    "hendrycksTest-logical_fallacies": 0,
    "crows_pairs_french_autre": 0,
    "hendrycksTest-virology": 0,
    "hendrycksTest-high_school_biology": 0,
    "crows_pairs_french_sexual_orientation": 0,
    "hendrycksTest-human_sexuality": 0,
    "crows_pairs_french_age": 0,
    "lambada_openai": 0,
    "hendrycksTest-high_school_mathematics": 0,
    "hendrycksTest-business_ethics": 0,
    "hendrycksTest-marketing": 0,
    "hendrycksTest-us_foreign_policy": 0,
    "hendrycksTest-econometrics": 0,
    "hendrycksTest-conceptual_physics": 0,
    "crows_pairs_english_sexual_orientation": 0,
    "crows_pairs_french_nationality": 0,
    "crows_pairs_french_religion": 0,
    "hendrycksTest-college_biology": 0,
    "hendrycksTest-computer_security": 0,
    "hendrycksTest-electrical_engineering": 0,
    "hendrycksTest-college_computer_science": 0,
    "hendrycksTest-professional_accounting": 0
  },
  "config": {
    "model": "hf-causal",
    "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step1",
    "num_fewshot": 0,
    "batch_size": 32,
    "device": null,
    "no_cache": true,
    "limit": null,
    "bootstrap_iters": 100000,
    "description_dict": {}
  }
}