{ "results": { "hellaswag": { "name": "hellaswag", "alias": "hellaswag", "sample_len": 10042, "acc,none": 0.2577175861382195, "acc_stderr,none": 0.004364838000336011, "acc_norm,none": 0.2502489543915555, "acc_norm_stderr,none": 0.004322710911026276 }, "arc_easy": { "name": "arc_easy", "alias": "arc_easy", "sample_len": 2376, "acc,none": 0.2857744107744108, "acc_stderr,none": 0.009270380606981266, "acc_norm,none": 0.2975589225589226, "acc_norm_stderr,none": 0.009381226721815406 }, "arc_challenge": { "name": "arc_challenge", "alias": "arc_challenge", "sample_len": 1172, "acc,none": 0.17320819112627986, "acc_stderr,none": 0.011058694183280343, "acc_norm,none": 0.22696245733788395, "acc_norm_stderr,none": 0.012240491536132965 }, "piqa": { "name": "piqa", "alias": "piqa", "sample_len": 1838, "acc,none": 0.5315560391730142, "acc_stderr,none": 0.011642567710878958, "acc_norm,none": 0.5250272034820457, "acc_norm_stderr,none": 0.011651200967623562 } }, "group_subtasks": {}, "configs": { "arc_challenge": { "task": "arc_challenge", "dataset_path": "allenai/ai2_arc", "dataset_name": "ARC-Challenge", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_target": "{{choices.label.index(answerKey)}}", "unsafe_code": false, "doc_to_choice": "{{choices.text}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_choice": "{{choices.text}}", "doc_to_target": "{{choices.label.index(answerKey)}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", "metadata": { "version": 1.0, "config_source": "/Users/jonah/Documents/ChatGPT/Training/.venv/lib/python3.14/site-packages/lm_eval/tasks/arc/arc_challenge.yaml" } }, "arc_easy": { "task": "arc_easy", "dataset_path": "allenai/ai2_arc", "dataset_name": "ARC-Easy", "training_split": "train", "validation_split": "validation", "test_split": "test", "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_target": "{{choices.label.index(answerKey)}}", "unsafe_code": false, "doc_to_choice": "{{choices.text}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "Question: {{question}}\nAnswer:", "doc_to_choice": "{{choices.text}}", "doc_to_target": "{{choices.label.index(answerKey)}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "Question: {{question}}\nAnswer:", "metadata": { "version": 1.0, "config_source": "/Users/jonah/Documents/ChatGPT/Training/.venv/lib/python3.14/site-packages/lm_eval/tasks/arc/arc_easy.yaml" } }, "hellaswag": { "task": "hellaswag", "dataset_path": "Rowan/hellaswag", "training_split": "train", "validation_split": "validation", "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n def _process_doc(doc):\n ctx = doc[\"ctx_a\"] + \" \" + doc[\"ctx_b\"].capitalize()\n out_doc = {\n \"query\": preprocess(doc[\"activity_label\"] + \": \" + ctx),\n \"choices\": [preprocess(ending) for ending in doc[\"endings\"]],\n \"gold\": int(doc[\"label\"]),\n }\n return out_doc\n\n return dataset.map(_process_doc)\n", "doc_to_text": "{{query}}", "doc_to_target": "{{label}}", "unsafe_code": false, "doc_to_choice": "choices", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": "", "fewshot_indices": null, "samples": null, "doc_to_text": "{{query}}", "doc_to_choice": "choices", "doc_to_target": "{{label}}", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": false, "metadata": { "version": 1.0, "config_source": "/Users/jonah/Documents/ChatGPT/Training/.venv/lib/python3.14/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml" } }, "piqa": { "task": "piqa", "dataset_path": "baber/piqa", "training_split": "train", "validation_split": "validation", "doc_to_text": "Question: {{goal}}\nAnswer:", "doc_to_target": "label", "unsafe_code": false, "doc_to_choice": "{{[sol1, sol2]}}", "description": "", "target_delimiter": " ", "fewshot_delimiter": "\n\n", "fewshot_config": { "sampler": "default", "split": null, "process_docs": null, "fewshot_indices": null, "samples": null, "doc_to_text": "Question: {{goal}}\nAnswer:", "doc_to_choice": "{{[sol1, sol2]}}", "doc_to_target": "label", "gen_prefix": null, "fewshot_delimiter": "\n\n", "target_delimiter": " " }, "num_fewshot": 0, "metric_list": [ { "metric": "acc", "aggregation": "mean", "higher_is_better": true }, { "metric": "acc_norm", "aggregation": "mean", "higher_is_better": true } ], "output_type": "multiple_choice", "repeats": 1, "should_decontaminate": true, "doc_to_decontamination_query": "goal", "metadata": { "version": 1.0, "config_source": "/Users/jonah/Documents/ChatGPT/Training/.venv/lib/python3.14/site-packages/lm_eval/tasks/piqa/piqa.yaml" } } }, "versions": { "arc_challenge": 1.0, "arc_easy": 1.0, "hellaswag": 1.0, "piqa": 1.0 }, "n-shot": { "arc_challenge": 0, "arc_easy": 0, "hellaswag": 0, "piqa": 0 }, "higher_is_better": { "arc_challenge": { "acc": true, "acc_norm": true }, "arc_easy": { "acc": true, "acc_norm": true }, "hellaswag": { "acc": true, "acc_norm": true }, "piqa": { "acc": true, "acc_norm": true } }, "n-samples": { "hellaswag": { "original": 10042, "effective": 10042 }, "arc_easy": { "original": 2376, "effective": 2376 }, "arc_challenge": { "original": 1172, "effective": 1172 }, "piqa": { "original": 1838, "effective": 1838 } }, "config": { "model": "MLXCheckpointLM", "model_args": null, "batch_size": 2, "batch_sizes": [], "device": null, "use_cache": null, "limit": null, "bootstrap_iters": 100000, "gen_kwargs": null, "random_seed": 0, "numpy_seed": 1234, "torch_seed": 1234, "fewshot_seed": 1234 }, "git_hash": "6bdcd14", "date": 1789940047.395892, "pretty_env_info": "PyTorch version: 2.13.0\nIs debug build: False\nCUDA used to build PyTorch: None\nROCM used to build PyTorch: N/A\n\nOS: macOS 27.0 (arm64)\nGCC version: Could not collect\nClang version: 21.0.0 (clang-2100.1.1.101)\nCMake version: version 4.3.3\nLibc version: N/A\n\nPython version: 3.14.7 (main, Aug 5 2026, 10:29:49) [Clang 21.0.0 (clang-2100.1.1.101)] (64-bit runtime)\nPython platform: macOS-27.0-arm64-arm-64bit-Mach-O\nIs CUDA available: False\nCUDA runtime version: No CUDA\nCUDA_MODULE_LOADING set to: N/A\nGPU models and configuration: No CUDA\nNvidia driver version: No CUDA\ncuDNN version: No CUDA\nIs XPU available: False\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: False\nCaching allocator config: N/A\n\nCPU:\nApple M5 Max\n\nVersions of relevant libraries:\n[pip3] numpy==2.5.2\n[pip3] torch==2.13.0\n[conda] Could not collect", "transformers_version": "5.16.1", "lm_eval_version": "0.4.12", "upper_git_hash": null, "tokenizer_pad_token": [ null, "None" ], "tokenizer_eos_token": [ "<|eos|>", "1" ], "tokenizer_bos_token": [ null, "None" ], "eot_token_id": 1, "max_length": 2048, "xsa_evaluation": { "checkpoint": "/Users/jonah/Documents/ChatGPT/Training/runs/sol_milkshake_recovery_500m_v1/checkpoint_000900005888", "checkpoint_tokens_seen": null, "lm_eval_version": "0.4.12", "tasks": [ "hellaswag", "arc_easy", "arc_challenge", "piqa" ], "num_fewshot": 0, "metric": "acc_norm", "max_length": 2048, "batch_size": 2, "elapsed_seconds": 174.03599381446838 } }