quantized-debug-models / results /llama-3.1-8b-instruct-q3_k_m /piqa-0 /skymizer__Llama-3.1-8B-Instruct-GGUF /results_2025-08-28T15-17-14.136330.json
| { | |
| "results": { | |
| "piqa": { | |
| "alias": "piqa", | |
| "acc,none": 0.7976060935799782, | |
| "acc_stderr,none": 0.009374289682807648, | |
| "acc_norm,none": 0.794885745375408, | |
| "acc_norm_stderr,none": 0.009420971671018023 | |
| } | |
| }, | |
| "group_subtasks": { | |
| "piqa": [] | |
| }, | |
| "configs": { | |
| "piqa": { | |
| "task": "piqa", | |
| "dataset_path": "baber/piqa", | |
| "dataset_kwargs": { | |
| "trust_remote_code": true | |
| }, | |
| "training_split": "train", | |
| "validation_split": "validation", | |
| "doc_to_text": "Question: {{goal}}\nAnswer:", | |
| "doc_to_target": "label", | |
| "unsafe_code": false, | |
| "doc_to_choice": "{{[sol1, sol2]}}", | |
| "description": "", | |
| "target_delimiter": " ", | |
| "fewshot_delimiter": "\n\n", | |
| "num_fewshot": 0, | |
| "metric_list": [ | |
| { | |
| "metric": "acc", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| }, | |
| { | |
| "metric": "acc_norm", | |
| "aggregation": "mean", | |
| "higher_is_better": true | |
| } | |
| ], | |
| "output_type": "multiple_choice", | |
| "repeats": 1, | |
| "should_decontaminate": true, | |
| "doc_to_decontamination_query": "goal", | |
| "metadata": { | |
| "version": 1.0, | |
| "pretrained": "skymizer/Llama-3.1-8B-Instruct-GGUF", | |
| "gguf_file": "llama-3.1-8b-instruct-q3_k_m.gguf", | |
| "tokenizer": "meta-llama/Meta-Llama-3.1-8B-Instruct" | |
| } | |
| } | |
| }, | |
| "versions": { | |
| "piqa": 1.0 | |
| }, | |
| "n-shot": { | |
| "piqa": 0 | |
| }, | |
| "higher_is_better": { | |
| "piqa": { | |
| "acc": true, | |
| "acc_norm": true | |
| } | |
| }, | |
| "n-samples": { | |
| "piqa": { | |
| "original": 1838, | |
| "effective": 1838 | |
| } | |
| }, | |
| "config": { | |
| "model": "hf", | |
| "model_args": "pretrained=skymizer/Llama-3.1-8B-Instruct-GGUF,gguf_file=llama-3.1-8b-instruct-q3_k_m.gguf,tokenizer=meta-llama/Meta-Llama-3.1-8B-Instruct", | |
| "model_num_parameters": 8030261248, | |
| "model_dtype": "torch.float32", | |
| "model_revision": "main", | |
| "model_sha": "73c4e4d5ac2f0b4554477740ce9621999127f12f", | |
| "batch_size": "auto:4", | |
| "batch_sizes": [ | |
| 64, | |
| 64, | |
| 64, | |
| 64, | |
| 64 | |
| ], | |
| "device": null, | |
| "use_cache": null, | |
| "limit": null, | |
| "bootstrap_iters": 100000, | |
| "gen_kwargs": null, | |
| "random_seed": 0, | |
| "numpy_seed": 1234, | |
| "torch_seed": 1234, | |
| "fewshot_seed": 1234 | |
| }, | |
| "git_hash": "v0.1.1", | |
| "date": 1756394025.9041305, | |
| "pretty_env_info": "'NoneType' object has no attribute 'splitlines'", | |
| "transformers_version": "4.55.4", | |
| "lm_eval_version": "0.4.8", | |
| "upper_git_hash": null, | |
| "tokenizer_pad_token": [ | |
| "<|eot_id|>", | |
| "128009" | |
| ], | |
| "tokenizer_eos_token": [ | |
| "<|eot_id|>", | |
| "128009" | |
| ], | |
| "tokenizer_bos_token": [ | |
| "<|begin_of_text|>", | |
| "128000" | |
| ], | |
| "eot_token_id": 128009, | |
| "max_length": 131072, | |
| "task_hashes": {}, | |
| "model_source": "hf", | |
| "model_name": "skymizer/Llama-3.1-8B-Instruct-GGUF", | |
| "model_name_sanitized": "skymizer__Llama-3.1-8B-Instruct-GGUF", | |
| "system_instruction": null, | |
| "system_instruction_sha": null, | |
| "fewshot_as_multiturn": true, | |
| "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n {%- set system_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n{%- else %}\n {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n {#- Extract the first user message so we can plug it in here #}\n {%- if messages | length != 0 %}\n {%- set first_user_message = messages[0]['content']|trim %}\n {%- set messages = messages[1:] %}\n {%- else %}\n {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n {{- \"Do not use variables.\\n\\n\" }}\n {%- for t in tools %}\n {{- t | tojson(indent=4) }}\n {{- \"\\n\\n\" }}\n {%- endfor %}\n {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n {%- elif 'tool_calls' in message %}\n {%- if not message.tool_calls|length == 1 %}\n {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n {%- endif %}\n {%- set tool_call = message.tool_calls[0].function %}\n {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n {%- for arg_name, arg_val in tool_call.arguments | items %}\n {{- arg_name + '=\"' + arg_val + '\"' }}\n {%- if not loop.last %}\n {{- \", \" }}\n {%- endif %}\n {%- endfor %}\n {{- \")\" }}\n {%- else %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n {{- '\"parameters\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- \"}\" }}\n {%- endif %}\n {%- if builtin_tools is defined %}\n {#- This means we're in ipython mode #}\n {{- \"<|eom_id|>\" }}\n {%- else %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n {%- if message.content is mapping or message.content is iterable %}\n {{- message.content | tojson }}\n {%- else %}\n {{- message.content }}\n {%- endif %}\n {{- \"<|eot_id|>\" }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n", | |
| "chat_template_sha": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65", | |
| "start_time": 6739723.437865958, | |
| "end_time": 6740024.257117974, | |
| "total_evaluation_time_seconds": "300.8192520160228" | |
| } |