{ "metadata": { "kernelspec": { "language": "python", "display_name": "Python 3", "name": "python3" }, "language_info": { "name": "python", "version": "3.12.13", "mimetype": "text/x-python", "codemirror_mode": { "name": "ipython", "version": 3 }, "pygments_lexer": "ipython3", "nbconvert_exporter": "python", "file_extension": ".py" }, "kaggle": { "accelerator": "nvidiaTeslaT4", "dataSources": [], "isInternetEnabled": true, "language": "python", "sourceType": "notebook", "isGpuEnabled": false } }, "nbformat_minor": 4, "nbformat": 4, "cells": [ { "cell_type": "code", "source": "# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n# Use the kagglehub client library to attach Kaggle resources like competitions, datasets, and models to your session\n# Learn more about kagglehub: https://github.com/Kaggle/kagglehub/blob/main/README.md\n\nimport kagglehub\n# kagglehub.dataset_download('/')", "metadata": { "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5", "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19", "trusted": true }, "outputs": [], "execution_count": null }, { "cell_type": "code", "source": "!pip uninstall -y torchao -q\n!pip install -q transformers accelerate datasets evalplus sentencepiece\nprint(\"Done\")", "metadata": { "trusted": true, "execution": { "iopub.status.busy": "2026-07-16T18:34:10.22367Z", "iopub.execute_input": "2026-07-16T18:34:10.223899Z", "iopub.status.idle": "2026-07-16T18:34:27.408138Z", "shell.execute_reply.started": "2026-07-16T18:34:10.223877Z", "shell.execute_reply": "2026-07-16T18:34:27.407041Z" } }, "outputs": [ { "name": "stdout", "text": " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n\u001b[2K \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m68.6/68.6 kB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n\u001b[2K \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m956.9/956.9 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n\u001b[2K \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m115.9/115.9 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[2K \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m667.5/667.5 kB\u001b[0m \u001b[31m35.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[2K \u001b[90m\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u2501\u001b[0m \u001b[32m108.1/108.1 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n\u001b[?25h Building wheel for stop-sequencer (setup.py) ... \u001b[?25l\u001b[?25hdone\n Building wheel for tempdir (setup.py) ... \u001b[?25l\u001b[?25hdone\n Building wheel for wget (setup.py) ... \u001b[?25l\u001b[?25hdone\nDone\n", "output_type": "stream" } ], "execution_count": 1 }, { "cell_type": "code", "source": "import os, gc, json, sys, subprocess, tempfile, time, traceback, warnings\nfrom datetime import datetime, timezone\nfrom typing import Any, Dict, List, Optional\nfrom collections import OrderedDict\nwarnings.filterwarnings(\"ignore\")\n\nimport torch\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\nos.environ[\"PYTORCH_CUDA_ALLOC_CONF\"] = \"expandable_segments:True\"\n\nfrom kaggle_secrets import UserSecretsClient\nos.environ[\"HF_TOKEN\"] = UserSecretsClient().get_secret(\"HF_TOKEN\")\n\nfrom huggingface_hub import login\nlogin(token=os.environ[\"HF_TOKEN\"])\n\nfor i in range(torch.cuda.device_count()):\n p = torch.cuda.get_device_properties(i)\n print(f\"GPU {i}: {p.name} ({p.total_memory/1e9:.1f} GB)\")", "metadata": { "trusted": true, "execution": { "iopub.status.busy": "2026-07-16T18:34:51.597293Z", "iopub.execute_input": "2026-07-16T18:34:51.598206Z", "iopub.status.idle": "2026-07-16T18:34:53.274328Z", "shell.execute_reply.started": "2026-07-16T18:34:51.598169Z", "shell.execute_reply": "2026-07-16T18:34:53.273346Z" } }, "outputs": [ { "name": "stderr", "text": "Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.\n", "output_type": "stream" }, { "name": "stdout", "text": "GPU 0: Tesla T4 (15.6 GB)\nGPU 1: Tesla T4 (15.6 GB)\n", "output_type": "stream" } ], "execution_count": 3 }, { "cell_type": "code", "source": "MODEL_ID = \"sandeeprdy1729/TIMPS-Coder-7B\"\nOUTPUT_DIR = \"/kaggle/working/timps-benchmarks\"\nos.makedirs(OUTPUT_DIR, exist_ok=True)\n\nMAX_NEW_TOKENS = 512\nTEMPERATURE = 0.1\nTOP_P = 0.95\n\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\n\nprint(f\"Loading {MODEL_ID} ...\")\nt0 = time.time()\ndtype = torch.bfloat16 if torch.cuda.is_bf16_supported() else torch.float16\nmodel = AutoModelForCausalLM.from_pretrained(\n MODEL_ID, torch_dtype=dtype, device_map=\"auto\", trust_remote_code=True,\n)\ntokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)\nif tokenizer.pad_token is None:\n tokenizer.pad_token = tokenizer.eos_token\nmodel.eval()\nprint(f\"Loaded in {time.time()-t0:.1f}s | VRAM: {torch.cuda.memory_allocated()/1e9:.2f} GB\")", "metadata": { "trusted": true, "execution": { "iopub.status.busy": "2026-07-16T18:35:00.552166Z", "iopub.execute_input": "2026-07-16T18:35:00.552989Z", "iopub.status.idle": "2026-07-16T18:37:05.693429Z", "shell.execute_reply.started": "2026-07-16T18:35:00.552957Z", "shell.execute_reply": "2026-07-16T18:37:05.692577Z" } }, "outputs": [ { "name": "stdout", "text": "Loading sandeeprdy1729/TIMPS-Coder-7B ...\n", "output_type": "stream" }, { "output_type": "display_data", "data": { "text/plain": "config.json: 0.00B [00:00, ?B/s]", "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "8b01012f34244751800d730837b69bbc" } }, "metadata": {} }, { "name": "stderr", "text": "`torch_dtype` is deprecated! Use `dtype` instead!\n", "output_type": "stream" }, { "output_type": "display_data", "data": { "text/plain": "model.safetensors.index.json: 0.00B [00:00, ?B/s]", "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "3cb7862321904ca1a3bdd5a505f9df38" } }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": "Downloading (incomplete total...): 0.00B [00:00, ?B/s]", "application/vnd.jupyter.widget-view+json": { "version_major": 2, "version_minor": 0, "model_id": "e02ebb8474854f1389f711f429ac0346" } }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": "Fetching 4 files: 0%| | 0/4 [00:00 str:\n text = generation.strip()\n if \"```python\" in text:\n parts = text.split(\"```python\")\n if len(parts) > 1:\n return parts[1].split(\"```\")[0].strip()\n if \"```\" in text:\n parts = text.split(\"```\")\n for part in parts[1:]:\n code = part.strip()\n first = code.split(\"\\n\")[0].strip()\n if first and not first.startswith((\"def \", \"class \", \"import \", \"from \")):\n code = \"\\n\".join(code.split(\"\\n\")[1:]).strip()\n if code:\n return code\n lines = text.split(\"\\n\")\n code_lines, started = [], False\n for line in lines:\n s = line.strip()\n if s.startswith((\"def \", \"class \", \"import \", \"from \")):\n started = True\n if started:\n code_lines.append(line)\n return \"\\n\".join(code_lines).strip() if code_lines else text\n\n\ndef run_test(code: str, test_code: str, timeout: float = 10.0) -> Dict[str, Any]:\n full = code + \"\\n\\n\" + test_code + \"\\n\\nprint('TEST_PASSED')\\n\"\n tmp = None\n try:\n with tempfile.NamedTemporaryFile(mode=\"w\", suffix=\".py\", delete=False) as f:\n f.write(full); tmp = f.name\n proc = subprocess.run(\n [sys.executable, tmp], capture_output=True, text=True, timeout=timeout,\n env={**os.environ, \"PYTHONDONTWRITEBYTECODE\": \"1\"},\n )\n passed = \"TEST_PASSED\" in proc.stdout\n return {\"passed\": passed, \"error\": None if passed else (proc.stderr[-300:] or \"tests did not pass\")}\n except subprocess.TimeoutExpired:\n return {\"passed\": False, \"error\": \"timeout\"}\n except Exception as e:\n return {\"passed\": False, \"error\": str(e)[:200]}\n finally:\n if tmp:\n try: os.unlink(tmp)\n except OSError: pass\n\n\ndef generate_one(prompt: str) -> str:\n if hasattr(tokenizer, \"apply_chat_template\"):\n messages = [\n {\"role\": \"system\", \"content\": \"You are an expert Python programmer. Write a complete, correct, and efficient solution. Output ONLY the code, no explanation.\"},\n {\"role\": \"user\", \"content\": prompt},\n ]\n try:\n chat = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n except Exception:\n chat = f\"<|im_start|>system\\nYou are an expert Python programmer. Write a complete, correct, and efficient solution. Output ONLY the code, no explanation.<|im_end|>\\n<|im_start|>user\\n{prompt}<|im_end|>\\n<|im_start|>assistant\\n\"\n else:\n chat = f\"<|im_start|>system\\nYou are an expert Python programmer. Write a complete, correct, and efficient solution. Output ONLY the code, no explanation.<|im_end|>\\n<|im_start|>user\\n{prompt}<|im_end|>\\n<|im_start|>assistant\\n\"\n\n inputs = tokenizer(chat, return_tensors=\"pt\", truncation=True, max_length=1536)\n inputs = {k: v.to(model.device) for k, v in inputs.items()}\n with torch.no_grad():\n out = model.generate(\n **inputs, max_new_tokens=MAX_NEW_TOKENS, temperature=TEMPERATURE,\n top_p=TOP_P, do_sample=(TEMPERATURE > 0),\n pad_token_id=tokenizer.pad_token_id, eos_token_id=tokenizer.eos_token_id,\n )\n new_tokens = out[0][inputs[\"input_ids\"].shape[1]:]\n return tokenizer.decode(new_tokens, skip_special_tokens=True)", "metadata": { "trusted": true, "execution": { "iopub.status.busy": "2026-07-16T18:37:33.497179Z", "iopub.execute_input": "2026-07-16T18:37:33.497638Z", "iopub.status.idle": "2026-07-16T18:37:33.51044Z", "shell.execute_reply.started": "2026-07-16T18:37:33.497608Z", "shell.execute_reply": "2026-07-16T18:37:33.509737Z" } }, "outputs": [], "execution_count": 9 }, { "cell_type": "code", "source": [ "from datasets import load_dataset\n", "\n", "print(\"=\" * 60)\n", "print(\" BENCHMARK: HumanEval (164 problems)\")\n", "print(\"=\" * 60)\n", "\n", "ds = load_dataset(\"openai/openai_humaneval\", split=\"test\")\n", "problems = list(ds)\n", "total = len(problems)\n", "passed = failed = errors = 0\n", "he_details = []\n", "\n", "t0 = time.time()\n", "for i, prob in enumerate(problems):\n", " prompt, test, entry_point = prob[\"prompt\"], prob[\"test\"], prob[\"entry_point\"]\n", " try:\n", " gen = generate_one(prompt)\n", " code = extract_code(gen)\n", " res = run_test(code, test)\n", " ok = res[\"passed\"]\n", " except Exception as e:\n", " ok, res, code = False, {\"passed\": False, \"error\": str(e)[:200]}, \"\"\n", "\n", " if ok: passed += 1; tag = \"PASS\"\n", " elif res.get(\"error\") and (\"Error\" in str(res[\"error\"]) or \"timeout\" in str(res.get(\"error\",\"\")).lower()):\n", " errors += 1; tag = \"ERR\"\n", " else: failed += 1; tag = \"FAIL\"\n", "\n", " # NOTE: we now store the generated `code` so HumanEval+ can reuse it\n", " # instead of re-running generation for all 164 problems a second time.\n", " he_details.append({\"idx\": i, \"task_id\": prob[\"task_id\"], \"entry_point\": entry_point,\n", " \"passed\": ok, \"tag\": tag, \"error\": res.get(\"error\"), \"code\": code})\n", " if (i + 1) % 10 == 0 or i == total - 1:\n", " print(f\" [{i+1:>3}/{total}] {tag:<5} running pass@1 = {passed/(i+1)*100:.1f}%\")\n", "\n", "he_result = {\"benchmark\": \"HumanEval\", \"total\": total, \"passed\": passed, \"failed\": failed,\n", " \"errors\": errors, \"pass@1\": round(passed/total*100, 1), \"per_problem\": he_details}\n", "print(f\"\\n HumanEval done in {(time.time()-t0)/60:.1f} min | pass@1 = {he_result['pass@1']}%\")" ], "metadata": { "trusted": true, "execution": { "iopub.status.busy": "2026-07-16T18:37:38.550056Z", "iopub.execute_input": "2026-07-16T18:37:38.551101Z", "execution_failed": "2026-07-16T18:44:48.67Z" } }, "outputs": [], "execution_count": null }, { "cell_type": "code", "source": [ "print(\"=\" * 60)\n", "print(\" BENCHMARK: HumanEval+ (extended tests via evalplus)\")\n", "print(\"=\" * 60)\n", "\n", "from evalplus.data import get_human_eval_plus\n", "from evalplus.eval import PASS\n", "\n", "t0 = time.time()\n", "hep_result = None\n", "\n", "try:\n", " hep_problems = get_human_eval_plus() # same 164 problems, with extra \"plus\" tests\n", "\n", " # Reuse the code already generated during the HumanEval pass above \u2014\n", " # no need to regenerate (this alone halves the GPU time this cell used to burn).\n", " samples_path = f\"{OUTPUT_DIR}/humaneval_plus_samples.jsonl\"\n", " result_path = samples_path.replace(\".jsonl\", \"_eval_results.json\")\n", " for p in (samples_path, result_path):\n", " if os.path.exists(p):\n", " os.remove(p) # evalplus prompts interactively if a stale result file exists\n", "\n", " covered = set()\n", " with open(samples_path, \"w\") as f:\n", " for d in he_details:\n", " task_id = d[\"task_id\"]\n", " if task_id in hep_problems and d[\"code\"]:\n", " f.write(json.dumps({\"task_id\": task_id, \"solution\": d[\"code\"]}) + \"\\n\")\n", " covered.add(task_id)\n", "\n", " missing = set(hep_problems) - covered\n", " if missing:\n", " print(f\" Filling {len(missing)} problem(s) missing/empty from the base run ...\")\n", " with open(samples_path, \"a\") as f:\n", " for task_id in missing:\n", " try:\n", " gen = generate_one(hep_problems[task_id][\"prompt\"])\n", " code = extract_code(gen)\n", " except Exception:\n", " code = \"\"\n", " f.write(json.dumps({\"task_id\": task_id, \"solution\": code}) + \"\\n\")\n", "\n", " from evalplus.evaluate import evaluate as ev_eval\n", " ev_eval(dataset=\"humaneval\", samples=samples_path, i_just_wanna_run=True)\n", "\n", " with open(result_path) as f:\n", " raw = json.load(f)\n", " n = len(raw[\"eval\"])\n", " base_pass = sum(1 for r in raw[\"eval\"].values() if r[0][\"base_status\"] == PASS)\n", " plus_pass = sum(1 for r in raw[\"eval\"].values()\n", " if r[0][\"base_status\"] == PASS and r[0][\"plus_status\"] == PASS)\n", " hep_result = {\n", " \"benchmark\": \"HumanEval+\", \"total\": n, \"passed\": plus_pass,\n", " \"pass@1\": round(plus_pass / n * 100, 1),\n", " \"base_pass@1\": round(base_pass / n * 100, 1),\n", " \"evalplus_raw_path\": result_path,\n", " }\n", " print(f\" HumanEval+ pass@1 = {hep_result['pass@1']}% (base-only pass@1 = {hep_result['base_pass@1']}%)\")\n", "\n", "except Exception as e:\n", " print(f\"[humaneval+] evalplus failed: {e}\")\n", " traceback.print_exc()\n", " passed = sum(1 for d in he_details if d[\"passed\"])\n", " hep_result = {\"benchmark\": \"HumanEval+\", \"total\": len(he_details), \"passed\": passed,\n", " \"pass@1\": round(passed/len(he_details)*100, 1), \"note\": f\"evalplus error: {e}\",\n", " \"per_problem\": he_details}\n", "\n", "print(f\" Done in {(time.time()-t0)/60:.1f} min\")" ], "metadata": { "trusted": true, "execution": { "execution_failed": "2026-07-16T18:44:48.67Z" } }, "outputs": [], "execution_count": null }, { "cell_type": "code", "source": [ "print(\"=\" * 60)\n", "print(\" BENCHMARK: MBPP (974 problems)\")\n", "print(\"=\" * 60)\n", "\n", "ds_mbpp = load_dataset(\"google-research-datasets/mbpp\", split=\"test\")\n", "mbpp_problems = list(ds_mbpp)\n", "total = len(mbpp_problems)\n", "passed = failed = errors = 0\n", "mbpp_details = []\n", "\n", "t0 = time.time()\n", "for i, prob in enumerate(mbpp_problems):\n", " # BUG FIX: this dataset's field is \"text\", not \"prompt\" (that key doesn't exist\n", " # and was throwing KeyError: 'prompt' before generating a single sample).\n", " prompt = prob.get(\"text\", prob.get(\"prompt\", \"\"))\n", " test_list = prob.get(\"test_list\", [])\n", " test_code = \"\\n\".join(test_list) if test_list else \"\"\n", " try:\n", " gen = generate_one(prompt)\n", " code = extract_code(gen)\n", " res = run_test(code, test_code)\n", " ok = res[\"passed\"]\n", " except Exception as e:\n", " ok, res, code = False, {\"passed\": False, \"error\": str(e)[:200]}, \"\"\n", "\n", " if ok: passed += 1; tag = \"PASS\"\n", " elif res.get(\"error\") and (\"Error\" in str(res[\"error\"]) or \"timeout\" in str(res.get(\"error\",\"\")).lower()):\n", " errors += 1; tag = \"ERR\"\n", " else: failed += 1; tag = \"FAIL\"\n", "\n", " mbpp_details.append({\"idx\": i, \"task_id\": prob.get(\"task_id\", f\"mbpp_{i}\"),\n", " \"passed\": ok, \"tag\": tag, \"error\": res.get(\"error\"), \"code\": code})\n", " if (i + 1) % 50 == 0 or i == total - 1:\n", " print(f\" [{i+1:>3}/{total}] {tag:<5} running pass@1 = {passed/(i+1)*100:.1f}%\")\n", "\n", "mbpp_result = {\"benchmark\": \"MBPP\", \"total\": total, \"passed\": passed, \"failed\": failed,\n", " \"errors\": errors, \"pass@1\": round(passed/total*100, 1), \"per_problem\": mbpp_details}\n", "print(f\"\\n MBPP done in {(time.time()-t0)/60:.1f} min | pass@1 = {mbpp_result['pass@1']}%\")" ], "metadata": { "trusted": true }, "outputs": [], "execution_count": null }, { "cell_type": "code", "source": [ "print(\"=\" * 60)\n", "print(\" BENCHMARK: MBPP+ (extended tests via evalplus)\")\n", "print(\"=\" * 60)\n", "\n", "from evalplus.data import get_mbpp_plus\n", "from evalplus.eval import PASS\n", "\n", "t0 = time.time()\n", "mbppp_result = None\n", "\n", "try:\n", " # NOTE: evalplus's MBPP+ is a separate, sanitized ~399-problem subset with its\n", " # own task_ids and prompts \u2014 it does NOT line up 1:1 with the raw 974-problem\n", " # MBPP test split used in the cell above, so we can't reuse those completions.\n", " # We generate fresh (but only for this smaller set, not all 974 again).\n", " mbppp_problems = get_mbpp_plus()\n", " mbppp_task_ids = list(mbppp_problems.keys())\n", " n_tasks = len(mbppp_task_ids)\n", " print(f\" evalplus MBPP+ has {n_tasks} sanitized problems (independent of the raw MBPP split above)\")\n", "\n", " samples_path = f\"{OUTPUT_DIR}/mbpp_plus_samples.jsonl\"\n", " result_path = samples_path.replace(\".jsonl\", \"_eval_results.json\")\n", " for p in (samples_path, result_path):\n", " if os.path.exists(p):\n", " os.remove(p)\n", "\n", " with open(samples_path, \"w\") as f:\n", " for i, task_id in enumerate(mbppp_task_ids):\n", " prompt = mbppp_problems[task_id][\"prompt\"]\n", " try:\n", " gen = generate_one(prompt)\n", " code = extract_code(gen)\n", " except Exception:\n", " code = \"\"\n", " f.write(json.dumps({\"task_id\": task_id, \"solution\": code}) + \"\\n\")\n", " if (i + 1) % 50 == 0 or i == n_tasks - 1:\n", " print(f\" Generated {i+1}/{n_tasks} solutions ...\")\n", "\n", " from evalplus.evaluate import evaluate as ev_eval\n", " ev_eval(dataset=\"mbpp\", samples=samples_path, i_just_wanna_run=True)\n", "\n", " with open(result_path) as f:\n", " raw = json.load(f)\n", " n = len(raw[\"eval\"])\n", " base_pass = sum(1 for r in raw[\"eval\"].values() if r[0][\"base_status\"] == PASS)\n", " plus_pass = sum(1 for r in raw[\"eval\"].values()\n", " if r[0][\"base_status\"] == PASS and r[0][\"plus_status\"] == PASS)\n", " mbppp_result = {\n", " \"benchmark\": \"MBPP+\", \"total\": n, \"passed\": plus_pass,\n", " \"pass@1\": round(plus_pass / n * 100, 1),\n", " \"base_pass@1\": round(base_pass / n * 100, 1),\n", " \"evalplus_raw_path\": result_path,\n", " }\n", " print(f\" MBPP+ pass@1 = {mbppp_result['pass@1']}% (base-only pass@1 = {mbppp_result['base_pass@1']}%)\")\n", "\n", "except Exception as e:\n", " print(f\"[mbpp+] evalplus failed: {e}\")\n", " traceback.print_exc()\n", " mbppp_result = {**mbpp_result, \"benchmark\": \"MBPP+\", \"note\": f\"evalplus error: {e}\"}\n", "\n", "print(f\" Done in {(time.time()-t0)/60:.1f} min\")" ], "metadata": { "trusted": true, "execution": { "execution_failed": "2026-07-16T18:44:48.67Z" } }, "outputs": [], "execution_count": null }, { "cell_type": "code", "source": "BASELINES = {\n \"Model\": [\"TIMPS-Coder-7B (this model)\", \"Qwen2.5-Coder-7B-Instruct\", \"Qwen2.5-Coder-7B\",\n \"DeepSeek-Coder-7B-Instruct-v1.5\", \"CodeLlama-7B-Instruct\", \"CodeGemma-7B-it\",\n \"StarCoder2-7B\", \"Llama-3.1-8B-Instruct\", \"Phi-3.5-mini-instruct (3.8B)\", \"Gemma-2-9B-it\"],\n \"HumanEval\": [\"\u2014\", \"86.6\", \"89.6\", \"84.1\", \"53.7\", \"56.1\", \"40.2\", \"72.6\", \"68.8\", \"54.3\"],\n \"HumanEval+\": [\"\u2014\", \"71.3\", \"76.2\", \"70.8\", \"44.5\", \"46.9\", \"32.9\", \"61.0\", \"57.9\", \"44.5\"],\n \"MBPP\": [\"\u2014\", \"82.0\", \"84.0\", \"79.6\", \"55.6\", \"61.8\", \"46.0\", \"70.8\", \"73.0\", \"59.6\"],\n \"MBPP+\": [\"\u2014\", \"69.6\", \"72.0\", \"68.4\", \"45.0\", \"50.6\", \"36.5\", \"58.7\", \"61.3\", \"49.3\"],\n \"Params\": [\"7B\", \"7.6B\", \"7.6B\", \"7.1B\", \"6.7B\", \"7.0B\", \"7.0B\", \"8.0B\", \"3.8B\", \"9.2B\"],\n}\n\nresults = {\"humaneval\": he_result, \"humaneval_plus\": hep_result, \"mbpp\": mbpp_result, \"mbpp_plus\": mbppp_result}\n\n# Fill in TIMPS-Coder scores\nbl = {k: list(v) for k, v in BASELINES.items()}\nbl[\"HumanEval\"][0] = str(he_result.get(\"pass@1\", \"\u2014\"))\nbl[\"HumanEval+\"][0] = str(hep_result.get(\"pass@1\", \"\u2014\"))\nbl[\"MBPP\"][0] = str(mbpp_result.get(\"pass@1\", \"\u2014\"))\nbl[\"MBPP+\"][0] = str(mbppp_result.get(\"pass@1\", \"\u2014\"))\n\ncols = list(bl.keys())\nhdr = \"| \" + \" | \".join(cols) + \" |\"\nsep = \"|\" + \"|\".join([\"---\"] * len(cols)) + \"|\"\nrows = [\"| \" + \" | \".join(str(bl[c][i]) for c in cols) + \" |\" for i in range(len(bl[\"Model\"]))]\ntable = \"\\n\".join([hdr, sep] + rows)\n\nprint(table)\n\n# Save results\npayload = {\n \"model\": MODEL_ID, \"timestamp\": datetime.now(timezone.utc).isoformat(),\n \"results_summary\": {name: {\"pass@1\": r.get(\"pass@1\"), \"total\": r.get(\"total\"), \"passed\": r.get(\"passed\")} for name, r in results.items()},\n \"detailed_results\": results,\n}\nwith open(f\"{OUTPUT_DIR}/benchmark_results.json\", \"w\") as f:\n json.dump(payload, f, indent=2, default=str)\nprint(f\"\\nResults saved to {OUTPUT_DIR}/benchmark_results.json\")", "metadata": { "trusted": true }, "outputs": [], "execution_count": null }, { "cell_type": "code", "source": "he = he_result.get(\"pass@1\", \"\u2014\")\nhep = hep_result.get(\"pass@1\", \"\u2014\")\nmb = mbpp_result.get(\"pass@1\", \"\u2014\")\nmbp = mbppp_result.get(\"pass@1\", \"\u2014\")\n\nMODEL_CARD = f\"\"\"---\nlicense: apache-2.0\nlanguage: [en]\nbase_model: Qwen/Qwen2.5-Coder-7B-Instruct\ntags: [code, qwen2.5, lora, merged, sft, dpo, grpo]\nlibrary_name: transformers\npipeline_tag: text-generation\nmodel-index:\n- name: TIMPS-Coder-7B\n results:\n - task: {{\"type\": \"text-generation\", \"name\": \"Code Generation\"}}\n dataset: {{\"type\": \"openai_humaneval\", \"name\": \"HumanEval\"}}\n metrics:\n - {{type: \"pass@1\", \"value\": {he}, \"name\": \"pass@1\"}}\n - task: {{\"type\": \"text-generation\", \"name\": \"Code Generation\"}}\n dataset: {{\"type\": \"evalplus/humanevalplus\", \"name\": \"HumanEval+\"}}\n metrics:\n - {{type: \"pass@1\", \"value\": {hep}, \"name\": \"pass@1\"}}\n - task: {{\"type\": \"text-generation\", \"name\": \"Code Generation\"}}\n dataset: {{\"type\": \"mbpp\", \"name\": \"MBPP\"}}\n metrics:\n - {{type: \"pass@1\", \"value\": {mb}, \"name\": \"pass@1\"}}\n - task: {{\"type\": \"text-generation\", \"name\": \"Code Generation\"}}\n dataset: {{\"type\": \"evalplus/mbppplus\", \"name\": \"MBPP+\"}}\n metrics:\n - {{type: \"pass@1\", \"value\": {mbp}, \"name\": \"pass@1\"}}\n---\n\n# TIMPS-Coder-7B\n\nTIMPS-Coder-7B is a code-generation model built by fine-tuning **Qwen2.5-Coder-7B-Instruct**\nthrough a 4-step pipeline: SFT, GRPO, DPO, and Self-Guided Self-Play.\n\n## Benchmark Results\n\n| Benchmark | Score |\n|-----------|-------|\n| **HumanEval pass@1** | **{he}%** |\n| **HumanEval+ pass@1** | **{hep}%** |\n| **MBPP pass@1** | **{mb}%** |\n| **MBPP+ pass@1** | **{mbp}%** |\n\n### Comparison with 7B-9B Code Models\n\n{table}\n\n## Usage\n\n```python\nfrom transformers import AutoModelForCausalLM, AutoTokenizer\nmodel = AutoModelForCausalLM.from_pretrained(\"sandeeprdy1729/TIMPS-Coder-7B\", device_map=\"auto\", torch_dtype=\"auto\")\ntokenizer = AutoTokenizer.from_pretrained(\"sandeeprdy1729/TIMPS-Coder-7B\")\nmessages = [{{\"role\": \"user\", \"content\": \"Write a fibonacci function.\"}}]\ninputs = tokenizer.apply_chat_template(messages, return_tensors=\"pt\").to(model.device)\nprint(tokenizer.decode(model.generate(inputs, max_new_tokens=512)[0]))\n\"\"\"\n\nwith open(f\"{OUTPUT_DIR}/model_card_results.md\", \"w\") as f:\n f.write(MODEL_CARD)\nprint(f\"Model card saved to {OUTPUT_DIR}/model_card_results.md\")\nprint(MODEL_CARD)", "metadata": { "trusted": true }, "outputs": [], "execution_count": null }, { "cell_type": "code", "source": "\n#**Cell 11 \u2014 Push scores to HuggingFace model card (optional):**\n# ```python\nfrom huggingface_hub import HfApi\n\napi = HfApi()\napi.upload_file(\n path_or_fileobj=f\"{OUTPUT_DIR}/model_card_results.md\",\n path_in_repo=\"README.md\",\n repo_id=MODEL_ID,\n repo_type=\"model\",\n commit_message=f\"Add benchmark results: HE={he}% HE+={hep}% MBPP={mb}% MBPP+={mbp}%\",\n)\nprint(f\"Model card updated at https://huggingface.co/{MODEL_ID}\")", "metadata": { "trusted": true }, "outputs": [], "execution_count": null } ] }