| Step | \n", "Training Loss | \n", "
|---|---|
| 10 | \n", "2.706355 | \n", "
| 20 | \n", "1.838779 | \n", "
| 30 | \n", "1.867870 | \n", "
| 40 | \n", "2.035277 | \n", "
| 50 | \n", "1.969270 | \n", "
| 60 | \n", "1.872686 | \n", "
| 70 | \n", "2.075605 | \n", "
| 80 | \n", "1.785248 | \n", "
| 90 | \n", "2.100350 | \n", "
| 100 | \n", "1.754046 | \n", "
| 110 | \n", "1.998036 | \n", "
| 120 | \n", "1.745317 | \n", "
| 130 | \n", "1.904688 | \n", "
| 140 | \n", "1.727313 | \n", "
| 150 | \n", "1.792165 | \n", "
| 160 | \n", "1.670163 | \n", "
| 170 | \n", "1.965536 | \n", "
| 180 | \n", "1.839244 | \n", "
| 190 | \n", "1.925561 | \n", "
| 200 | \n", "1.705184 | \n", "
| 210 | \n", "1.922308 | \n", "
| 220 | \n", "1.864636 | \n", "
| 230 | \n", "2.037172 | \n", "
| 240 | \n", "1.757723 | \n", "
| 250 | \n", "1.750287 | \n", "
| 260 | \n", "1.888365 | \n", "
| 270 | \n", "1.853752 | \n", "
| 280 | \n", "2.061565 | \n", "
| 290 | \n", "1.907451 | \n", "
| 300 | \n", "1.904591 | \n", "
| 310 | \n", "1.873063 | \n", "
| 320 | \n", "1.751838 | \n", "
| 330 | \n", "1.842035 | \n", "
| 340 | \n", "1.935822 | \n", "
| 350 | \n", "1.817710 | \n", "
| 360 | \n", "1.690598 | \n", "
| 370 | \n", "1.703986 | \n", "
| 380 | \n", "1.966968 | \n", "
| 390 | \n", "1.734728 | \n", "
| 400 | \n", "1.810708 | \n", "
"
]
},
"metadata": {}
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Unsloth: Double buffering enabled (parallel H2D + compute) for backward pass.\n",
"Unsloth: Will smartly offload gradients to save VRAM!\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"Unsloth: Restored added_tokens_decoder metadata in outputs/checkpoint-100/tokenizer_config.json.\n",
"Unsloth: Restored added_tokens_decoder metadata in outputs/checkpoint-200/tokenizer_config.json.\n",
"Unsloth: Restored added_tokens_decoder metadata in outputs/checkpoint-300/tokenizer_config.json.\n",
"Unsloth: Restored added_tokens_decoder metadata in outputs/checkpoint-400/tokenizer_config.json.\n"
]
}
],
"source": [
"trainer_stats = trainer.train()"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "5nZFbvLdKDRw"
},
"source": [
"## 10. Save LoRA Adapter"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "6aTYoeq2KDRw",
"outputId": "13479621-2144-4e33-af6e-3702e49a7092"
},
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"Unsloth: Restored added_tokens_decoder metadata in lora_model/tokenizer_config.json.\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"('lora_model/tokenizer_config.json',\n",
" 'lora_model/chat_template.jinja',\n",
" 'lora_model/tokenizer.json')"
]
},
"metadata": {},
"execution_count": 16
}
],
"source": [
"model.save_pretrained(\"lora_model\")\n",
"tokenizer.save_pretrained(\"lora_model\")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "S_fMRKMzKDRw"
},
"source": [
"## 11. Evaluation\n",
"### Comparing Base Model vs Fine-Tuned Model using Perplexity, BLEU, and ROUGE"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000,
"referenced_widgets": [
"00e19d6ed461407eb28c9df466be8dda",
"2ff1682f6b6f485c883d484de79da73b",
"1e58e4a2be8b44cc9a89b0a88a5b65e6",
"0626e191b6e94572a5cd7fdca2824b4c",
"86a6a0168d164a51bc0f9d2934aa8542",
"1e3f887c390e418abff1c3cffdcd1ee1",
"82356062b3e54bad8a4c95c1304778ce",
"b8ce7d8de96648689ebc19b007b30203",
"1f6f838232444091a206066f34d89124",
"0c3508594bad464c8360000e73618acb",
"bfc83fb252504ec38f9084c4257453b1",
"fe13174dba9840fa955ffc2f5e3c0271",
"0b2c9890329348ad95a93e642987ad14",
"1ce28ad47fe148d3ace6f86f9a3910bb",
"f4e89cd9f96c43e6905db6e5fff6f218",
"a2287821709e443298b5d883f32bdf6f",
"d95be78e47884d13b73cbfb0a81991ae",
"70aee1ce297e450486815009d414ffa2",
"1a3bdce7c68b4c57a0f94d122a743c73",
"2e55b52b30ec46e9915e67452a1a72f5",
"07b1dc758d17486b8a3bd6695cf0bb9c",
"8a4001b8943a4021b5b9dd5f0e0b60de"
]
},
"id": "9yxheKaDKDRw",
"outputId": "4ee747b0-872c-4358-efb1-96bdb39c5b97",
"collapsed": true
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"==((====))== Unsloth 2026.5.4: Fast Qwen2 patching. Transformers: 5.5.0.\n",
" \\\\ /| Tesla T4. Num GPUs = 1. Max memory: 14.563 GB. Platform: Linux.\n",
"O^O/ \\_/ \\ Torch: 2.10.0+cu128. CUDA: 7.5. CUDA Toolkit: 12.8. Triton: 3.6.0\n",
"\\ / Bfloat16 = FALSE. FA [Xformers = None. FA2 = False]\n",
" \"-____-\" Free license: http://github.com/unslothai/unsloth\n",
"Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"Loading weights: 0%| | 0/338 [00:00.\n",
"==((====))== Unsloth 2026.5.4: Fast Qwen2 patching. Transformers: 5.5.0.\n",
" \\\\ /| Tesla T4. Num GPUs = 1. Max memory: 14.563 GB. Platform: Linux.\n",
"O^O/ \\_/ \\ Torch: 2.10.0+cu128. CUDA: 7.5. CUDA Toolkit: 12.8. Triton: 3.6.0\n",
"\\ / Bfloat16 = FALSE. FA [Xformers = None. FA2 = False]\n",
" \"-____-\" Free license: http://github.com/unslothai/unsloth\n",
"Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
]
},
{
"output_type": "display_data",
"data": {
"text/plain": [
"Loading weights: 0%| | 0/338 [00:00.\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"PeftModelForCausalLM(\n",
" (base_model): LoraModel(\n",
" (model): Qwen2ForCausalLM(\n",
" (model): Qwen2Model(\n",
" (embed_tokens): Embedding(151936, 1536, padding_idx=151665)\n",
" (layers): ModuleList(\n",
" (0-27): 28 x Qwen2DecoderLayer(\n",
" (self_attn): Qwen2Attention(\n",
" (q_proj): lora.Linear4bit(\n",
" (base_layer): Linear4bit(in_features=1536, out_features=1536, bias=True)\n",
" (lora_dropout): ModuleDict(\n",
" (default): Dropout(p=0.05, inplace=False)\n",
" )\n",
" (lora_A): ModuleDict(\n",
" (default): Linear(in_features=1536, out_features=32, bias=False)\n",
" )\n",
" (lora_B): ModuleDict(\n",
" (default): Linear(in_features=32, out_features=1536, bias=False)\n",
" )\n",
" (lora_embedding_A): ParameterDict()\n",
" (lora_embedding_B): ParameterDict()\n",
" (lora_magnitude_vector): ModuleDict()\n",
" )\n",
" (k_proj): lora.Linear4bit(\n",
" (base_layer): Linear4bit(in_features=1536, out_features=256, bias=True)\n",
" (lora_dropout): ModuleDict(\n",
" (default): Dropout(p=0.05, inplace=False)\n",
" )\n",
" (lora_A): ModuleDict(\n",
" (default): Linear(in_features=1536, out_features=32, bias=False)\n",
" )\n",
" (lora_B): ModuleDict(\n",
" (default): Linear(in_features=32, out_features=256, bias=False)\n",
" )\n",
" (lora_embedding_A): ParameterDict()\n",
" (lora_embedding_B): ParameterDict()\n",
" (lora_magnitude_vector): ModuleDict()\n",
" )\n",
" (v_proj): lora.Linear4bit(\n",
" (base_layer): Linear4bit(in_features=1536, out_features=256, bias=True)\n",
" (lora_dropout): ModuleDict(\n",
" (default): Dropout(p=0.05, inplace=False)\n",
" )\n",
" (lora_A): ModuleDict(\n",
" (default): Linear(in_features=1536, out_features=32, bias=False)\n",
" )\n",
" (lora_B): ModuleDict(\n",
" (default): Linear(in_features=32, out_features=256, bias=False)\n",
" )\n",
" (lora_embedding_A): ParameterDict()\n",
" (lora_embedding_B): ParameterDict()\n",
" (lora_magnitude_vector): ModuleDict()\n",
" )\n",
" (o_proj): lora.Linear4bit(\n",
" (base_layer): Linear4bit(in_features=1536, out_features=1536, bias=False)\n",
" (lora_dropout): ModuleDict(\n",
" (default): Dropout(p=0.05, inplace=False)\n",
" )\n",
" (lora_A): ModuleDict(\n",
" (default): Linear(in_features=1536, out_features=32, bias=False)\n",
" )\n",
" (lora_B): ModuleDict(\n",
" (default): Linear(in_features=32, out_features=1536, bias=False)\n",
" )\n",
" (lora_embedding_A): ParameterDict()\n",
" (lora_embedding_B): ParameterDict()\n",
" (lora_magnitude_vector): ModuleDict()\n",
" )\n",
" (rotary_emb): LlamaRotaryEmbedding()\n",
" )\n",
" (mlp): Qwen2MLP(\n",
" (gate_proj): lora.Linear4bit(\n",
" (base_layer): Linear4bit(in_features=1536, out_features=8960, bias=False)\n",
" (lora_dropout): ModuleDict(\n",
" (default): Dropout(p=0.05, inplace=False)\n",
" )\n",
" (lora_A): ModuleDict(\n",
" (default): Linear(in_features=1536, out_features=32, bias=False)\n",
" )\n",
" (lora_B): ModuleDict(\n",
" (default): Linear(in_features=32, out_features=8960, bias=False)\n",
" )\n",
" (lora_embedding_A): ParameterDict()\n",
" (lora_embedding_B): ParameterDict()\n",
" (lora_magnitude_vector): ModuleDict()\n",
" )\n",
" (up_proj): lora.Linear4bit(\n",
" (base_layer): Linear4bit(in_features=1536, out_features=8960, bias=False)\n",
" (lora_dropout): ModuleDict(\n",
" (default): Dropout(p=0.05, inplace=False)\n",
" )\n",
" (lora_A): ModuleDict(\n",
" (default): Linear(in_features=1536, out_features=32, bias=False)\n",
" )\n",
" (lora_B): ModuleDict(\n",
" (default): Linear(in_features=32, out_features=8960, bias=False)\n",
" )\n",
" (lora_embedding_A): ParameterDict()\n",
" (lora_embedding_B): ParameterDict()\n",
" (lora_magnitude_vector): ModuleDict()\n",
" )\n",
" (down_proj): lora.Linear4bit(\n",
" (base_layer): Linear4bit(in_features=8960, out_features=1536, bias=False)\n",
" (lora_dropout): ModuleDict(\n",
" (default): Dropout(p=0.05, inplace=False)\n",
" )\n",
" (lora_A): ModuleDict(\n",
" (default): Linear(in_features=8960, out_features=32, bias=False)\n",
" )\n",
" (lora_B): ModuleDict(\n",
" (default): Linear(in_features=32, out_features=1536, bias=False)\n",
" )\n",
" (lora_embedding_A): ParameterDict()\n",
" (lora_embedding_B): ParameterDict()\n",
" (lora_magnitude_vector): ModuleDict()\n",
" )\n",
" (act_fn): SiLUActivation()\n",
" )\n",
" (input_layernorm): Qwen2RMSNorm((1536,), eps=1e-06)\n",
" (post_attention_layernorm): Qwen2RMSNorm((1536,), eps=1e-06)\n",
" )\n",
" )\n",
" (norm): Qwen2RMSNorm((1536,), eps=1e-06)\n",
" (rotary_emb): LlamaRotaryEmbedding()\n",
" )\n",
" (lm_head): Linear(in_features=1536, out_features=151936, bias=False)\n",
" )\n",
" )\n",
")"
]
},
"metadata": {},
"execution_count": 17
}
],
"source": [
"# Base model\n",
"base_model, tokenizer = FastLanguageModel.from_pretrained(\n",
" model_name=Base_Model,\n",
" max_seq_length=max_seq_length,\n",
" dtype=dtype,\n",
" load_in_4bit=load_in_4bit,\n",
")\n",
"\n",
"FastLanguageModel.for_inference(base_model)\n",
"\n",
"# Fine-tuned model\n",
"ft_model, _ = FastLanguageModel.from_pretrained(\n",
" model_name=\"lora_model\",\n",
" max_seq_length=max_seq_length,\n",
" dtype=dtype,\n",
" load_in_4bit=load_in_4bit,\n",
")\n",
"\n",
"FastLanguageModel.for_inference(ft_model)"
]
},
{
"cell_type": "markdown",
"source": [
"# Prepare evaluation data\n",
"***I will use only 50 samples for evaluation because it taking too long to evaluate on mare than 50 samples***"
],
"metadata": {
"id": "xDeLjPqpsrag"
}
},
{
"cell_type": "code",
"source": [
"N_EVAL = 50\n",
"\n",
"test_raw = test_dataset.select(range(N_EVAL))\n",
"\n",
"prompts = []\n",
"references = []\n",
"ppl_texts = []\n",
"\n",
"for example in test_raw:\n",
"\n",
" instruction_content = example[\"question\"]\n",
" reference_answer = example[\"answer\"]\n",
" full_text_for_perplexity = example[\"text\"]\n",
"\n",
" prompt_messages_for_generation = [\n",
" {\"role\": \"system\", \"content\": custom_system_message},\n",
" {\"role\": \"user\", \"content\": instruction_content}\n",
" ]\n",
" prompt = tokenizer.apply_chat_template(\n",
" prompt_messages_for_generation,\n",
" tokenize=False,\n",
" add_generation_prompt=True\n",
" )\n",
"\n",
" prompts.append(prompt)\n",
" references.append(reference_answer)\n",
" ppl_texts.append(full_text_for_perplexity)\n",
"\n",
"print(f\"Prepared {len(prompts)} evaluation samples.\")\n",
"\n",
"print(\"\\n── Sample Prompt ──\\n\")\n",
"print(prompts[0][:1000])\n",
"\n",
"print(\"\\n── Sample Reference ──\\n\")\n",
"print(references[0][:500])"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "TEyKchuPstVH",
"outputId": "b5a79eeb-175a-405d-e0eb-6c863e8e3b62"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Prepared 50 evaluation samples.\n",
"\n",
"── Sample Prompt ──\n",
"\n",
"<|im_start|>system\n",
"You are a financial intelligence assistantProvide accurate, structured financial explanations.<|im_end|>\n",
"<|im_start|>user\n",
"Relation between interest rates and currency for a nation<|im_end|>\n",
"<|im_start|>assistant\n",
"\n",
"\n",
"── Sample Reference ──\n",
"\n",
"From Indian context, there are a number of factors that are influencing the economic condition and the exchange rate, interest rate etc. are reflection of the situation. I shall try and answer the question through the above Indian example. India is running a budget deficit of 4 odd % for last 6-7 years, which means that gov.in is spending more than their revenue collection, this money is not in the system, so the govt. has to print the money, either the direct 4% or the interest it has to pay on\n"
]
}
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "zQtOngmtKDRw"
},
"source": [
"### 12. Perplexity"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "QdzPqPCYKDRw",
"outputId": "08ad7521-2984-4bef-e7d6-2ab2b167f5eb",
"collapsed": true
},
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"/usr/local/lib/python3.12/dist-packages/transformers/modeling_attn_mask_utils.py:71: FutureWarning: The attention mask API under `transformers.modeling_attn_mask_utils` (`AttentionMaskConverter`) is deprecated and will be removed in Transformers v5.10. Please use the new API in `transformers.masking_utils`.\n",
" warnings.warn(DEPRECATION_MESSAGE, FutureWarning)\n",
"/usr/local/lib/python3.12/dist-packages/transformers/modeling_attn_mask_utils.py:281: FutureWarning: The attention mask API under `transformers.modeling_attn_mask_utils` (`AttentionMaskConverter`) is deprecated and will be removed in Transformers v5.10. Please use the new API in `transformers.masking_utils`.\n",
" warnings.warn(DEPRECATION_MESSAGE, FutureWarning)\n",
"/usr/local/lib/python3.12/dist-packages/transformers/modeling_attn_mask_utils.py:71: FutureWarning: The attention mask API under `transformers.modeling_attn_mask_utils` (`AttentionMaskConverter`) is deprecated and will be removed in Transformers v5.10. Please use the new API in `transformers.masking_utils`.\n",
" warnings.warn(DEPRECATION_MESSAGE, FutureWarning)\n",
"/usr/local/lib/python3.12/dist-packages/transformers/modeling_attn_mask_utils.py:281: FutureWarning: The attention mask API under `transformers.modeling_attn_mask_utils` (`AttentionMaskConverter`) is deprecated and will be removed in Transformers v5.10. Please use the new API in `transformers.masking_utils`.\n",
" warnings.warn(DEPRECATION_MESSAGE, FutureWarning)\n",
"/usr/local/lib/python3.12/dist-packages/transformers/modeling_attn_mask_utils.py:71: FutureWarning: The attention mask API under `transformers.modeling_attn_mask_utils` (`AttentionMaskConverter`) is deprecated and will be removed in Transformers v5.10. Please use the new API in `transformers.masking_utils`.\n",
" warnings.warn(DEPRECATION_MESSAGE, FutureWarning)\n",
"/usr/local/lib/python3.12/dist-packages/transformers/modeling_attn_mask_utils.py:281: FutureWarning: The attention mask API under `transformers.modeling_attn_mask_utils` (`AttentionMaskConverter`) is deprecated and will be removed in Transformers v5.10. Please use the new API in `transformers.masking_utils`.\n",
" warnings.warn(DEPRECATION_MESSAGE, FutureWarning)\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Base model PPL : 28.07\n",
"Fine-tuned PPL : 5.81\n",
"Improvement : 79.3%\n"
]
}
],
"source": [
"def compute_perplexity(model, tokenizer, texts, max_len=2048):\n",
"\n",
" model.eval()\n",
"\n",
" losses = []\n",
"\n",
" with torch.no_grad():\n",
"\n",
" for text in texts:\n",
"\n",
" enc = tokenizer(\n",
" text,\n",
" return_tensors=\"pt\",\n",
" truncation=True,\n",
" max_length=max_len\n",
" ).to(model.device)\n",
"\n",
" outputs = model(\n",
" **enc,\n",
" labels=enc[\"input_ids\"]\n",
" )\n",
"\n",
" losses.append(outputs.loss.item())\n",
"\n",
" ppl = np.exp(np.mean(losses))\n",
"\n",
" return round(float(ppl), 2)\n",
"\n",
"\n",
"base_ppl = compute_perplexity(\n",
" base_model,\n",
" tokenizer,\n",
" ppl_texts\n",
")\n",
"\n",
"ft_ppl = compute_perplexity(\n",
" ft_model,\n",
" tokenizer,\n",
" ppl_texts\n",
")\n",
"\n",
"print(f\"Base model PPL : {base_ppl}\")\n",
"print(f\"Fine-tuned PPL : {ft_ppl}\")\n",
"\n",
"print(\n",
" f\"Improvement : \"\n",
" f\"{round((base_ppl - ft_ppl) / base_ppl * 100, 1)}%\"\n",
")"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "fy9HksHHKDRw"
},
"source": [
"### 13. BLEU & ROUGE"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "OsyGJGr3KDRw",
"outputId": "f9fee623-e52d-4979-f784-5b1dca15b633",
"collapsed": true
},
"outputs": [
{
"output_type": "stream",
"name": "stderr",
"text": [
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Generating base model predictions...\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n"
]
},
{
"output_type": "stream",
"name": "stdout",
"text": [
"Generating fine-tuned model predictions...\n"
]
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n",
"Both `max_new_tokens` (=128) and `max_length`(=32768) seem to have been set. `max_new_tokens` will take precedence. Please refer to the documentation for more information. (https://huggingface.co/docs/transformers/main/en/main_classes/text_generation)\n"
]
}
],
"source": [
"def generate(model, tokenizer, prompt, max_new=128):\n",
"\n",
" inputs = tokenizer(\n",
" prompt,\n",
" return_tensors=\"pt\"\n",
" ).to(model.device)\n",
"\n",
" with torch.no_grad():\n",
"\n",
" outputs = model.generate(\n",
" **inputs,\n",
" max_new_tokens=max_new,\n",
" do_sample=False,\n",
" pad_token_id=tokenizer.eos_token_id,\n",
" )\n",
"\n",
" prediction = tokenizer.decode(\n",
" outputs[0][inputs.input_ids.shape[1]:],\n",
" skip_special_tokens=True\n",
" )\n",
"\n",
" return prediction.strip()\n",
"\n",
"print(\"Generating base model predictions...\")\n",
"base_preds = [\n",
" generate(base_model, tokenizer, p)\n",
" for p in prompts\n",
"]\n",
"\n",
"print(\"Generating fine-tuned model predictions...\")\n",
"ft_preds = [\n",
" generate(ft_model, tokenizer, p)\n",
" for p in prompts\n",
"]"
]
},
{
"cell_type": "code",
"source": [
"bleu = BLEU()\n",
"\n",
"base_bleu = bleu.corpus_score(\n",
" base_preds,\n",
" [references]\n",
").score\n",
"\n",
"ft_bleu = bleu.corpus_score(\n",
" ft_preds,\n",
" [references]\n",
").score\n",
"\n",
"print(f\"\\nBLEU base={base_bleu:.2f}\")\n",
"print(f\"BLEU fine-tuned={ft_bleu:.2f}\")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "5Z9HKSFUtWKA",
"outputId": "6e20b8cf-925c-4bf7-b587-b8883a811f1f"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"BLEU base=2.36\n",
"BLEU fine-tuned=3.82\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"rouge = rouge_scorer.RougeScorer(\n",
" [\"rouge1\", \"rougeL\"],\n",
" use_stemmer=True\n",
")\n",
"\n",
"def avg_rouge(preds, refs):\n",
"\n",
" r1_scores = []\n",
" rL_scores = []\n",
"\n",
" for pred, ref in zip(preds, refs):\n",
"\n",
" scores = rouge.score(ref, pred)\n",
"\n",
" r1_scores.append(\n",
" scores[\"rouge1\"].fmeasure\n",
" )\n",
"\n",
" rL_scores.append(\n",
" scores[\"rougeL\"].fmeasure\n",
" )\n",
"\n",
" return (\n",
" round(sum(r1_scores) / len(r1_scores), 4),\n",
" round(sum(rL_scores) / len(rL_scores), 4),\n",
" )\n",
"\n",
"base_r1, base_rL = avg_rouge(\n",
" base_preds,\n",
" references\n",
")\n",
"\n",
"ft_r1, ft_rL = avg_rouge(\n",
" ft_preds,\n",
" references\n",
")\n",
"\n",
"print(f\"ROUGE-1 base={base_r1}\")\n",
"print(f\"ROUGE-1 fine-tuned={ft_r1}\")\n",
"\n",
"print(f\"ROUGE-L base={base_rL}\")\n",
"print(f\"ROUGE-L fine-tuned={ft_rL}\")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "dXp3DsPMtgxG",
"outputId": "e05ba708-bd80-4fb7-b40f-b8596fb78c11"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"ROUGE-1 base=0.2596\n",
"ROUGE-1 fine-tuned=0.3069\n",
"ROUGE-L base=0.1439\n",
"ROUGE-L fine-tuned=0.1805\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# Final Summary Table"
],
"metadata": {
"id": "TG5RUV8yti6C"
}
},
{
"cell_type": "code",
"source": [
"print(\"\\n\" + \"─\" * 60)\n",
"\n",
"print(\n",
" f\"{'Metric':<15}\"\n",
" f\"{'Base':>12}\"\n",
" f\"{'Fine-tuned':>15}\"\n",
" f\"{'Result':>15}\"\n",
")\n",
"\n",
"print(\"─\" * 60)\n",
"\n",
"metrics = [\n",
" (\"PPL ↓\", base_ppl, ft_ppl, True),\n",
" (\"BLEU ↑\", base_bleu, ft_bleu, False),\n",
" (\"ROUGE-1 ↑\", base_r1, ft_r1, False),\n",
" (\"ROUGE-L ↑\", base_rL, ft_rL, False),\n",
"]\n",
"\n",
"for metric, base_v, ft_v, lower_better in metrics:\n",
"\n",
" improved = (\n",
" ft_v < base_v\n",
" if lower_better\n",
" else ft_v > base_v\n",
" )\n",
"\n",
" result = (\n",
" \"✓ improved\"\n",
" if improved\n",
" else \"✗ regressed\"\n",
" )\n",
"\n",
" print(\n",
" f\"{metric:<15}\"\n",
" f\"{base_v:>12.4f}\"\n",
" f\"{ft_v:>15.4f}\"\n",
" f\"{result:>15}\"\n",
" )\n",
"\n",
"print(\"─\" * 60)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "8HnIE9cdtmdC",
"outputId": "253ab707-7e37-4136-e9f2-3281aa85e062"
},
"execution_count": null,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\n",
"────────────────────────────────────────────────────────────\n",
"Metric Base Fine-tuned Result\n",
"────────────────────────────────────────────────────────────\n",
"PPL ↓ 28.0700 5.8100 ✓ improved\n",
"BLEU ↑ 2.3573 3.8204 ✓ improved\n",
"ROUGE-1 ↑ 0.2596 0.3069 ✓ improved\n",
"ROUGE-L ↑ 0.1439 0.1805 ✓ improved\n",
"────────────────────────────────────────────────────────────\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# Metrics data\n",
"metrics = [\"PPL ↓\", \"BLEU ↑\", \"ROUGE-1 ↑\", \"ROUGE-L ↑\"]\n",
"base_values = [base_ppl, base_bleu, base_r1, base_rL]\n",
"finetuned_values = [ft_ppl, ft_bleu, ft_r1, ft_rL]\n",
"\n",
"# Plot style\n",
"sns.set(style=\"whitegrid\", font_scale=1.2)\n",
"\n",
"x = range(len(metrics))\n",
"width = 0.35\n",
"\n",
"plt.figure(figsize=(10,6))\n",
"plt.bar([i - width/2 for i in x], base_values, width, label=\"Base Model\", color=\"skyblue\")\n",
"plt.bar([i + width/2 for i in x], finetuned_values, width, label=\"Fine-tuned Model\", color=\"orange\")\n",
"\n",
"# Labels and formatting\n",
"plt.xticks(x, metrics)\n",
"plt.ylabel(\"Score / Value\")\n",
"plt.title(\"Base vs Fine-tuned Model Performance\")\n",
"plt.legend()\n",
"\n",
"# Annotate improvements\n",
"for i, (b, f) in enumerate(zip(base_values, finetuned_values)):\n",
" plt.text(i - width/2, b + 0.01, f\"{b:.2f}\", ha=\"center\", va=\"bottom\", fontsize=10)\n",
" plt.text(i + width/2, f + 0.01, f\"{f:.2f}\", ha=\"center\", va=\"bottom\", fontsize=10)\n",
"\n",
"plt.tight_layout()\n",
"plt.show()"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 593
},
"id": "6c9fKUIzoxF1",
"outputId": "cd34de87-bc24-4492-fec4-616ce554775e"
},
"execution_count": null,
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
"