final

README.md

@@ -16,6 +16,8 @@ https://github.com/tloen/alpaca-lora
 https://huggingface.co/docs/transformers/main/en/model_doc/llama#llama
 https://huggingface.co/docs/transformers/index
 https://github.com/salesforce/WikiSQL
+https://github.com/huggingface/peft
+
 
 
 https://arxiv.org/pdf/1910.13461.pdf

inf2.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d423a6b369859e281c8abe3c9b9a586c941d9add5e30ef0fb0f128c819de54a
+size 4305279

wikisql_retrain.ipynb

@@ -0,0 +1,1611 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "True"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "torch.cuda.is_available()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/matt/hf/sqllama-V0/.venv/lib/python3.7/site-packages/bitsandbytes/cuda_setup/main.py:136: UserWarning: /opt/conda did not contain libcudart.so as expected! Searching further paths...\n",
+      "  warn(msg)\n",
+      "The tokenizer class you load from this checkpoint is not the same type as the class this function is called from. It may result in unexpected tokenization. \n",
+      "The tokenizer class you load from this checkpoint is 'LLaMATokenizer'. \n",
+      "The class this function is called from is 'LlamaTokenizer'.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "===================================BUG REPORT===================================\n",
+      "Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues\n",
+      "================================================================================\n",
+      "CUDA SETUP: CUDA runtime path found: /usr/local/cuda/lib64/libcudart.so\n",
+      "CUDA SETUP: Highest compute capability among GPUs detected: 7.5\n",
+      "CUDA SETUP: Detected CUDA version 113\n",
+      "CUDA SETUP: Loading binary /home/matt/hf/sqllama-V0/.venv/lib/python3.7/site-packages/bitsandbytes/libbitsandbytes_cuda113.so...\n",
+      "PeftConfig(peft_type='LORA', base_model_name_or_path='decapoda-research/llama-7b-hf', task_type='CASUAL_LM', inference_mode=True)\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "4581e257066e463d9638f5d3a407a70e",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/33 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from transformers import LlamaTokenizer, LlamaForCausalLM\n",
+    "from peft import get_peft_model, PeftConfig, PeftModel\n",
+    "\n",
+    "loc = 'sqllama-out3'\n",
+    "\n",
+    "config = PeftConfig.from_pretrained(loc)\n",
+    "print(config)\n",
+    "\n",
+    "\n",
+    "tokenizer = LlamaTokenizer.from_pretrained(\n",
+    "    \"decapoda-research/llama-7b-hf\")\n",
+    "   \n",
+    "tokenizer.pad_token_id = 0\n",
+    "tokenizer.padding_side = 'left'\n",
+    "\n",
+    "model = LlamaForCausalLM.from_pretrained(\n",
+    "    \"decapoda-research/llama-7b-hf\",\n",
+    "    load_in_8bit=True,\n",
+    "    device_map=\"auto\",\n",
+    "    torch_dtype=torch.float16\n",
+    ")\n",
+    "\n",
+    "model = PeftModel.from_pretrained(\n",
+    "    model, loc,\n",
+    "    torch_dtype=torch.float16,\n",
+    "    device_map=\"auto\"\n",
+    "    )\n",
+    "\n",
+    "#model.push_to_hub('LlamaSQL-3')\n",
+    "\n",
+    "#model = prepare_model_for_int8_training(model)\n",
+    "\n",
+    "#model = get_peft_model(model,config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "table: 1-20246201-9\n",
+      "columns: Candidate,Office,Home state,Popular vote,States – first place,States – second place,States – third place\n",
+      "Q: How many states-first place are there for the office of Governor?\n",
+      "A: SELECT COUNT States – first place FROM 1-20246201-9 WHERE Office = 'Governor'\n",
+      "END\n",
+      "\n",
+      "\n",
+      "table: 2-17429402-7\n",
+      "columns: School,Years of Participation,OCC Championships,Last OCC Championship,Last Outright OCC Championship\n",
+      "Q: How many times have Central Crossing won the OCC Championship?\n",
+      "A: SELECT SUM OCC Championships FROM 2-17429402-7 WHERE School = 'central crossing'\n",
+      "END\n",
+      "\n",
+      "\n",
+      "table: 1-11677691-10\n",
+      "columns: Player,Position,School,Hometown,College\n",
+      "Q: What town is Muscle Shoals High School located in?\n",
+      "A: SELECT  Hometown FROM 1-11677691-10 WHERE School = 'Muscle Shoals High School'\n",
+      "END\n",
+      "\n",
+      "\n",
+      "table: 2-10701914-2\n",
+      "columns: Home team,Home team score,Away team,Away team score,Venue,Crowd,Date\n",
+      "Q: What is the largest crowd when st kilda was the away team?\n",
+      "A: SELECT MAX Crowd FROM 2-10701914-2 WHERE Away team = 'st kilda'\n",
+      "END\n",
+      "\n",
+      "\n",
+      "table: 2-1122152-1\n",
+      "columns: Driver,Constructor,Laps,Time/Retired,Grid\n",
+      "Q: What is the lap total for the grid under 15 that retired due to transmission?\n",
+      "A: SELECT SUM Laps FROM 2-1122152-1 WHERE Grid < 15 AND Time/Retired = 'transmission'\n",
+      "END\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "import random\n",
+    "import json\n",
+    "\n",
+    "# defined by WikiSQL\n",
+    "\n",
+    "agg_ops = ['', 'MAX', 'MIN', 'COUNT', 'SUM', 'AVG']\n",
+    "cond_ops = ['=', '>', '<', 'OP']\n",
+    "syms = ['SELECT', 'WHERE', 'AND', 'COL', 'TABLE', 'CAPTION', 'PAGE', 'SECTION', 'OP', 'COND', 'QUESTION', 'AGG', 'AGGOPS', 'CONDOPS']\n",
+    "\n",
+    "def fix_repr(d,cols,types,tid):\n",
+    "    sel_index=d['sel'] \n",
+    "    agg_index=d['agg']\n",
+    "    conditions=d['conds']\n",
+    "    col = cols[sel_index]\n",
+    "    rep = 'SELECT {agg} {sel} FROM {tid}'.format(\n",
+    "            agg=agg_ops[agg_index],\n",
+    "            sel=col,\n",
+    "            tid=tid\n",
+    "            )\n",
+    "    if conditions:\n",
+    "        cs = []\n",
+    "        for i, o, v in conditions:\n",
+    "            #print(i,cols)\n",
+    "            nm = cols[i]\n",
+    "            op = cond_ops[o]\n",
+    "            \n",
+    "            if types[i] in ['text']:\n",
+    "                val = f\"\\'{v}\\'\"\n",
+    "            else:\n",
+    "                val = v\n",
+    "            cs.append(f'{nm} {op} {val}')\n",
+    "        #print(cs)\n",
+    "\n",
+    "        rep +=  ' WHERE ' + ' AND '.join(cs)\n",
+    "    \n",
+    "    return rep\n",
+    "\n",
+    "tbl_cols = {}\n",
+    "tbl_types = {}\n",
+    "tbl_str = {}\n",
+    "\n",
+    "prefix = 'Below is a question that describes a data request, paired with an input that describes a SQL table.  Write a SQL query that retrieves the data.'\n",
+    "\n",
+    "def tbl_def_to_string(id, header, types):\n",
+    "    s = f'table: {id}\\ncolumns: ' + ','.join(header)\n",
+    "    return s\n",
+    "\n",
+    "with open('data/train.tables.jsonl') as f:\n",
+    "    for line in f:\n",
+    "        js = json.loads(line)\n",
+    "        id = js['id']\n",
+    "        hdr = js['header']\n",
+    "        ts = js['types']\n",
+    "        tbl_str[id] = tbl_def_to_string(id,hdr,ts)\n",
+    "        tbl_cols[id] = hdr\n",
+    "        tbl_types[id] = ts\n",
+    "\n",
+    "q_s = []\n",
+    "a_s = []\n",
+    "\n",
+    "with open('data/train.jsonl') as f:\n",
+    "    for line in f:\n",
+    "        js = json.loads(line)\n",
+    "        id = js['table_id']\n",
+    "        s = tbl_str[id]\n",
+    "        qst = js['question']\n",
+    "        nl = s + '\\nQ: ' + qst + '\\nA: '\n",
+    "        q_s.append(nl)\n",
+    "\n",
+    "        sql = js['sql']\n",
+    "        a = fix_repr(sql,tbl_cols[id],tbl_types[id],id)\n",
+    "        a = a + \"\\nEND\\n\"\n",
+    "        a_s.append(a)\n",
+    "\n",
+    "M = len(q_s)\n",
+    "\n",
+    "data_txt = [q_s[i] + a_s[i] for i in range(M)]\n",
+    "\n",
+    "for i in range(5):\n",
+    "    j = random.randint(0,M-1)\n",
+    "    print()\n",
+    "    print(data_txt[j]) \n",
+    "        \n",
+    "   "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "toks = [tokenizer(s) for s in data_txt]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "92\n",
+      "                  0\n",
+      "count  56355.000000\n",
+      "mean     101.219519\n",
+      "std       21.740325\n",
+      "min       63.000000\n",
+      "25%       87.500000\n",
+      "50%       97.000000\n",
+      "75%      109.000000\n",
+      "max      461.000000\n",
+      "32084\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "\n",
+    "print(len(toks[0].input_ids))\n",
+    "lens = np.array([len(tok.input_ids) for tok in toks])\n",
+    "print(pd.DataFrame(lens).describe())\n",
+    "\n",
+    "z = zip(q_s,lens)\n",
+    "q_red = [a for a,b in z if b < 100]\n",
+    "z = zip(a_s,lens)\n",
+    "a_red = [a for a,b in z if b < 100]\n",
+    "\n",
+    "data_red = [q_red[i] + a_red[i] for i in range(len(q_red))]\n",
+    "print(len(data_red))\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3215cace7aef45b1b040a12f11509a7d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Map:   0%|          | 0/32084 [00:00<?, ? examples/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import random, datasets\n",
+    "#d = {'prompt': random.sample(data_red, 1000)}\n",
+    "d = {'prompt': data_red}\n",
+    "\n",
+    "data = datasets.Dataset.from_dict(d)\n",
+    "data = data.map(lambda x:\n",
+    "        tokenizer(\n",
+    "        x['prompt'],\n",
+    "        truncation=True,\n",
+    "        max_length=100,\n",
+    "        padding=\"max_length\"\n",
+    "        ))\n",
+    "\n",
+    "data = data.remove_columns('prompt')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#from peft import get_peft_model,PrefixTuningConfig, TaskType, PeftType\n",
+    "from peft import get_peft_config, get_peft_model, PromptTuningInit, PromptTuningConfig, TaskType, PeftType\n",
+    "import torch\n",
+    "import transformers\n",
+    "import datasets\n",
+    "\n",
+    "BATCH = 128\n",
+    "MICRO_BATCH = 4\n",
+    "N_GAS = BATCH//MICRO_BATCH\n",
+    "EPOCHS = 1\n",
+    "LR = 1e-6\n",
+    "\n",
+    "#peft_cfg = PrefixTuningConfig(task_type=TaskType.CAUSAL_LM, num_virtual_tokens=30)\n",
+    "#model = get_peft_model(model,peft_config)\n",
+    "#model = model.to(torch.device('cuda'))\n",
+    "\n",
+    "targs = transformers.TrainingArguments(\n",
+    "    per_device_train_batch_size=MICRO_BATCH,\n",
+    "    gradient_accumulation_steps=N_GAS,\n",
+    "    warmup_steps=20,\n",
+    "    num_train_epochs=EPOCHS,\n",
+    "    learning_rate=LR,\n",
+    "    fp16=True,\n",
+    "    logging_steps=1,\n",
+    "    output_dir='sqllama-out3-rt',\n",
+    "    save_total_limit=3,\n",
+    "    remove_unused_columns=False,\n",
+    "    \n",
+    ")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "    <div>\n",
+       "      \n",
+       "      <progress value='250' max='250' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [250/250 3:05:44, Epoch 0/1]\n",
+       "    </div>\n",
+       "    <table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       " <tr style=\"text-align: left;\">\n",
+       "      <th>Step</th>\n",
+       "      <th>Training Loss</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <td>1</td>\n",
+       "      <td>1.121900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>2</td>\n",
+       "      <td>1.103000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>3</td>\n",
+       "      <td>1.128600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>4</td>\n",
+       "      <td>1.098400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>5</td>\n",
+       "      <td>1.094800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>6</td>\n",
+       "      <td>1.135100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>7</td>\n",
+       "      <td>1.114100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>8</td>\n",
+       "      <td>1.125100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>9</td>\n",
+       "      <td>1.122000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>10</td>\n",
+       "      <td>1.138500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>11</td>\n",
+       "      <td>1.114100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>12</td>\n",
+       "      <td>1.124700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>13</td>\n",
+       "      <td>1.133100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>14</td>\n",
+       "      <td>1.118900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>15</td>\n",
+       "      <td>1.121700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>16</td>\n",
+       "      <td>1.131600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>17</td>\n",
+       "      <td>1.137700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>18</td>\n",
+       "      <td>1.147000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>19</td>\n",
+       "      <td>1.113200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>20</td>\n",
+       "      <td>1.158000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>21</td>\n",
+       "      <td>1.114100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>22</td>\n",
+       "      <td>1.117700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>23</td>\n",
+       "      <td>1.151200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>24</td>\n",
+       "      <td>1.154200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>25</td>\n",
+       "      <td>1.153600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>26</td>\n",
+       "      <td>1.119700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>27</td>\n",
+       "      <td>1.138500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>28</td>\n",
+       "      <td>1.148200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>29</td>\n",
+       "      <td>1.103300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>30</td>\n",
+       "      <td>1.117700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>31</td>\n",
+       "      <td>1.116300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>32</td>\n",
+       "      <td>1.134400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>33</td>\n",
+       "      <td>1.125200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>34</td>\n",
+       "      <td>1.137900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>35</td>\n",
+       "      <td>1.150100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>36</td>\n",
+       "      <td>1.126200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>37</td>\n",
+       "      <td>1.129100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>38</td>\n",
+       "      <td>1.093200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>39</td>\n",
+       "      <td>1.153100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>40</td>\n",
+       "      <td>1.108000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>41</td>\n",
+       "      <td>1.137300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>42</td>\n",
+       "      <td>1.101600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>43</td>\n",
+       "      <td>1.140600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>44</td>\n",
+       "      <td>1.159900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>45</td>\n",
+       "      <td>1.112600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>46</td>\n",
+       "      <td>1.101200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>47</td>\n",
+       "      <td>1.088000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>48</td>\n",
+       "      <td>1.135300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>49</td>\n",
+       "      <td>1.118100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>50</td>\n",
+       "      <td>1.140300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>51</td>\n",
+       "      <td>1.104000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>52</td>\n",
+       "      <td>1.122900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>53</td>\n",
+       "      <td>1.162200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>54</td>\n",
+       "      <td>1.108500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>55</td>\n",
+       "      <td>1.121900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>56</td>\n",
+       "      <td>1.092100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>57</td>\n",
+       "      <td>1.109500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>58</td>\n",
+       "      <td>1.139400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>59</td>\n",
+       "      <td>1.120800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>60</td>\n",
+       "      <td>1.132200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>61</td>\n",
+       "      <td>1.138700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>62</td>\n",
+       "      <td>1.128700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>63</td>\n",
+       "      <td>1.122500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>64</td>\n",
+       "      <td>1.145800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>65</td>\n",
+       "      <td>1.135000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>66</td>\n",
+       "      <td>1.107900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>67</td>\n",
+       "      <td>1.120700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>68</td>\n",
+       "      <td>1.128000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>69</td>\n",
+       "      <td>1.107600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>70</td>\n",
+       "      <td>1.155700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>71</td>\n",
+       "      <td>1.142400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>72</td>\n",
+       "      <td>1.118900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>73</td>\n",
+       "      <td>1.129900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>74</td>\n",
+       "      <td>1.134400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>75</td>\n",
+       "      <td>1.105500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>76</td>\n",
+       "      <td>1.104100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>77</td>\n",
+       "      <td>1.100900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>78</td>\n",
+       "      <td>1.148200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>79</td>\n",
+       "      <td>1.116100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>80</td>\n",
+       "      <td>1.121700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>81</td>\n",
+       "      <td>1.154100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>82</td>\n",
+       "      <td>1.118900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>83</td>\n",
+       "      <td>1.109600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>84</td>\n",
+       "      <td>1.109300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>85</td>\n",
+       "      <td>1.147900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>86</td>\n",
+       "      <td>1.094300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>87</td>\n",
+       "      <td>1.130000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>88</td>\n",
+       "      <td>1.095100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>89</td>\n",
+       "      <td>1.145900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>90</td>\n",
+       "      <td>1.131600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>91</td>\n",
+       "      <td>1.114200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>92</td>\n",
+       "      <td>1.126600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>93</td>\n",
+       "      <td>1.100300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>94</td>\n",
+       "      <td>1.140900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>95</td>\n",
+       "      <td>1.132800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>96</td>\n",
+       "      <td>1.105900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>97</td>\n",
+       "      <td>1.106200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>98</td>\n",
+       "      <td>1.097400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>99</td>\n",
+       "      <td>1.114500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>100</td>\n",
+       "      <td>1.113700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>101</td>\n",
+       "      <td>1.093300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>102</td>\n",
+       "      <td>1.121900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>103</td>\n",
+       "      <td>1.133600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>104</td>\n",
+       "      <td>1.131500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>105</td>\n",
+       "      <td>1.136800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>106</td>\n",
+       "      <td>1.130800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>107</td>\n",
+       "      <td>1.102100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>108</td>\n",
+       "      <td>1.128300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>109</td>\n",
+       "      <td>1.163500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>110</td>\n",
+       "      <td>1.144200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>111</td>\n",
+       "      <td>1.125600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>112</td>\n",
+       "      <td>1.119700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>113</td>\n",
+       "      <td>1.111100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>114</td>\n",
+       "      <td>1.122400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>115</td>\n",
+       "      <td>1.142500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>116</td>\n",
+       "      <td>1.124500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>117</td>\n",
+       "      <td>1.117700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>118</td>\n",
+       "      <td>1.130500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>119</td>\n",
+       "      <td>1.118500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>120</td>\n",
+       "      <td>1.097200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>121</td>\n",
+       "      <td>1.123600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>122</td>\n",
+       "      <td>1.135700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>123</td>\n",
+       "      <td>1.153400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>124</td>\n",
+       "      <td>1.088200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>125</td>\n",
+       "      <td>1.123600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>126</td>\n",
+       "      <td>1.143000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>127</td>\n",
+       "      <td>1.121800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>128</td>\n",
+       "      <td>1.091200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>129</td>\n",
+       "      <td>1.116700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>130</td>\n",
+       "      <td>1.124400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>131</td>\n",
+       "      <td>1.139100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>132</td>\n",
+       "      <td>1.119400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>133</td>\n",
+       "      <td>1.115000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>134</td>\n",
+       "      <td>1.133600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>135</td>\n",
+       "      <td>1.100900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>136</td>\n",
+       "      <td>1.095100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>137</td>\n",
+       "      <td>1.142600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>138</td>\n",
+       "      <td>1.097300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>139</td>\n",
+       "      <td>1.113100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>140</td>\n",
+       "      <td>1.150800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>141</td>\n",
+       "      <td>1.149600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>142</td>\n",
+       "      <td>1.106700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>143</td>\n",
+       "      <td>1.086100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>144</td>\n",
+       "      <td>1.134200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>145</td>\n",
+       "      <td>1.096400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>146</td>\n",
+       "      <td>1.099200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>147</td>\n",
+       "      <td>1.168300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>148</td>\n",
+       "      <td>1.105900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>149</td>\n",
+       "      <td>1.119700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>150</td>\n",
+       "      <td>1.100200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>151</td>\n",
+       "      <td>1.089600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>152</td>\n",
+       "      <td>1.128200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>153</td>\n",
+       "      <td>1.148300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>154</td>\n",
+       "      <td>1.119800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>155</td>\n",
+       "      <td>1.102700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>156</td>\n",
+       "      <td>1.107800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>157</td>\n",
+       "      <td>1.113100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>158</td>\n",
+       "      <td>1.156100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>159</td>\n",
+       "      <td>1.091500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>160</td>\n",
+       "      <td>1.118000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>161</td>\n",
+       "      <td>1.145600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>162</td>\n",
+       "      <td>1.115400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>163</td>\n",
+       "      <td>1.121900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>164</td>\n",
+       "      <td>1.130100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>165</td>\n",
+       "      <td>1.123400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>166</td>\n",
+       "      <td>1.090900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>167</td>\n",
+       "      <td>1.144400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>168</td>\n",
+       "      <td>1.125100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>169</td>\n",
+       "      <td>1.110700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>170</td>\n",
+       "      <td>1.134300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>171</td>\n",
+       "      <td>1.092600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>172</td>\n",
+       "      <td>1.123000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>173</td>\n",
+       "      <td>1.080100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>174</td>\n",
+       "      <td>1.104100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>175</td>\n",
+       "      <td>1.105800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>176</td>\n",
+       "      <td>1.156000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>177</td>\n",
+       "      <td>1.104000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>178</td>\n",
+       "      <td>1.118500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>179</td>\n",
+       "      <td>1.123100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>180</td>\n",
+       "      <td>1.117000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>181</td>\n",
+       "      <td>1.122100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>182</td>\n",
+       "      <td>1.141200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>183</td>\n",
+       "      <td>1.135600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>184</td>\n",
+       "      <td>1.093600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>185</td>\n",
+       "      <td>1.156300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>186</td>\n",
+       "      <td>1.095600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>187</td>\n",
+       "      <td>1.128900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>188</td>\n",
+       "      <td>1.101200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>189</td>\n",
+       "      <td>1.149900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>190</td>\n",
+       "      <td>1.112300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>191</td>\n",
+       "      <td>1.117600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>192</td>\n",
+       "      <td>1.090600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>193</td>\n",
+       "      <td>1.097700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>194</td>\n",
+       "      <td>1.084700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>195</td>\n",
+       "      <td>1.128900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>196</td>\n",
+       "      <td>1.126400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>197</td>\n",
+       "      <td>1.113000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>198</td>\n",
+       "      <td>1.107500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>199</td>\n",
+       "      <td>1.160100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>200</td>\n",
+       "      <td>1.125800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>201</td>\n",
+       "      <td>1.125300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>202</td>\n",
+       "      <td>1.127200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>203</td>\n",
+       "      <td>1.114200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>204</td>\n",
+       "      <td>1.114300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>205</td>\n",
+       "      <td>1.119200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>206</td>\n",
+       "      <td>1.114500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>207</td>\n",
+       "      <td>1.086100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>208</td>\n",
+       "      <td>1.096200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>209</td>\n",
+       "      <td>1.115800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>210</td>\n",
+       "      <td>1.094500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>211</td>\n",
+       "      <td>1.106400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>212</td>\n",
+       "      <td>1.121400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>213</td>\n",
+       "      <td>1.137600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>214</td>\n",
+       "      <td>1.107000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>215</td>\n",
+       "      <td>1.095700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>216</td>\n",
+       "      <td>1.083000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>217</td>\n",
+       "      <td>1.088700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>218</td>\n",
+       "      <td>1.133700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>219</td>\n",
+       "      <td>1.115500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>220</td>\n",
+       "      <td>1.152900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>221</td>\n",
+       "      <td>1.100100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>222</td>\n",
+       "      <td>1.112500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>223</td>\n",
+       "      <td>1.119200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>224</td>\n",
+       "      <td>1.122600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>225</td>\n",
+       "      <td>1.100100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>226</td>\n",
+       "      <td>1.082500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>227</td>\n",
+       "      <td>1.094800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>228</td>\n",
+       "      <td>1.123600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>229</td>\n",
+       "      <td>1.124700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>230</td>\n",
+       "      <td>1.148800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>231</td>\n",
+       "      <td>1.109600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>232</td>\n",
+       "      <td>1.096100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>233</td>\n",
+       "      <td>1.123000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>234</td>\n",
+       "      <td>1.102200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>235</td>\n",
+       "      <td>1.113200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>236</td>\n",
+       "      <td>1.150700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>237</td>\n",
+       "      <td>1.131900</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>238</td>\n",
+       "      <td>1.107200</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>239</td>\n",
+       "      <td>1.137600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>240</td>\n",
+       "      <td>1.094800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>241</td>\n",
+       "      <td>1.068000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>242</td>\n",
+       "      <td>1.122100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>243</td>\n",
+       "      <td>1.153700</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>244</td>\n",
+       "      <td>1.045100</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>245</td>\n",
+       "      <td>1.131400</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>246</td>\n",
+       "      <td>1.134600</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>247</td>\n",
+       "      <td>1.105300</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>248</td>\n",
+       "      <td>1.108800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>249</td>\n",
+       "      <td>1.080800</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <td>250</td>\n",
+       "      <td>1.119200</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table><p>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "trainer = transformers.Trainer(\n",
+    "    model = model,\n",
+    "    train_dataset = data,\n",
+    "    args = targs,\n",
+    "    #data_collator=transformers.DefaultDataCollator\n",
+    "    data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False)\n",
+    ")\n",
+    "trainer.train(resume_from_checkpoint=False)\n",
+    "model.save_pretrained('sqllama-out3')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "from model\n",
+      "<unk>table: 2-10806592-14\n",
+      "columns: Home team,Home team score,Away team,Away team score,Venue,Crowd,Date\n",
+      "Q: What was the away score when the home team was Melbourne?\n",
+      "A:  SELECT  Home team,Home team score,Away team,Away team score,Venue,Crowd,Date\n",
+      "FROM 2-1080\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  Away team score FROM 2-10806592-14 WHERE Home team = 'melbourne'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 2-17978030-6\n",
+      "columns: Date,Time,Score,Set 1,Set 2,Set 3,Total\n",
+      "Q: What is the score when the set 3 is 26–28?\n",
+      "A:  SELECT  Score FROM 2-17978030-6 WHERE Set 3 = 26–28\n",
+      "END\n",
+      "\\end{code}\n",
+      "\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  Score FROM 2-17978030-6 WHERE Set 3 = '26–28'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 2-12487101-6\n",
+      "columns: Position,Number,Name,Class,Injury (Status),Last Update\n",
+      "Q: for the position of rb what is the name?\n",
+      "A:  SELECT Name FROM 2-12487101-6 WHERE Position = 'rb'\n",
+      "END\n",
+      "\\end{code}\n",
+      "\n",
+      "Comment: I'm not sure what you mean by \"the name of\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  Name FROM 2-12487101-6 WHERE Position = 'rb'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 2-16780011-2\n",
+      "columns: Week,Date,Opponent,Result,TV Time,Attendance\n",
+      "Q: Which week had an attendance of 55,158?\n",
+      "A:  SELECT  Attendance FROM 2-16780011-2 WHERE Week = '2019-01-06'\n",
+      "END\n",
+      "\\end{code}\n",
+      "\n",
+      "\n",
+      "\n",
+      "expected answer\n",
+      "SELECT SUM Week FROM 2-16780011-2 WHERE Attendance = '55,158'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 1-1671401-1\n",
+      "columns: Year,Starts,Wins,Top 5,Top 10,Poles,Avg. Start,Avg. Finish,Winnings,Position,Team(s)\n",
+      "Q: Name the least top 5\n",
+      "A:  SELECT MIN Top 5 FROM 1-1671401-1\n",
+      "END\n",
+      "\\end{code}\n",
+      "\n",
+      "\n",
+      "expected answer\n",
+      "SELECT MIN Top 5 FROM 1-1671401-1\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 2-17540875-4\n",
+      "columns: Tie no,Home team,Score,Away team,Date\n",
+      "Q: What team was the home team when Manchester City was the away team?\n",
+      "A:  SELECT  Tie no,Home team FROM 2-17540875-4 WHERE Away team = 'Manchester City'\n",
+      "END\n",
+      "\n",
+      "Comment: @user378326\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  Home team FROM 2-17540875-4 WHERE Away team = 'manchester city'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 2-15524351-11\n",
+      "columns: Date,Opponent,Location,Result,Attendance\n",
+      "Q: What team was the opponent when the result was l 0-27?\n",
+      "A:  SELECT  Opponent FROM 2-15524351-11 WHERE Result = 'l 0-27'\n",
+      "END\n",
+      "\\end{code}\n",
+      "\n",
+      "Comment\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  Opponent FROM 2-15524351-11 WHERE Result = 'l 0-27'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 1-12962773-1\n",
+      "columns: No,Player,Height,Position,Year born,Current Club\n",
+      "Q: what's current club with height being 2.09\n",
+      "A:  SELECT  Current Club\n",
+      "FROM 1-12962773-1\n",
+      "WHERE Height = 2.09\n",
+      "END\n",
+      "\\end{code}\n",
+      "\n",
+      "Comment: I'm\n",
+      "\n",
+      "expected answer\n",
+      "SELECT  Current Club FROM 1-12962773-1 WHERE Height = '2.09'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 1-12441518-1\n",
+      "columns: Character,Portrayed by,Main cast seasons,Recurring cast seasons,# of episodes\n",
+      "Q: How many people named Nick Lucas are on the show?\n",
+      "A:  SELECT  COUNT(Character) FROM 1-12441518-1 WHERE Character = 'Nick Lucas'\n",
+      "END\n",
+      "\n",
+      "Comment: @user3783264,\n",
+      "\n",
+      "expected answer\n",
+      "SELECT COUNT Portrayed by FROM 1-12441518-1 WHERE Character = 'Nick Lucas'\n",
+      "END\n",
+      "\n",
+      "from model\n",
+      "<unk>table: 1-17176509-4\n",
+      "columns: Position,Driver / Passenger,Equipment,Bike No,Points\n",
+      "Q: Name the driver/passenger for 30\n",
+      "A:  SELECT  Driver / Passenger\n",
+      "FROM 1-17176509-4\n",
+      "WHERE 30 = 'Driver / Passenger'\n",
+      "END\n",
+      "\n",
+      "Comment: I'm not sure what you mean by\n",
+      "\n",
+      "expected answer\n",
+      "SELECT COUNT Driver / Passenger FROM 1-17176509-4 WHERE Position = 30\n",
+      "END\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "def get_query(q):\n",
+    "    \n",
+    "    toks = tokenizer(q , return_tensors='pt')\n",
+    "    ctoks = toks.input_ids.to('cuda')\n",
+    "    gen = model.generate(ctoks, max_length=100)\n",
+    "    return tokenizer.decode(gen[0])\n",
+    "\n",
+    "M = len(q_red)\n",
+    "\n",
+    "for _ in range(10):\n",
+    "    j = random.randint(0,M-1)\n",
+    "    qs = q_red[j]\n",
+    "    a = a_red[j]\n",
+    "\n",
+    "    ma = get_query(qs)\n",
+    "\n",
+    "    #print(qs)\n",
+    "    print('from model')\n",
+    "    print(ma)\n",
+    "    print()\n",
+    "    print('expected answer')\n",
+    "    print(a)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.3"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "6a381460736e8a0eabfb35eafae436ba15c06439de44e28b965ea473bd8dda90"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}