{
 "cells": [
  {
   "metadata": {},
   "cell_type": "raw",
   "source": "",
   "id": "f50fa4bf6edb488e"
  },
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "import pandas as pd\n",
    "from datasets import load_dataset\n",
    "from processor import Preprocessor\n",
    "from similarity import SentenceTransformerSimilarity, BertSimilarity, TFIDFSimilarity\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T08:44:53.568166Z",
     "start_time": "2025-07-08T08:44:53.563295Z"
    }
   },
   "cell_type": "code",
   "source": "data_path = \"cnamuangtoun/resume-job-description-fit\"",
   "id": "c5adb8b5a5057b1b",
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T08:44:59.342259Z",
     "start_time": "2025-07-08T08:44:55.506716Z"
    }
   },
   "cell_type": "code",
   "source": [
    "datasets = load_dataset(data_path)\n",
    "\n",
    "train_dataset = datasets['train']\n",
    "test_dataset = datasets['test']"
   ],
   "id": "ee31d8d2097183ae",
   "outputs": [],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T08:45:14.160133Z",
     "start_time": "2025-07-08T08:45:09.061062Z"
    }
   },
   "cell_type": "code",
   "source": [
    "preprocessor = Preprocessor()\n",
    "sentence_transformer = SentenceTransformerSimilarity(\"mixedbread-ai/mxbai-embed-large-v1\")\n",
    "bert_similarity = BertSimilarity()\n",
    "tfidf_similarity = TFIDFSimilarity()"
   ],
   "id": "8a5ab5d98b0212c2",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T08:45:15.854458Z",
     "start_time": "2025-07-08T08:45:15.850088Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def compute_similarity(example):\n",
    "    resume = preprocessor.preprocess(example[\"resume_text\"])\n",
    "    job_description = preprocessor.preprocess(example[\"job_description_text\"])\n",
    "    label = 0 if example[\"label\"] == \"No Fit\" else 1\n",
    "\n",
    "    tfidf = tfidf_similarity.similarity(resume, job_description)\n",
    "    bert = bert_similarity.similarity(resume, job_description)\n",
    "    transformer = sentence_transformer.similarity(resume, job_description)\n",
    "\n",
    "    return {\n",
    "        \"resume\": resume, \"job_description\": job_description, \"label\": label,\n",
    "        \"tfidf_similarity\": tfidf, \"bert_similarity\": bert, \"sentence_transformer_similarity\": transformer\n",
    "    }"
   ],
   "id": "1edb7165aa0fa91a",
   "outputs": [],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T09:52:30.395538Z",
     "start_time": "2025-07-08T08:45:18.599810Z"
    }
   },
   "cell_type": "code",
   "source": "test_dataset = test_dataset.map(compute_similarity)",
   "id": "49885541fea7acf9",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|██████████| 1759/1759 [1:06:38<00:00,  2.27s/ examples]\n"
     ]
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-05T10:41:18.629642Z",
     "start_time": "2025-07-05T10:41:18.622846Z"
    }
   },
   "cell_type": "code",
   "source": "test_dataset",
   "id": "abf86ee1b574eee8",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['resume_text', 'job_description_text', 'label', 'resume', 'job_description', 'tfidf_similarity', 'bert_similarity', 'sentence_transformer_similarity'],\n",
       "    num_rows: 1759\n",
       "})"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 14
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T10:13:38.193054Z",
     "start_time": "2025-07-08T10:13:38.189061Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def convert_to_label(example):\n",
    "    tfidf = 0 if example[\"tfidf_similarity\"] < 0.5 else 1\n",
    "    bert = 0 if example[\"bert_similarity\"] < 0.5 else 1\n",
    "    transformer = 0 if example[\"sentence_transformer_similarity\"] < 0.65 else 1\n",
    "\n",
    "    return {\"tfidf\": tfidf, \"bert\": bert, \"transformer\": transformer}\n"
   ],
   "id": "9f6448edfb548f6c",
   "outputs": [],
   "execution_count": 73
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T10:13:39.971426Z",
     "start_time": "2025-07-08T10:13:39.830089Z"
    }
   },
   "cell_type": "code",
   "source": "test_dataset = test_dataset.map(convert_to_label)",
   "id": "2d688ab6cef016f3",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|██████████| 1759/1759 [00:00<00:00, 12958.71 examples/s]\n"
     ]
    }
   ],
   "execution_count": 74
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T10:13:41.559694Z",
     "start_time": "2025-07-08T10:13:41.555362Z"
    }
   },
   "cell_type": "code",
   "source": "test_dataset",
   "id": "e1ff007dcc6a9974",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['resume_text', 'job_description_text', 'label', 'resume', 'job_description', 'tfidf_similarity', 'bert_similarity', 'sentence_transformer_similarity', 'tfidf', 'bert', 'transformer'],\n",
       "    num_rows: 1759\n",
       "})"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 75
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T10:14:18.161742Z",
     "start_time": "2025-07-08T10:14:18.155650Z"
    }
   },
   "cell_type": "code",
   "source": [
    "actual = test_dataset[\"label\"]\n",
    "bert_predict = test_dataset[\"bert\"]\n",
    "tfidf_predict = test_dataset[\"tfidf\"]\n",
    "transformer_predict = test_dataset[\"transformer\"]"
   ],
   "id": "a86f6993fe289ac6",
   "outputs": [],
   "execution_count": 76
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T10:14:31.353217Z",
     "start_time": "2025-07-08T10:14:31.349873Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def evaluate(actual, predict):\n",
    "    accuracy = accuracy_score(actual, predict)\n",
    "    precision = precision_score(actual, predict)\n",
    "    recall = recall_score(actual, predict)\n",
    "    f1 = f1_score(actual, predict)\n",
    "    \n",
    "    return {\"accuracy\": accuracy, \"precision\": precision, \"recall\": recall, \"f1\": f1}"
   ],
   "id": "b838d444f65bdc9b",
   "outputs": [],
   "execution_count": 79
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T10:14:32.609153Z",
     "start_time": "2025-07-08T10:14:32.583900Z"
    }
   },
   "cell_type": "code",
   "source": [
    "bert_evaluate = evaluate(actual, bert_predict)\n",
    "tfidf_evaluate = evaluate(actual, tfidf_predict)\n",
    "transformer_evaluate = evaluate(actual, transformer_predict)"
   ],
   "id": "6aefb66598a946a8",
   "outputs": [],
   "execution_count": 80
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T10:14:34.137687Z",
     "start_time": "2025-07-08T10:14:34.133737Z"
    }
   },
   "cell_type": "code",
   "source": [
    "print(\"bert evaluate : \", bert_evaluate)\n",
    "print(\"tfidf evaluate : \", tfidf_evaluate)\n",
    "print(\"transformer evaluate : \", transformer_evaluate)"
   ],
   "id": "f529a915f203f1f2",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert evaluate :  {'accuracy': 0.5127913587265491, 'precision': 0.5127913587265491, 'recall': 1.0, 'f1': 0.6779406238256295}\n",
      "tfidf evaluate :  {'accuracy': 0.4860716316088687, 'precision': 0.0, 'recall': 0.0, 'f1': 0.0}\n",
      "transformer evaluate :  {'accuracy': 0.6031836270608301, 'precision': 0.5791925465838509, 'recall': 0.8270509977827051, 'f1': 0.6812785388127854}\n"
     ]
    }
   ],
   "execution_count": 81
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T10:14:35.938911Z",
     "start_time": "2025-07-08T10:14:35.935413Z"
    }
   },
   "cell_type": "code",
   "source": [
    "data = {\n",
    "    'BERT': bert_evaluate,\n",
    "    'TF-IDF': tfidf_evaluate,\n",
    "    'Transformer': transformer_evaluate\n",
    "}\n",
    "\n",
    "dataframe = pd.DataFrame(data)"
   ],
   "id": "b9260a6b2ba65916",
   "outputs": [],
   "execution_count": 82
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T09:56:17.793633Z",
     "start_time": "2025-07-08T09:56:17.786996Z"
    }
   },
   "cell_type": "code",
   "source": "dataframe",
   "id": "e67956d4d96cfb3",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "               BERT    TF-IDF  Transformer\n",
       "accuracy   0.512791  0.486072     0.603184\n",
       "precision  0.512791  0.000000     0.579193\n",
       "recall     1.000000  0.000000     0.827051\n",
       "f1         0.677941  0.000000     0.681279"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BERT</th>\n",
       "      <th>TF-IDF</th>\n",
       "      <th>Transformer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.512791</td>\n",
       "      <td>0.486072</td>\n",
       "      <td>0.603184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.512791</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.579193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.827051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.677941</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.681279</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 40
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T09:56:37.438085Z",
     "start_time": "2025-07-08T09:56:37.425383Z"
    }
   },
   "cell_type": "code",
   "source": "dataframe.to_csv(\"evaluation.csv\")",
   "id": "a14601b6c4afef68",
   "outputs": [],
   "execution_count": 41
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}