{ "cells": [ { "cell_type": "code", "execution_count": 2, "id": "761980ec", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "āœ… Created/Verified utils/scalers directory\n", "\n", "==================================================\n", "šŸ” CHECKING FOR EXISTING SCALERS\n", "==================================================\n", "āœ… continuous_scaler.pkl exists (0.49 KB) - Temperature scaler (StandardScaler for Avg_Temp and Soil)\n", "āœ… days_scaler.pkl exists (0.49 KB) - Days scaler (MinMaxScaler for Number_of_Days)\n", "āœ… nitrogen_map.pkl exists (0.05 KB) - Nitrogen mapping (fertilizer to categories)\n", "āœ… scaler_y.pkl exists (0.46 KB) - Target scaler (StandardScaler for Nitrogen_Cont)\n", "āœ… temperature_means.pkl exists (0.04 KB)\n", "\n", "==================================================\n", "āœ… ALL SCALERS ARE PRESENT!\n", "==================================================\n", "\n", "The web application will use these scalers for predictions.\n", "No further action needed.\n", "\n", "==================================================\n", "šŸ” VERIFYING SCALER INTEGRITY\n", "==================================================\n", "āœ… All scalers loaded successfully\n", "\n", "šŸ“Š Testing continuous_scaler (temperature):\n", " Input temps: [20. 25.] ā†’ Scaled: [ 1.1058507 -0.14331882]\n", " Input temps: [25. 30.] ā†’ Scaled: [2.334751 0.69844556]\n", " Input temps: [30. 35.] ā†’ Scaled: [3.5636516 1.5402099]\n", "\n", "šŸ“Š Testing days_scaler:\n", " Input days: 60.0 ā†’ Scaled: -0.070\n", " Input days: 90.0 ā†’ Scaled: 0.628\n", " Input days: 120.0 ā†’ Scaled: 1.326\n", "\n", "šŸ“Š Testing nitrogen_map:\n", " Fertilizer 0 kg/ha ā†’ Category 0\n", " Fertilizer 30 kg/ha ā†’ Category 1\n", " Fertilizer 60 kg/ha ā†’ Category 2\n", " Fertilizer 90 kg/ha ā†’ Category 3\n", " Fertilizer 120 kg/ha ā†’ Category 4\n", " Fertilizer 150 kg/ha ā†’ Category 5\n", " Fertilizer 180 kg/ha ā†’ Category 6\n", " Fertilizer 210 kg/ha ā†’ Category 7\n", "\n", "šŸ“Š Testing scaler_y inverse transform:\n", " Normalized 0.50 ā†’ Original 3.99%\n", " Normalized 1.00 ā†’ Original 4.35%\n", " Normalized 1.50 ā†’ Original 4.71%\n", "\n", "šŸ“Š Testing temperature_means:\n", " Soil Temperature Mean: 29.8Ā°C\n", "\n", "==================================================\n", "āœ… ALL SCALERS ARE VALID AND READY FOR USE!\n", "==================================================\n", "\n", "==================================================\n", "šŸ“ SCALER FILES SUMMARY\n", "==================================================\n", "\n", "āœ… All scaler files are present in utils/scalers/:\n", " ā€¢ continuous_scaler.pkl (0.49 KB)\n", " ā€¢ days_scaler.pkl (0.49 KB)\n", " ā€¢ nitrogen_map.pkl (0.05 KB)\n", " ā€¢ scaler_y.pkl (0.46 KB)\n", " ā€¢ temperature_means.pkl (0.04 KB)\n", "\n", "āœ… The web application will use these scalers for:\n", " ā€¢ Scaling temperature features (air and soil temperature)\n", " ā€¢ Scaling days after sowing (60-120 days range)\n", " ā€¢ Mapping fertilizer amounts to categories (0-7)\n", " ā€¢ Denormalizing model predictions to actual nitrogen content\n", " ā€¢ Providing soil temperature mean for missing values\n", "\n", "==================================================\n", "šŸŽ‰ SCALER SETUP COMPLETE!\n", "==================================================\n" ] } ], "source": [ "# %% [markdown]\n", "# # Step 5: Verify Scalers for Web Application\n", "# ## Check that all scalers are properly saved for the Flask app\n", "\n", "# %%\n", "import os\n", "import pickle\n", "import numpy as np\n", "\n", "# %% [markdown]\n", "# ### Verify scalers directory exists\n", "\n", "# %%\n", "# Create utils/scalers directory if it doesn't exist\n", "os.makedirs('utils/scalers', exist_ok=True)\n", "print(\"āœ… Created/Verified utils/scalers directory\")\n", "\n", "# %% [markdown]\n", "# ### Check if scalers are already saved\n", "\n", "# %%\n", "print(\"\\n\" + \"=\"*50)\n", "print(\"šŸ” CHECKING FOR EXISTING SCALERS\")\n", "print(\"=\"*50)\n", "\n", "# Define scaler files to check\n", "scaler_files = {\n", " 'continuous_scaler.pkl': 'Temperature scaler (StandardScaler for Avg_Temp and Soil)',\n", " 'days_scaler.pkl': 'Days scaler (MinMaxScaler for Number_of_Days)',\n", " 'nitrogen_map.pkl': 'Nitrogen mapping (fertilizer to categories)',\n", " 'scaler_y.pkl': 'Target scaler (StandardScaler for Nitrogen_Cont)'\n", "}\n", "\n", "all_scalers_exist = True\n", "existing_scalers = []\n", "missing_scalers = []\n", "\n", "for filename, description in scaler_files.items():\n", " filepath = os.path.join('utils/scalers', filename)\n", " if os.path.exists(filepath):\n", " file_size = os.path.getsize(filepath) / 1024 # KB\n", " existing_scalers.append(filename)\n", " print(f\"āœ… {filename} exists ({file_size:.2f} KB) - {description}\")\n", " else:\n", " missing_scalers.append(filename)\n", " print(f\"āŒ {filename} missing - {description}\")\n", " all_scalers_exist = False\n", "\n", "# Check temperature means\n", "temp_means_path = os.path.join('utils', 'temperature_means.pkl')\n", "if os.path.exists(temp_means_path):\n", " file_size = os.path.getsize(temp_means_path) / 1024\n", " print(f\"āœ… temperature_means.pkl exists ({file_size:.2f} KB)\")\n", "else:\n", " print(f\"āŒ temperature_means.pkl missing\")\n", " all_scalers_exist = False\n", "\n", "# %% [markdown]\n", "# ### If scalers are missing, provide instructions to create them\n", "\n", "# %%\n", "if not all_scalers_exist:\n", " print(\"\\n\" + \"=\"*50)\n", " print(\"āš ļø SCALERS MISSING - INSTRUCTIONS\")\n", " print(\"=\"*50)\n", " print(\"\\nTo create the missing scalers, you need to:\")\n", " print(\"1. Place your training Excel file in the 'data' folder\")\n", " print(\"2. Update the path below to your training data\")\n", " print(\"3. Run the code in the next cell\")\n", " print(\"\\nMissing scalers: \" + \", \".join(missing_scalers))\n", " \n", " # Optional: Create a data folder if it doesn't exist\n", " os.makedirs('data', exist_ok=True)\n", " print(\"\\nāœ… Created 'data' folder for training data\")\n", " \n", "else:\n", " print(\"\\n\" + \"=\"*50)\n", " print(\"āœ… ALL SCALERS ARE PRESENT!\")\n", " print(\"=\"*50)\n", " print(\"\\nThe web application will use these scalers for predictions.\")\n", " print(\"No further action needed.\")\n", "\n", "# %% [markdown]\n", "# ### If scalers are missing, uncomment and run this cell to create them\n", "# ### Place your Train.xlsx file in the 'data' folder first\n", "\n", "# %%\n", "# UNCOMMENT THE FOLLOWING CODE IF SCALERS ARE MISSING\n", "\"\"\"\n", "import pandas as pd\n", "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n", "\n", "# Path to your training data (place Excel file in 'data' folder)\n", "TRAIN_EXCEL_PATH = os.path.join('data', 'Train.xlsx')\n", "\n", "if os.path.exists(TRAIN_EXCEL_PATH):\n", " try:\n", " # Load training data\n", " df = pd.read_excel(TRAIN_EXCEL_PATH, header=0)\n", " print(f\"āœ… Loaded {len(df)} samples from training data\")\n", " \n", " # Extract features\n", " temperature_features = df[['Avg_Temp', 'Soil']].values.astype('float32')\n", " days = df['Number_of_Days'].values.reshape(-1, 1).astype('float32')\n", " nitrogen_content = df['Nitrogen_Cont'].values.reshape(-1, 1).astype('float32')\n", " \n", " # Create and fit temperature scaler (StandardScaler)\n", " continuous_scaler = StandardScaler()\n", " continuous_scaler.fit(temperature_features)\n", " print(f\"āœ… Fitted continuous_scaler on {temperature_features.shape[0]} samples\")\n", " print(f\" Temperature means: {continuous_scaler.mean_}\")\n", " \n", " # Create and fit days scaler (MinMaxScaler)\n", " days_scaler = MinMaxScaler()\n", " days_scaler.fit(days)\n", " print(f\"āœ… Fitted days_scaler on {days.shape[0]} samples\")\n", " print(f\" Days range: [{days.min()}, {days.max()}] -> [{days_scaler.data_min_[0]:.2f}, {days_scaler.data_max_[0]:.2f}]\")\n", " \n", " # Create and fit target scaler (StandardScaler for nitrogen)\n", " scaler_y = StandardScaler()\n", " scaler_y.fit(nitrogen_content)\n", " print(f\"āœ… Fitted scaler_y on {nitrogen_content.shape[0]} samples\")\n", " print(f\" Nitrogen mean: {scaler_y.mean_[0]:.4f}, std: {scaler_y.scale_[0]:.4f}\")\n", " \n", " # Save all scalers\n", " with open('utils/scalers/continuous_scaler.pkl', 'wb') as f:\n", " pickle.dump(continuous_scaler, f)\n", " print(\"āœ… Saved continuous_scaler.pkl\")\n", " \n", " with open('utils/scalers/days_scaler.pkl', 'wb') as f:\n", " pickle.dump(days_scaler, f)\n", " print(\"āœ… Saved days_scaler.pkl\")\n", " \n", " with open('utils/scalers/scaler_y.pkl', 'wb') as f:\n", " pickle.dump(scaler_y, f)\n", " print(\"āœ… Saved scaler_y.pkl\")\n", " \n", " # Save nitrogen mapping\n", " nitrogen_map = {0:0, 30:1, 60:2, 90:3, 120:4, 150:5, 180:6, 210:7}\n", " with open('utils/scalers/nitrogen_map.pkl', 'wb') as f:\n", " pickle.dump(nitrogen_map, f)\n", " print(\"āœ… Saved nitrogen_map.pkl\")\n", " \n", " # Save temperature means\n", " temperature_values = {'soil_temp_mean': 29.8}\n", " with open('utils/temperature_means.pkl', 'wb') as f:\n", " pickle.dump(temperature_values, f)\n", " print(\"āœ… Saved temperature_means.pkl\")\n", " \n", " print(\"\\nāœ… All scalers created successfully!\")\n", " \n", " except Exception as e:\n", " print(f\"āŒ Error creating scalers: {e}\")\n", " print(\" Please check your Excel file format and columns\")\n", "else:\n", " print(f\"āŒ Training data not found at: {TRAIN_EXCEL_PATH}\")\n", " print(\" Please place your Train.xlsx file in the 'data' folder\")\n", "\"\"\"\n", "\n", "# %% [markdown]\n", "# ### Verify all scalers are valid\n", "\n", "# %%\n", "print(\"\\n\" + \"=\"*50)\n", "print(\"šŸ” VERIFYING SCALER INTEGRITY\")\n", "print(\"=\"*50)\n", "\n", "try:\n", " # Load and test scalers if they exist\n", " if all_scalers_exist:\n", " # Load scalers\n", " with open('utils/scalers/continuous_scaler.pkl', 'rb') as f:\n", " continuous_scaler = pickle.load(f)\n", " \n", " with open('utils/scalers/days_scaler.pkl', 'rb') as f:\n", " days_scaler = pickle.load(f)\n", " \n", " with open('utils/scalers/nitrogen_map.pkl', 'rb') as f:\n", " nitrogen_map = pickle.load(f)\n", " \n", " with open('utils/scalers/scaler_y.pkl', 'rb') as f:\n", " scaler_y = pickle.load(f)\n", " \n", " with open('utils/temperature_means.pkl', 'rb') as f:\n", " temp_means = pickle.load(f)\n", " \n", " print(\"āœ… All scalers loaded successfully\")\n", " \n", " # Test continuous scaler\n", " print(\"\\nšŸ“Š Testing continuous_scaler (temperature):\")\n", " test_temps = np.array([[20, 25], [25, 30], [30, 35]], dtype=np.float32)\n", " scaled_temps = continuous_scaler.transform(test_temps)\n", " print(f\" Input temps: {test_temps[0]} ā†’ Scaled: {scaled_temps[0]}\")\n", " print(f\" Input temps: {test_temps[1]} ā†’ Scaled: {scaled_temps[1]}\")\n", " print(f\" Input temps: {test_temps[2]} ā†’ Scaled: {scaled_temps[2]}\")\n", " \n", " # Test days scaler\n", " print(\"\\nšŸ“Š Testing days_scaler:\")\n", " test_days = np.array([[60], [90], [120]], dtype=np.float32)\n", " scaled_days = days_scaler.transform(test_days)\n", " print(f\" Input days: {test_days[0][0]} ā†’ Scaled: {scaled_days[0][0]:.3f}\")\n", " print(f\" Input days: {test_days[1][0]} ā†’ Scaled: {scaled_days[1][0]:.3f}\")\n", " print(f\" Input days: {test_days[2][0]} ā†’ Scaled: {scaled_days[2][0]:.3f}\")\n", " \n", " # Test nitrogen mapping\n", " print(\"\\nšŸ“Š Testing nitrogen_map:\")\n", " test_fertilizers = [0, 30, 60, 90, 120, 150, 180, 210]\n", " for fert in test_fertilizers:\n", " category = nitrogen_map.get(fert, 0)\n", " print(f\" Fertilizer {fert} kg/ha ā†’ Category {category}\")\n", " \n", " # Test inverse transform\n", " print(\"\\nšŸ“Š Testing scaler_y inverse transform:\")\n", " test_predictions = np.array([[0.5], [1.0], [1.5]], dtype=np.float32)\n", " original_values = scaler_y.inverse_transform(test_predictions)\n", " for i, (pred, orig) in enumerate(zip(test_predictions, original_values)):\n", " print(f\" Normalized {pred[0]:.2f} ā†’ Original {orig[0]:.2f}%\")\n", " \n", " # Test temperature means\n", " print(\"\\nšŸ“Š Testing temperature_means:\")\n", " print(f\" Soil Temperature Mean: {temp_means.get('soil_temp_mean', 'N/A')}Ā°C\")\n", " \n", " print(\"\\n\" + \"=\"*50)\n", " print(\"āœ… ALL SCALERS ARE VALID AND READY FOR USE!\")\n", " print(\"=\"*50)\n", " \n", " else:\n", " print(\"\\nāš ļø Cannot verify scalers because some are missing.\")\n", " print(\" Please create the missing scalers first.\")\n", " \n", "except Exception as e:\n", " print(f\"āŒ Verification failed: {e}\")\n", "\n", "# %% [markdown]\n", "# ### Summary\n", "\n", "# %%\n", "print(\"\\n\" + \"=\"*50)\n", "print(\"šŸ“ SCALER FILES SUMMARY\")\n", "print(\"=\"*50)\n", "\n", "if all_scalers_exist:\n", " print(\"\\nāœ… All scaler files are present in utils/scalers/:\")\n", " for filename in scaler_files.keys():\n", " filepath = os.path.join('utils/scalers', filename)\n", " if os.path.exists(filepath):\n", " file_size = os.path.getsize(filepath) / 1024\n", " print(f\" ā€¢ {filename} ({file_size:.2f} KB)\")\n", " \n", " if os.path.exists(temp_means_path):\n", " file_size = os.path.getsize(temp_means_path) / 1024\n", " print(f\" ā€¢ temperature_means.pkl ({file_size:.2f} KB)\")\n", " \n", " print(\"\\nāœ… The web application will use these scalers for:\")\n", " print(\" ā€¢ Scaling temperature features (air and soil temperature)\")\n", " print(\" ā€¢ Scaling days after sowing (60-120 days range)\")\n", " print(\" ā€¢ Mapping fertilizer amounts to categories (0-7)\")\n", " print(\" ā€¢ Denormalizing model predictions to actual nitrogen content\")\n", " print(\" ā€¢ Providing soil temperature mean for missing values\")\n", " \n", "else:\n", " print(\"\\nāš ļø Some scaler files are missing. Please create them using the instructions above.\")\n", " print(\"\\nRequired files for web application:\")\n", " for filename in scaler_files.keys():\n", " print(f\" ā€¢ utils/scalers/{filename}\")\n", " print(f\" ā€¢ utils/temperature_means.pkl\")\n", "\n", "print(\"\\n\" + \"=\"*50)\n", "print(\"šŸŽ‰ SCALER SETUP COMPLETE!\")\n", "print(\"=\"*50)" ] }, { "cell_type": "code", "execution_count": null, "id": "25c62665", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.5" } }, "nbformat": 4, "nbformat_minor": 5 }