{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "-iRvitW_mOmI"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "ZTL4F90RnqNA"
   },
   "outputs": [],
   "source": [
    "# https://www.statsmodels.org/stable/index.html\n",
    "import statsmodels.api as sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "fK4vZwBPnA5z"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Record</th>\n",
       "      <th>Spam</th>\n",
       "      <th>Recipients</th>\n",
       "      <th>Hyperlinks</th>\n",
       "      <th>Characters</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>11</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>496</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>97</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>497</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>5</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>498</td>\n",
       "      <td>1</td>\n",
       "      <td>41</td>\n",
       "      <td>11</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>499</td>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>11</td>\n",
       "      <td>74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>500</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Record  Spam  Recipients  Hyperlinks  Characters\n",
       "0         1     0          19           1          47\n",
       "1         2     0          15           1          58\n",
       "2         3     1          13          11          88\n",
       "3         4     1          17          11          68\n",
       "4         5     0          15           1          87\n",
       "..      ...   ...         ...         ...         ...\n",
       "495     496     0          15           2          97\n",
       "496     497     0          20           5          72\n",
       "497     498     1          41          11          52\n",
       "498     499     1          16          11          74\n",
       "499     500     1          13           2          32\n",
       "\n",
       "[500 rows x 5 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spamDf = pd.read_excel(\"https://www.dropbox.com/scl/fi/v24mmhg5hmefmnv99uqsy/Spam.xlsx?rlkey=iq7exnueq84sy7y2b8ud70mp0&dl=1\")\n",
    "spamDf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "AgPRgw9TnYLJ"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2500, (500, 5))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spamDf.size, spamDf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "zqcLaMdZoasO"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "Y_JGlYFloXHm"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((350, 5), (150, 5))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Split the dataset into training and testing sets\n",
    "trainSet, testSet = train_test_split(\n",
    "  spamDf,\n",
    "  test_size=0.3,\n",
    "  random_state=1,\n",
    "  stratify=spamDf['Spam']\n",
    ")\n",
    "trainSet.shape, testSet.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "P8pFCQgIpAu3"
   },
   "outputs": [],
   "source": [
    "# Fit the logistic regression model\n",
    "features = ['Recipients', 'Hyperlinks', 'Characters']\n",
    "xTrain = trainSet[features]\n",
    "yTrain = trainSet['Spam'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "6sHvxFpspMKh"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.430522\n",
      "         Iterations 6\n",
      "                           Logit Regression Results                           \n",
      "==============================================================================\n",
      "Dep. Variable:                   Spam   No. Observations:                  350\n",
      "Model:                          Logit   Df Residuals:                      346\n",
      "Method:                           MLE   Df Model:                            3\n",
      "Date:                Sun, 09 Jun 2024   Pseudo R-squ.:                  0.3784\n",
      "Time:                        15:05:50   Log-Likelihood:                -150.68\n",
      "converged:                       True   LL-Null:                       -242.40\n",
      "Covariance Type:            nonrobust   LLR p-value:                 1.606e-39\n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "const         -4.3440      0.757     -5.741      0.000      -5.827      -2.861\n",
      "Recipients     0.1071      0.035      3.083      0.002       0.039       0.175\n",
      "Hyperlinks     0.5803      0.059      9.833      0.000       0.465       0.696\n",
      "Characters    -0.0132      0.006     -2.154      0.031      -0.025      -0.001\n",
      "==============================================================================\n"
     ]
    }
   ],
   "source": [
    "spamBasedOnRecipientsHyperlinksCharactersLogitModel = sm.Logit(\n",
    "  yTrain,\n",
    "  sm.add_constant(xTrain)\n",
    ")\n",
    "spamBasedOnRecipientsHyperlinksCharactersLogitModelFit = spamBasedOnRecipientsHyperlinksCharactersLogitModel.fit()\n",
    "print(spamBasedOnRecipientsHyperlinksCharactersLogitModelFit.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "5YbGrnLcp4EK"
   },
   "outputs": [],
   "source": [
    "predict1 = spamBasedOnRecipientsHyperlinksCharactersLogitModelFit.predict(sm.add_constant(testSet[features]))\n",
    "testSet['predict1'] = predict1\n",
    "sumTable = pd.DataFrame({'A': testSet['Spam'], 'Prob': testSet['predict1']})\n",
    "sumTable.to_csv(\"ROC.csv\", index=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "nnM895bnYFuU"
   },
   "outputs": [],
   "source": [
    "sumTable1 = pd.DataFrame({'A': testSet['Spam'], 'Prob': testSet['predict1']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "id": "N0GKRfOerVZk"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>Prob</th>\n",
       "      <th>P</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>0</td>\n",
       "      <td>0.739633</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>423</th>\n",
       "      <td>0</td>\n",
       "      <td>0.079193</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>369</th>\n",
       "      <td>1</td>\n",
       "      <td>0.712801</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>1</td>\n",
       "      <td>0.838428</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>266</th>\n",
       "      <td>1</td>\n",
       "      <td>0.789240</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>1</td>\n",
       "      <td>0.850576</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>0</td>\n",
       "      <td>0.180012</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322</th>\n",
       "      <td>0</td>\n",
       "      <td>0.376942</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>314</th>\n",
       "      <td>0</td>\n",
       "      <td>0.040472</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>296</th>\n",
       "      <td>0</td>\n",
       "      <td>0.102076</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>150 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     A      Prob  P\n",
       "186  0  0.739633  1\n",
       "423  0  0.079193  0\n",
       "369  1  0.712801  1\n",
       "283  1  0.838428  1\n",
       "266  1  0.789240  1\n",
       "..  ..       ... ..\n",
       "156  1  0.850576  1\n",
       "54   0  0.180012  0\n",
       "322  0  0.376942  0\n",
       "314  0  0.040472  0\n",
       "296  0  0.102076  0\n",
       "\n",
       "[150 rows x 3 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Make predictions based on probability threshold of 0.5\n",
    "testSet['predictions'] = (testSet['predict1'] > 0.5).astype(int)\n",
    "sumTable1['P'] = testSet['predictions']\n",
    "sumTable1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "xlQk7hqYsHwL"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, recall_score, precision_score, roc_auc_score, roc_curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "7FS8w-2ysIlk"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.78\n"
     ]
    }
   ],
   "source": [
    "# Calculate accuracy\n",
    "accuracy = accuracy_score(sumTable1['A'], sumTable1['P'])\n",
    "spamBasedOnRecipientsHyperlinksCharactersLogitModelFit.customMetrics = {}\n",
    "spamBasedOnRecipientsHyperlinksCharactersLogitModelFit.customMetrics['accuracy'] = accuracy\n",
    "print(f'Accuracy: {spamBasedOnRecipientsHyperlinksCharactersLogitModelFit.customMetrics['accuracy']}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "yuSL_r7AsYT3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall: 0.7532467532467533\n"
     ]
    }
   ],
   "source": [
    "# Calculate recall\n",
    "recall = recall_score(sumTable1['A'], sumTable1['P'])\n",
    "spamBasedOnRecipientsHyperlinksCharactersLogitModelFit.customMetrics['recall'] = recall\n",
    "print(f'Recall: {recall}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "NicDWx4esa9G"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision: 0.8055555555555556\n"
     ]
    }
   ],
   "source": [
    "# Calculate precision\n",
    "precision = precision_score(sumTable1['A'], sumTable1['P'])\n",
    "spamBasedOnRecipientsHyperlinksCharactersLogitModelFit.customMetrics['precision'] = precision\n",
    "print(f'Precision: {precision}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "id": "SgxhSyW-spz7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sensitivity: 0.7532467532467533\n",
      "Specificity: 0.8082191780821918\n"
     ]
    }
   ],
   "source": [
    "# Sensitivity and Specificity (Sensitivity is same as recall)\n",
    "sensitivity = recall\n",
    "specificity = sum((sumTable1['A'] == 0) & (sumTable1['P'] == 0)) / sum(sumTable1['A'] == 0)\n",
    "spamBasedOnRecipientsHyperlinksCharactersLogitModelFit.customMetrics['sensitivity'] = sensitivity\n",
    "spamBasedOnRecipientsHyperlinksCharactersLogitModelFit.customMetrics['specificity'] = specificity\n",
    "print(f'Sensitivity: {sensitivity}')\n",
    "print(f'Specificity: {specificity}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "Y4Bufrh8tPIp"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 Score: 0.778523489932886\n"
     ]
    }
   ],
   "source": [
    "# Calculate F1 Score\n",
    "f1Score = 2 * (precision * recall) / (precision + recall)\n",
    "spamBasedOnRecipientsHyperlinksCharactersLogitModelFit.customMetrics['f1Score'] = f1Score\n",
    "print(f'F1 Score: {f1Score}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "7NS_N1R_tcf9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AUC: 0.8305461661626046\n"
     ]
    }
   ],
   "source": [
    "# Plot ROC curve\n",
    "fpr, tpr, _ = roc_curve(testSet['Spam'], testSet['predict1'])\n",
    "roc_auc = roc_auc_score(testSet['Spam'], testSet['predict1'])\n",
    "spamBasedOnRecipientsHyperlinksCharactersLogitModelFit.customMetrics['roc_auc'] = roc_auc\n",
    "# Calculate AUC\n",
    "print(f'AUC: {roc_auc}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "OZLGYNGpuGWY"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "1K-2SMbUt90Z"
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 42556 characters]",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "plt.figure()\n",
    "plt.plot(fpr, tpr, color='darkorange', lw=2, label=f'ROC curve (area = {roc_auc:.2f})')\n",
    "plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('Receiver Operating Characteristic')\n",
    "plt.legend(loc=\"lower right\")\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Record</th>\n",
       "      <th>Spam</th>\n",
       "      <th>Recipients</th>\n",
       "      <th>Hyperlinks</th>\n",
       "      <th>Characters</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>1</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>88</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>11</td>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Record  Spam  Recipients  Hyperlinks  Characters\n",
       "0       1     0          19           1          47\n",
       "1       2     0          15           1          58\n",
       "2       3     1          13          11          88\n",
       "3       4     1          17          11          68\n",
       "4       5     0          15           1          87"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# KFold\n",
    "from sklearn.model_selection import KFold\n",
    "# Initialize KFold\n",
    "# k=2\n",
    "# k=5\n",
    "k=10\n",
    "kf = KFold(n_splits=k, shuffle=True, random_state=55)\n",
    "spamDf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Optimization terminated successfully.\n",
      "         Current function value: 0.460658\n",
      "         Iterations 6\n",
      "expr=1\n",
      "                           Logit Regression Results                           \n",
      "==============================================================================\n",
      "Dep. Variable:                   Spam   No. Observations:                  450\n",
      "Model:                          Logit   Df Residuals:                      446\n",
      "Method:                           MLE   Df Model:                            3\n",
      "Date:                Sun, 09 Jun 2024   Pseudo R-squ.:                  0.3343\n",
      "Time:                        15:23:34   Log-Likelihood:                -207.30\n",
      "converged:                       True   LL-Null:                       -311.38\n",
      "Covariance Type:            nonrobust   LLR p-value:                 7.258e-45\n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "const         -4.0452      0.691     -5.857      0.000      -5.399      -2.691\n",
      "Recipients     0.1205      0.035      3.407      0.001       0.051       0.190\n",
      "Hyperlinks     0.5087      0.047     10.832      0.000       0.417       0.601\n",
      "Characters    -0.0123      0.005     -2.405      0.016      -0.022      -0.002\n",
      "==============================================================================\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.458531\n",
      "         Iterations 6\n",
      "expr=2\n",
      "                           Logit Regression Results                           \n",
      "==============================================================================\n",
      "Dep. Variable:                   Spam   No. Observations:                  450\n",
      "Model:                          Logit   Df Residuals:                      446\n",
      "Method:                           MLE   Df Model:                            3\n",
      "Date:                Sun, 09 Jun 2024   Pseudo R-squ.:                  0.3383\n",
      "Time:                        15:23:34   Log-Likelihood:                -206.34\n",
      "converged:                       True   LL-Null:                       -311.85\n",
      "Covariance Type:            nonrobust   LLR p-value:                 1.759e-45\n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "const         -3.7954      0.651     -5.835      0.000      -5.070      -2.520\n",
      "Recipients     0.0955      0.031      3.036      0.002       0.034       0.157\n",
      "Hyperlinks     0.5174      0.047     10.961      0.000       0.425       0.610\n",
      "Characters    -0.0126      0.005     -2.451      0.014      -0.023      -0.003\n",
      "==============================================================================\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.459673\n",
      "         Iterations 6\n",
      "expr=3\n",
      "                           Logit Regression Results                           \n",
      "==============================================================================\n",
      "Dep. Variable:                   Spam   No. Observations:                  450\n",
      "Model:                          Logit   Df Residuals:                      446\n",
      "Method:                           MLE   Df Model:                            3\n",
      "Date:                Sun, 09 Jun 2024   Pseudo R-squ.:                  0.3365\n",
      "Time:                        15:23:34   Log-Likelihood:                -206.85\n",
      "converged:                       True   LL-Null:                       -311.76\n",
      "Covariance Type:            nonrobust   LLR p-value:                 3.205e-45\n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "const         -3.5764      0.692     -5.172      0.000      -4.932      -2.221\n",
      "Recipients     0.0957      0.037      2.578      0.010       0.023       0.169\n",
      "Hyperlinks     0.5080      0.047     10.921      0.000       0.417       0.599\n",
      "Characters    -0.0153      0.005     -2.954      0.003      -0.025      -0.005\n",
      "==============================================================================\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.452198\n",
      "         Iterations 6\n",
      "expr=4\n",
      "                           Logit Regression Results                           \n",
      "==============================================================================\n",
      "Dep. Variable:                   Spam   No. Observations:                  450\n",
      "Model:                          Logit   Df Residuals:                      446\n",
      "Method:                           MLE   Df Model:                            3\n",
      "Date:                Sun, 09 Jun 2024   Pseudo R-squ.:                  0.3476\n",
      "Time:                        15:23:34   Log-Likelihood:                -203.49\n",
      "converged:                       True   LL-Null:                       -311.92\n",
      "Covariance Type:            nonrobust   LLR p-value:                 9.609e-47\n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "const         -4.0845      0.673     -6.068      0.000      -5.404      -2.765\n",
      "Recipients     0.1071      0.034      3.176      0.001       0.041       0.173\n",
      "Hyperlinks     0.5174      0.047     10.997      0.000       0.425       0.610\n",
      "Characters    -0.0112      0.005     -2.152      0.031      -0.021      -0.001\n",
      "==============================================================================\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.456077\n",
      "         Iterations 6\n",
      "expr=5\n",
      "                           Logit Regression Results                           \n",
      "==============================================================================\n",
      "Dep. Variable:                   Spam   No. Observations:                  450\n",
      "Model:                          Logit   Df Residuals:                      446\n",
      "Method:                           MLE   Df Model:                            3\n",
      "Date:                Sun, 09 Jun 2024   Pseudo R-squ.:                  0.3407\n",
      "Time:                        15:23:34   Log-Likelihood:                -205.23\n",
      "converged:                       True   LL-Null:                       -311.28\n",
      "Covariance Type:            nonrobust   LLR p-value:                 1.033e-45\n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "const         -3.7435      0.657     -5.695      0.000      -5.032      -2.455\n",
      "Recipients     0.1010      0.033      3.026      0.002       0.036       0.166\n",
      "Hyperlinks     0.5043      0.046     11.030      0.000       0.415       0.594\n",
      "Characters    -0.0124      0.005     -2.361      0.018      -0.023      -0.002\n",
      "==============================================================================\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.449368\n",
      "         Iterations 6\n",
      "expr=6\n",
      "                           Logit Regression Results                           \n",
      "==============================================================================\n",
      "Dep. Variable:                   Spam   No. Observations:                  450\n",
      "Model:                          Logit   Df Residuals:                      446\n",
      "Method:                           MLE   Df Model:                            3\n",
      "Date:                Sun, 09 Jun 2024   Pseudo R-squ.:                  0.3512\n",
      "Time:                        15:23:34   Log-Likelihood:                -202.22\n",
      "converged:                       True   LL-Null:                       -311.70\n",
      "Covariance Type:            nonrobust   LLR p-value:                 3.360e-47\n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "const         -3.9098      0.694     -5.635      0.000      -5.270      -2.550\n",
      "Recipients     0.1211      0.036      3.363      0.001       0.051       0.192\n",
      "Hyperlinks     0.5235      0.048     10.982      0.000       0.430       0.617\n",
      "Characters    -0.0172      0.005     -3.297      0.001      -0.028      -0.007\n",
      "==============================================================================\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.455797\n",
      "         Iterations 6\n",
      "expr=7\n",
      "                           Logit Regression Results                           \n",
      "==============================================================================\n",
      "Dep. Variable:                   Spam   No. Observations:                  450\n",
      "Model:                          Logit   Df Residuals:                      446\n",
      "Method:                           MLE   Df Model:                            3\n",
      "Date:                Sun, 09 Jun 2024   Pseudo R-squ.:                  0.3415\n",
      "Time:                        15:23:34   Log-Likelihood:                -205.11\n",
      "converged:                       True   LL-Null:                       -311.47\n",
      "Covariance Type:            nonrobust   LLR p-value:                 7.501e-46\n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "const         -3.7370      0.671     -5.573      0.000      -5.051      -2.423\n",
      "Recipients     0.1079      0.034      3.193      0.001       0.042       0.174\n",
      "Hyperlinks     0.5177      0.047     10.908      0.000       0.425       0.611\n",
      "Characters    -0.0153      0.005     -2.941      0.003      -0.026      -0.005\n",
      "==============================================================================\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.445224\n",
      "         Iterations 6\n",
      "expr=8\n",
      "                           Logit Regression Results                           \n",
      "==============================================================================\n",
      "Dep. Variable:                   Spam   No. Observations:                  450\n",
      "Model:                          Logit   Df Residuals:                      446\n",
      "Method:                           MLE   Df Model:                            3\n",
      "Date:                Sun, 09 Jun 2024   Pseudo R-squ.:                  0.3572\n",
      "Time:                        15:23:34   Log-Likelihood:                -200.35\n",
      "converged:                       True   LL-Null:                       -311.70\n",
      "Covariance Type:            nonrobust   LLR p-value:                 5.249e-48\n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "const         -3.7848      0.667     -5.677      0.000      -5.092      -2.478\n",
      "Recipients     0.1079      0.033      3.243      0.001       0.043       0.173\n",
      "Hyperlinks     0.5328      0.048     11.053      0.000       0.438       0.627\n",
      "Characters    -0.0162      0.005     -3.070      0.002      -0.027      -0.006\n",
      "==============================================================================\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.461358\n",
      "         Iterations 6\n",
      "expr=9\n",
      "                           Logit Regression Results                           \n",
      "==============================================================================\n",
      "Dep. Variable:                   Spam   No. Observations:                  450\n",
      "Model:                          Logit   Df Residuals:                      446\n",
      "Method:                           MLE   Df Model:                            3\n",
      "Date:                Sun, 09 Jun 2024   Pseudo R-squ.:                  0.3338\n",
      "Time:                        15:23:34   Log-Likelihood:                -207.61\n",
      "converged:                       True   LL-Null:                       -311.63\n",
      "Covariance Type:            nonrobust   LLR p-value:                 7.718e-45\n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "const         -3.7859      0.659     -5.741      0.000      -5.078      -2.493\n",
      "Recipients     0.1045      0.032      3.224      0.001       0.041       0.168\n",
      "Hyperlinks     0.5039      0.046     10.879      0.000       0.413       0.595\n",
      "Characters    -0.0133      0.005     -2.574      0.010      -0.023      -0.003\n",
      "==============================================================================\n",
      "Optimization terminated successfully.\n",
      "         Current function value: 0.467026\n",
      "         Iterations 6\n",
      "expr=10\n",
      "                           Logit Regression Results                           \n",
      "==============================================================================\n",
      "Dep. Variable:                   Spam   No. Observations:                  450\n",
      "Model:                          Logit   Df Residuals:                      446\n",
      "Method:                           MLE   Df Model:                            3\n",
      "Date:                Sun, 09 Jun 2024   Pseudo R-squ.:                  0.3258\n",
      "Time:                        15:23:34   Log-Likelihood:                -210.16\n",
      "converged:                       True   LL-Null:                       -311.70\n",
      "Covariance Type:            nonrobust   LLR p-value:                 9.141e-44\n",
      "==============================================================================\n",
      "                 coef    std err          z      P>|z|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "const         -3.8264      0.669     -5.716      0.000      -5.138      -2.514\n",
      "Recipients     0.1148      0.035      3.319      0.001       0.047       0.183\n",
      "Hyperlinks     0.5033      0.047     10.821      0.000       0.412       0.594\n",
      "Characters    -0.0151      0.005     -2.921      0.003      -0.025      -0.005\n",
      "==============================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:22: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['val_predictions'] = val_predictions\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['yHatCross'] = valSet['val_predictions'].apply(lambda x: 1 if x > 0.5 else 0)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:24: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['isCrossCorrect'] = valSet.apply(lambda row: 1 if row['Spam'] == row['yHatCross'] else 0, axis=1)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:22: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['val_predictions'] = val_predictions\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['yHatCross'] = valSet['val_predictions'].apply(lambda x: 1 if x > 0.5 else 0)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:24: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['isCrossCorrect'] = valSet.apply(lambda row: 1 if row['Spam'] == row['yHatCross'] else 0, axis=1)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:22: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['val_predictions'] = val_predictions\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['yHatCross'] = valSet['val_predictions'].apply(lambda x: 1 if x > 0.5 else 0)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:24: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['isCrossCorrect'] = valSet.apply(lambda row: 1 if row['Spam'] == row['yHatCross'] else 0, axis=1)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:22: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['val_predictions'] = val_predictions\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['yHatCross'] = valSet['val_predictions'].apply(lambda x: 1 if x > 0.5 else 0)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:24: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['isCrossCorrect'] = valSet.apply(lambda row: 1 if row['Spam'] == row['yHatCross'] else 0, axis=1)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:22: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['val_predictions'] = val_predictions\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['yHatCross'] = valSet['val_predictions'].apply(lambda x: 1 if x > 0.5 else 0)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:24: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['isCrossCorrect'] = valSet.apply(lambda row: 1 if row['Spam'] == row['yHatCross'] else 0, axis=1)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:22: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['val_predictions'] = val_predictions\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['yHatCross'] = valSet['val_predictions'].apply(lambda x: 1 if x > 0.5 else 0)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:24: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['isCrossCorrect'] = valSet.apply(lambda row: 1 if row['Spam'] == row['yHatCross'] else 0, axis=1)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:22: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['val_predictions'] = val_predictions\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['yHatCross'] = valSet['val_predictions'].apply(lambda x: 1 if x > 0.5 else 0)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:24: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['isCrossCorrect'] = valSet.apply(lambda row: 1 if row['Spam'] == row['yHatCross'] else 0, axis=1)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:22: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['val_predictions'] = val_predictions\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['yHatCross'] = valSet['val_predictions'].apply(lambda x: 1 if x > 0.5 else 0)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:24: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['isCrossCorrect'] = valSet.apply(lambda row: 1 if row['Spam'] == row['yHatCross'] else 0, axis=1)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:22: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['val_predictions'] = val_predictions\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['yHatCross'] = valSet['val_predictions'].apply(lambda x: 1 if x > 0.5 else 0)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:24: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['isCrossCorrect'] = valSet.apply(lambda row: 1 if row['Spam'] == row['yHatCross'] else 0, axis=1)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:22: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['val_predictions'] = val_predictions\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:23: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['yHatCross'] = valSet['val_predictions'].apply(lambda x: 1 if x > 0.5 else 0)\n",
      "/var/folders/v4/9b_k_xyj56ggnxlhf09pt8y40000gn/T/ipykernel_38981/1702950247.py:24: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  valSet['isCrossCorrect'] = valSet.apply(lambda row: 1 if row['Spam'] == row['yHatCross'] else 0, axis=1)\n"
     ]
    }
   ],
   "source": [
    "check = kf.split(spamDf)\n",
    "check\n",
    "experiment = 1\n",
    "# Loop through each fold\n",
    "# Initialize variables to store results\n",
    "accuracies = []\n",
    "\n",
    "for train_index, val_index in check:\n",
    "    # Split the data\n",
    "    trainSet, valSet = spamDf.iloc[train_index], spamDf.iloc[val_index]\n",
    "\n",
    "    # Fit the model\n",
    "\n",
    "    trainModel = sm.Logit(\n",
    "      trainSet[\"Spam\"],\n",
    "      sm.add_constant(trainSet[['Recipients', 'Hyperlinks', 'Characters']])\n",
    "    )\n",
    "    trainModelFit = trainModel.fit()\n",
    "\n",
    "    # Predict on the validation set\n",
    "    val_predictions = trainModelFit.predict(sm.add_constant(valSet[['Recipients', 'Hyperlinks', 'Characters']]))\n",
    "    valSet['val_predictions'] = val_predictions\n",
    "    valSet['yHatCross'] = valSet['val_predictions'].apply(lambda x: 1 if x > 0.5 else 0)\n",
    "    valSet['isCrossCorrect'] = valSet.apply(lambda row: 1 if row['Spam'] == row['yHatCross'] else 0, axis=1)\n",
    "    accuracy = (np.sum(valSet['isCrossCorrect']) / len(valSet['yHatCross'])) * 100\n",
    "    accuracies.append(accuracy)\n",
    "\n",
    "\n",
    "    # Print summary for each fold (optional)\n",
    "    print(f'expr={experiment}')\n",
    "    experiment = experiment +1\n",
    "    print(trainModelFit.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average accuracies across all folds: 78.8\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([82.0, 78.0, 82.0, 74.0, 80.0, 78.0, 78.0, 68.0, 80.0, 88.0], None)"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracies, print(f\"Average accuracies across all folds: {sum(accuracies) /len(accuracies)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'accuracy': 0.78,\n",
       " 'recall': 0.7532467532467533,\n",
       " 'precision': 0.8055555555555556,\n",
       " 'sensitivity': 0.7532467532467533,\n",
       " 'specificity': 0.8082191780821918,\n",
       " 'f1Score': 0.778523489932886,\n",
       " 'roc_auc': 0.8305461661626046,\n",
       " 'k-fold5': {'k': 5,\n",
       "  'accuracies': [80.0, 78.0, 80.0, 73.0, 84.0],\n",
       "  'accuracyAvg': 79.0},\n",
       " 'k-fold10': {'k': 10,\n",
       "  'accuracies': [82.0, 78.0, 82.0, 74.0, 80.0, 78.0, 78.0, 68.0, 80.0, 88.0],\n",
       "  'accuracyAvg': 78.8},\n",
       " 'k-fold2': {'k': 2, 'accuracies': [76.0, 78.8], 'accuracyAvg': 77.4}}"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spamBasedOnRecipientsHyperlinksCharactersLogitModelFit.customMetrics[f'k-fold{k}'] = {\n",
    "    \"k\": k,\n",
    "    \"accuracies\": accuracies,\n",
    "    \"accuracyAvg\": sum(accuracies) /len(accuracies)\n",
    "    \n",
    "}\n",
    "spamBasedOnRecipientsHyperlinksCharactersLogitModelFit.customMetrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<statsmodels.discrete.discrete_model.BinaryResultsWrapper at 0x3689745f0>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from functions.exportModel import exportModel\n",
    "exportModel({\n",
    "    \"modelName\": \"spamBasedOnRecipientsHyperlinksCharactersLogitModelFit\",\n",
    "    \"model\": spamBasedOnRecipientsHyperlinksCharactersLogitModelFit,\n",
    "    \"description\": \"spamDf Logit with hold out\",\n",
    "    \"modelType\": \"sm.Logit\",\n",
    "    \"baseRelativePath\": \"..\",\n",
    "    \"inputs\": [\n",
    "        {\n",
    "            \"name\": \"const\",\n",
    "            \"type\": \"int\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"Recipients\",\n",
    "            \"type\": \"int\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"Hyperlinks\",\n",
    "            \"type\": \"int\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"Characters\",\n",
    "            \"type\": \"int\"\n",
    "        }\n",
    "    ],\n",
    "    \"output\": {\n",
    "        \"name\": \"Spam_probibility\",\n",
    "        \"type\": \"float\"\n",
    "    }\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
