Make_Final_Predictions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np \n",
    "import re\n",
    "import io\n",
    "#from utilities.features_csv import to_csv\n",
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "from sklearn import svm\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk import word_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re \n",
    "import csv \n",
    "docs = {}\n",
    "labels = {}\n",
    "#I only work if you run me in the same folder as the actual text files \n",
    "\n",
    "with open('training_labels.csv') as csv_file:\n",
    "    csv_reader = csv.reader(csv_file, delimiter=',')\n",
    "    line_count = 0\n",
    "    for row in csv_reader:\n",
    "        if line_count == 0:\n",
    "            line_count += 1\n",
    "        else:\n",
    "            id = int(row[1])\n",
    "            score= float(row[2])\n",
    "            labels[id]=score\n",
    "            line_count += 1\n",
    "\n",
    "for root, dirs, files in os.walk(\"corpus/\"):\n",
    "    for file in files:\n",
    "        if file.endswith(\".txt\"):\n",
    "            id = re.sub(\"[^0-9]\", \"\", file)\n",
    "            id = int(id) \n",
    "            path_file = os.path.join(root,file)\n",
    "            curdir = path_file\n",
    "            file = open(curdir, 'r', errors='ignore')\n",
    "            text = file.read()\n",
    "            docs[id] = text\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(434, 6)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data</th>\n",
       "      <th>information</th>\n",
       "      <th>may</th>\n",
       "      <th>personal</th>\n",
       "      <th>privacy</th>\n",
       "      <th>use</th>\n",
       "      <th>iD</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    data  information  may  personal  privacy  use  iD\n",
       "27     1            1    1         1        0    1  27\n",
       "31     1            1    1         1        1    1  31\n",
       "38     1            1    1         1        1    1  38\n",
       "39     1            1    1         1        0    1  39\n",
       "43     1            1    1         1        1    1  43"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#auto features here \n",
    "import os \n",
    "import numpy as np \n",
    "import os\n",
    "import re \n",
    "import csv \n",
    "import math \n",
    "from sklearn import svm\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk import word_tokenize\n",
    "import pandas as pd \n",
    "train_x,train_y = [],[]\n",
    "for k in labels:\n",
    "    try:\n",
    "        if docs[k] and labels[k]:\n",
    "            train_x.append(docs[k]) \n",
    "            train_y.append(labels[k])\n",
    "    except KeyError:\n",
    "        pass\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "def vectorize(txt):\n",
    "    count_vect = CountVectorizer(stop_words=stopwords.words('english'),max_features = 6,strip_accents ='ascii')\n",
    "    #print(count_vect.get_feature_names)\n",
    "    vects = count_vect.fit_transform(txt)\n",
    "    print(vects.shape) \n",
    "    return count_vect\n",
    "\n",
    "from sklearn.preprocessing import normalize \n",
    "def normalize_vect(v):\n",
    "    return normalize(v)\n",
    "    \n",
    "v = vectorize(train_x)\n",
    "imp_words = v.get_feature_names()\n",
    "id_ct = {}\n",
    "for id in docs:\n",
    "    id_ct[id] = [0]*len(imp_words)\n",
    "for id in id_ct:\n",
    "    txt = docs[id] \n",
    "    for x in range(len(imp_words)):\n",
    "        word = imp_words[x]\n",
    "        if word in txt:\n",
    "            id_ct[id][x] +=1\n",
    "dat = pd.DataFrame(data = id_ct)\n",
    "dat = dat.transpose()\n",
    "dat.columns = imp_words\n",
    "dat['iD'] = dat.index\n",
    "dat.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# dat.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>iD</th>\n",
       "      <th>minor</th>\n",
       "      <th>geo-location</th>\n",
       "      <th>contact_email</th>\n",
       "      <th>vendors</th>\n",
       "      <th>sell_personal</th>\n",
       "      <th>share_personal</th>\n",
       "      <th>Cookies</th>\n",
       "      <th>fog_index</th>\n",
       "      <th>avg_sentence_length</th>\n",
       "      <th>flesch_reading_ease</th>\n",
       "      <th>dale_chall_readability_score</th>\n",
       "      <th>Score</th>\n",
       "      <th>data</th>\n",
       "      <th>information</th>\n",
       "      <th>may</th>\n",
       "      <th>personal</th>\n",
       "      <th>privacy</th>\n",
       "      <th>use</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20481</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>75.245659</td>\n",
       "      <td>183.114146</td>\n",
       "      <td>-4.42</td>\n",
       "      <td>9.08</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20484</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>58.618557</td>\n",
       "      <td>141.546392</td>\n",
       "      <td>37.79</td>\n",
       "      <td>7.02</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>854</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>55.264865</td>\n",
       "      <td>133.162162</td>\n",
       "      <td>46.30</td>\n",
       "      <td>6.60</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20490</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>70.405195</td>\n",
       "      <td>171.012987</td>\n",
       "      <td>7.88</td>\n",
       "      <td>8.48</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20493</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>81.172242</td>\n",
       "      <td>197.930605</td>\n",
       "      <td>-10.99</td>\n",
       "      <td>9.82</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>173</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>47.731507</td>\n",
       "      <td>114.328767</td>\n",
       "      <td>65.41</td>\n",
       "      <td>5.67</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>11096</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>49.727168</td>\n",
       "      <td>119.317919</td>\n",
       "      <td>60.35</td>\n",
       "      <td>5.92</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>20500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>92.804233</td>\n",
       "      <td>227.010582</td>\n",
       "      <td>-48.97</td>\n",
       "      <td>11.26</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>516</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>66.037158</td>\n",
       "      <td>160.092896</td>\n",
       "      <td>18.96</td>\n",
       "      <td>7.94</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>20506</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>42.903529</td>\n",
       "      <td>102.258824</td>\n",
       "      <td>77.66</td>\n",
       "      <td>5.07</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      iD  minor  geo-location  contact_email  vendors  sell_personal  \\\n",
       "0  20481      1             0              0        1              0   \n",
       "1  20484      0             0              1        0              0   \n",
       "2    854      0             0              0        0              0   \n",
       "3  20490      1             0              1        0              0   \n",
       "4  20493      0             0              0        0              0   \n",
       "5    173      0             0              1        0              0   \n",
       "6  11096      0             0              1        0              0   \n",
       "7  20500      0             0              0        0              0   \n",
       "8    516      0             1              0        0              0   \n",
       "9  20506      0             0              0        0              0   \n",
       "\n",
       "   share_personal  Cookies  fog_index  avg_sentence_length  \\\n",
       "0               0        0  75.245659           183.114146   \n",
       "1               1        1  58.618557           141.546392   \n",
       "2               0        0  55.264865           133.162162   \n",
       "3               0        1  70.405195           171.012987   \n",
       "4               0        0  81.172242           197.930605   \n",
       "5               0        1  47.731507           114.328767   \n",
       "6               0        0  49.727168           119.317919   \n",
       "7               0        1  92.804233           227.010582   \n",
       "8               0        1  66.037158           160.092896   \n",
       "9               0        0  42.903529           102.258824   \n",
       "\n",
       "   flesch_reading_ease  dale_chall_readability_score  Score  data  \\\n",
       "0                -4.42                          9.08    3.0     1   \n",
       "1                37.79                          7.02    3.0     1   \n",
       "2                46.30                          6.60    2.0     1   \n",
       "3                 7.88                          8.48    3.0     1   \n",
       "4               -10.99                          9.82    2.0     1   \n",
       "5                65.41                          5.67    3.0     1   \n",
       "6                60.35                          5.92    2.0     1   \n",
       "7               -48.97                         11.26    2.0     1   \n",
       "8                18.96                          7.94    2.0     1   \n",
       "9                77.66                          5.07    2.0     0   \n",
       "\n",
       "   information  may  personal  privacy  use  \n",
       "0            1    1         1        1    1  \n",
       "1            1    1         1        1    1  \n",
       "2            1    1         1        1    1  \n",
       "3            1    1         1        1    1  \n",
       "4            1    1         0        1    1  \n",
       "5            1    1         1        1    1  \n",
       "6            1    1         1        1    1  \n",
       "7            1    1         1        1    1  \n",
       "8            1    1         1        1    1  \n",
       "9            1    1         1        1    1  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data = pd.read_csv(\"final.csv\")\n",
    "frames = [data,dat]\n",
    "data =  data.merge(dat,how='left',on='iD')\n",
    "display(data.head(n=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>minor</th>\n",
       "      <th>contact_email</th>\n",
       "      <th>vendors</th>\n",
       "      <th>share_personal</th>\n",
       "      <th>Cookies</th>\n",
       "      <th>fog_index</th>\n",
       "      <th>avg_sentence_length</th>\n",
       "      <th>flesch_reading_ease</th>\n",
       "      <th>dale_chall_readability_score</th>\n",
       "      <th>may</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>75.245659</td>\n",
       "      <td>183.114146</td>\n",
       "      <td>-4.42</td>\n",
       "      <td>9.08</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>58.618557</td>\n",
       "      <td>141.546392</td>\n",
       "      <td>37.79</td>\n",
       "      <td>7.02</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>55.264865</td>\n",
       "      <td>133.162162</td>\n",
       "      <td>46.30</td>\n",
       "      <td>6.60</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>70.405195</td>\n",
       "      <td>171.012987</td>\n",
       "      <td>7.88</td>\n",
       "      <td>8.48</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>81.172242</td>\n",
       "      <td>197.930605</td>\n",
       "      <td>-10.99</td>\n",
       "      <td>9.82</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   minor  contact_email  vendors  share_personal  Cookies  fog_index  \\\n",
       "0      1              0        1               0        0  75.245659   \n",
       "1      0              1        0               1        1  58.618557   \n",
       "2      0              0        0               0        0  55.264865   \n",
       "3      1              1        0               0        1  70.405195   \n",
       "4      0              0        0               0        0  81.172242   \n",
       "\n",
       "   avg_sentence_length  flesch_reading_ease  dale_chall_readability_score  may  \n",
       "0           183.114146                -4.42                          9.08    1  \n",
       "1           141.546392                37.79                          7.02    1  \n",
       "2           133.162162                46.30                          6.60    1  \n",
       "3           171.012987                 7.88                          8.48    1  \n",
       "4           197.930605               -10.99                          9.82    1  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#spliting labels and features\n",
    "scores = data['Score']\n",
    "features_raw = data.drop(['Score', 'iD'] , axis = 1)\n",
    "features_raw = features_raw.drop([\"sell_personal\",'geo-location'],axis=1)\n",
    "# features_raw = features_raw.drop(['geo-location'],axis=1)\n",
    "features_raw = features_raw.drop(['data','information','use','privacy','personal'],axis=1)\n",
    "features_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>minor</th>\n",
       "      <th>contact_email</th>\n",
       "      <th>vendors</th>\n",
       "      <th>share_personal</th>\n",
       "      <th>Cookies</th>\n",
       "      <th>fog_index</th>\n",
       "      <th>avg_sentence_length</th>\n",
       "      <th>flesch_reading_ease</th>\n",
       "      <th>dale_chall_readability_score</th>\n",
       "      <th>may</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.660803</td>\n",
       "      <td>0.660803</td>\n",
       "      <td>0.332320</td>\n",
       "      <td>0.660400</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.418325</td>\n",
       "      <td>0.418325</td>\n",
       "      <td>0.569989</td>\n",
       "      <td>0.418331</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.369417</td>\n",
       "      <td>0.369417</td>\n",
       "      <td>0.617905</td>\n",
       "      <td>0.368978</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.590213</td>\n",
       "      <td>0.590213</td>\n",
       "      <td>0.401577</td>\n",
       "      <td>0.589894</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.747232</td>\n",
       "      <td>0.747232</td>\n",
       "      <td>0.295327</td>\n",
       "      <td>0.747356</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.259556</td>\n",
       "      <td>0.259556</td>\n",
       "      <td>0.725507</td>\n",
       "      <td>0.259694</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   minor  contact_email  vendors  share_personal  Cookies  fog_index  \\\n",
       "0      1              0        1               0        0   0.660803   \n",
       "1      0              1        0               1        1   0.418325   \n",
       "2      0              0        0               0        0   0.369417   \n",
       "3      1              1        0               0        1   0.590213   \n",
       "4      0              0        0               0        0   0.747232   \n",
       "5      0              1        0               0        1   0.259556   \n",
       "\n",
       "   avg_sentence_length  flesch_reading_ease  dale_chall_readability_score  may  \n",
       "0             0.660803             0.332320                      0.660400    1  \n",
       "1             0.418325             0.569989                      0.418331    1  \n",
       "2             0.369417             0.617905                      0.368978    1  \n",
       "3             0.590213             0.401577                      0.589894    1  \n",
       "4             0.747232             0.295327                      0.747356    1  \n",
       "5             0.259556             0.725507                      0.259694    1  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "# Initialize a scaler, then apply it to the features\n",
    "scaler = MinMaxScaler()\n",
    "numerical = ['fog_index', 'avg_sentence_length', 'flesch_reading_ease', 'dale_chall_readability_score']\n",
    "features_log_minmax_transform = pd.DataFrame(data = features_raw)\n",
    "features_log_minmax_transform[numerical] = scaler.fit_transform(features_raw[numerical])\n",
    "\n",
    "\n",
    "# Show an example of a record with scaling applied\n",
    "display(features_log_minmax_transform.head(n = 6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set has 347 samples.\n",
      "Testing set has 87 samples.\n"
     ]
    }
   ],
   "source": [
    "# Import train_test_split\n",
    "from sklearn.model_selection import train_test_split\n",
    "# Split the 'features' and 'income' data into training and testing sets\n",
    "X_train, X_test, y_train, y_test = train_test_split(features_log_minmax_transform, \n",
    "                                                    scores, \n",
    "                                                    test_size = .2, \n",
    "                                                    random_state = 0)\n",
    "# Show the results of the split\n",
    "print(\"Training set has {} samples.\".format(X_train.shape[0]))\n",
    "print(\"Testing set has {} samples.\".format(X_test.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
       "            max_depth=31, max_features=3, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=15, n_jobs=1,\n",
       "            oob_score=False, random_state=1, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "forest = RandomForestClassifier(n_estimators=15, max_depth=31,random_state=1, criterion=\"entropy\", max_features = 3)\n",
    "forest.fit(X_train, y_train)\n",
    "\n",
    "# predictions = forest.predict(X_test)\n",
    "# from sklearn.metrics import f1_score\n",
    "# f1_score(y_test, predictions, average='macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for x in range(1,200):\n",
    "#     for y in range(1,200):\n",
    "#         for z in range(1,200):\n",
    "#             forest  = RandomForestClassifier(n_estimators=y, max_depth=z,random_state=x, criterion=\"gini\")\n",
    "#             forest.fit(X_train, y_train)\n",
    "#             predi = forest.predict(X_test)\n",
    "#             ans = f1_score(y_test, predi, average = 'macro')\n",
    "#             if ans > .92:\n",
    "#                 print (\"random_state: \" + str(x))\n",
    "#                 print (\"n_estimators: \" + str(y))\n",
    "#                 print(\"max_depth: \" + str(z))\n",
    "#                 print(ans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=14,\n",
       "            max_features=3, max_leaf_nodes=None, min_impurity_decrease=0.0,\n",
       "            min_impurity_split=None, min_samples_leaf=1,\n",
       "            min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "            presort=False, random_state=47, splitter='random')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "dt  = DecisionTreeClassifier(random_state = 47, criterion='gini', splitter = 'random', max_depth=14, max_features = 3)\n",
    "dt.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for x in range(0,100):\n",
    "#     for y in range(1,100):\n",
    "#         for z in range(1,11):\n",
    "#             dt  = DecisionTreeClassifier( presort = True,random_state = x, criterion='gini', splitter = 'random', max_depth=y, max_features = z)\n",
    "#             dt.fit(X_train, y_train)\n",
    "#             predi = dt.predict(X_test)\n",
    "#             ans = f1_score(y_test, predi, average = 'macro')\n",
    "#             if ans > .952:\n",
    "#                 print (\"random_state: \" + str(x))\n",
    "#                 print (\"n_estimators: \" + str(y))\n",
    "#                 print(\"max_features: \" + str(z))\n",
    "#                 print(ans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.951861763131738"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dt.fit(X_train, y_train)\n",
    "predictions = dt.predict(X_test)\n",
    "from sklearn.metrics import f1_score \n",
    "f1_score(y_test, predictions, average = 'macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from sklearn.tree import DecisionTreeClassifier\n",
    "# dt  = DecisionTreeClassifier( random_state = 1, criterion='gini', splitter = 'random', max_depth=20)\n",
    "# dt.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from sklearn.ensemble import AdaBoostClassifier\n",
    "# #good params base_estimator = dt,n_estimators=5, learning_rate=1.0, algorithm='SAMME.R', random_state=1)\n",
    "\n",
    "# #good params w nltk stop words model = AdaBoostClassifier(base_estimator = dt,n_estimators=14, learning_rate=.2, algorithm='SAMME.R', random_state=1)\n",
    "# ada = AdaBoostClassifier(random_state = 21,base_estimator = dt,n_estimators=13, learning_rate=.2, algorithm='SAMME.R')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# ada.fit(X_train, y_train)\n",
    "# predi = ada.predict(X_test)\n",
    "# f1_score(y_test, predi, average = 'macro')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ExtraTreesClassifier(bootstrap=False, class_weight=None, criterion='entropy',\n",
       "           max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators=35, n_jobs=1,\n",
       "           oob_score=False, random_state=0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "\n",
    "\n",
    "ET = ExtraTreesClassifier(n_estimators = 35, random_state = 0, criterion='entropy')\n",
    "ET.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 81.973% (5.400%)\n"
     ]
    }
   ],
   "source": [
    "from sklearn import model_selection\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "X = features_log_minmax_transform\n",
    "y = scores\n",
    "num_folds = 5\n",
    "num_instances = len(X)\n",
    "skf = StratifiedKFold(n_splits=10, shuffle=True)\n",
    "model = ET\n",
    "results = model_selection.cross_val_score(model, X, y, cv=skf)\n",
    "print(\"Accuracy: %.3f%% (%.3f%%)\" % (results.mean()*100.0, results.std()*100.0))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        1.0       0.88      0.94      0.91        16\n",
      "        2.0       0.98      0.95      0.97        44\n",
      "        3.0       1.00      0.95      0.98        21\n",
      "        4.0       0.83      1.00      0.91         5\n",
      "        5.0       1.00      1.00      1.00         1\n",
      "\n",
      "avg / total       0.96      0.95      0.95        87\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "#target_names = ['1', '2', '3', '4', '5']\n",
    "print(classification_report(y_test, predictions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Test Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(62, 6)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>data</th>\n",
       "      <th>information</th>\n",
       "      <th>may</th>\n",
       "      <th>personal</th>\n",
       "      <th>privacy</th>\n",
       "      <th>us</th>\n",
       "      <th>iD</th>\n",
       "      <th>minor</th>\n",
       "      <th>geo-location</th>\n",
       "      <th>contact_email</th>\n",
       "      <th>vendors</th>\n",
       "      <th>sell_personal</th>\n",
       "      <th>share_personal</th>\n",
       "      <th>cookies</th>\n",
       "      <th>fog_index</th>\n",
       "      <th>avg_sentence_length</th>\n",
       "      <th>flesch_reading_ease</th>\n",
       "      <th>dale_chall_readability_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>40043</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>75.463861</td>\n",
       "      <td>183.656250</td>\n",
       "      <td>-4.97</td>\n",
       "      <td>9.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>40063</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>46.412647</td>\n",
       "      <td>111.025806</td>\n",
       "      <td>68.76</td>\n",
       "      <td>5.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>40157</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>53.939328</td>\n",
       "      <td>129.841121</td>\n",
       "      <td>49.67</td>\n",
       "      <td>6.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>40160</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>70.235947</td>\n",
       "      <td>170.586592</td>\n",
       "      <td>8.31</td>\n",
       "      <td>8.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>40163</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>53.032733</td>\n",
       "      <td>127.576389</td>\n",
       "      <td>51.96</td>\n",
       "      <td>6.33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   data  information  may  personal  privacy  us     iD  minor  geo-location  \\\n",
       "0     1            1    1         1        1   1  40043      0             0   \n",
       "1     1            1    1         1        1   1  40063      1             0   \n",
       "2     1            1    1         1        1   1  40157      0             0   \n",
       "3     1            1    1         1        1   1  40160      1             0   \n",
       "4     1            1    1         0        1   1  40163      0             0   \n",
       "\n",
       "   contact_email  vendors  sell_personal  share_personal  cookies  fog_index  \\\n",
       "0              1        1              0               1        1  75.463861   \n",
       "1              1        1              0               0        1  46.412647   \n",
       "2              1        1              0               0        1  53.939328   \n",
       "3              1        1              0               0        1  70.235947   \n",
       "4              1        1              0               0        1  53.032733   \n",
       "\n",
       "   avg_sentence_length  flesch_reading_ease  dale_chall_readability_score  \n",
       "0           183.656250                -4.97                          9.11  \n",
       "1           111.025806                68.76                          5.51  \n",
       "2           129.841121                49.67                          6.44  \n",
       "3           170.586592                 8.31                          8.46  \n",
       "4           127.576389                51.96                          6.33  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# del train_x \n",
    "# del id_ct\n",
    "# del docs \n",
    "# del labels \n",
    "# del data \n",
    "# del dat \n",
    "##clear old vars\n",
    "\n",
    "import os \n",
    "import numpy as np \n",
    "import os\n",
    "import re \n",
    "import csv \n",
    "import math \n",
    "from sklearn import svm\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk import word_tokenize\n",
    "import pandas as pd \n",
    "import re \n",
    "import csv \n",
    "\n",
    "docs = {}\n",
    "\n",
    "for root, dirs, files in os.walk(\"corpus/testing_set\"):\n",
    "    for file in files:\n",
    "        if file.endswith(\".txt\"):\n",
    "            id = re.sub(\"[^0-9]\", \"\", file)\n",
    "            id = int(id) \n",
    "            #print(id)\n",
    "            path_file = os.path.join(root,file)\n",
    "            curdir = path_file\n",
    "            file = open(curdir, 'r')\n",
    "            text = file.read()\n",
    "            docs[id] = text\n",
    "\n",
    "#mind you the test data set does not come with the labels...\n",
    "train_x= []\n",
    "\n",
    "for k in docs:\n",
    "    try:\n",
    "        if docs[k]:\n",
    "            train_x.append(docs[k]) \n",
    "    except KeyError:\n",
    "        pass\n",
    "    \n",
    "#get vectoized features \n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "def vectorize(txt):\n",
    "    count_vect = CountVectorizer(stop_words=stopwords.words('english'),max_features = 6,strip_accents ='ascii')\n",
    "    #print(count_vect.get_feature_names)\n",
    "    vects = count_vect.fit_transform(txt)\n",
    "    print(vects.shape) \n",
    "    return count_vect\n",
    "\n",
    "v = vectorize(train_x)\n",
    "imp_words = v.get_feature_names()\n",
    "id_ct = {}\n",
    "for id in docs:\n",
    "    id_ct[id] = [0]*len(imp_words)\n",
    "for id in id_ct:\n",
    "    txt = docs[id] \n",
    "    for x in range(len(imp_words)):\n",
    "        word = imp_words[x]\n",
    "        if word in txt:\n",
    "            id_ct[id][x] +=1\n",
    "            \n",
    "vectorized = pd.DataFrame(data = id_ct)\n",
    "vectorized = vectorized.transpose()\n",
    "vectorized.columns = imp_words\n",
    "vectorized['iD'] = vectorized.index\n",
    "data = pd.read_csv(\"testing_data_set.csv\")\n",
    "data =  vectorized.merge(data,how='left',on='iD')\n",
    "display(data.head(n=5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>may</th>\n",
       "      <th>minor</th>\n",
       "      <th>contact_email</th>\n",
       "      <th>vendors</th>\n",
       "      <th>share_personal</th>\n",
       "      <th>cookies</th>\n",
       "      <th>fog_index</th>\n",
       "      <th>avg_sentence_length</th>\n",
       "      <th>flesch_reading_ease</th>\n",
       "      <th>dale_chall_readability_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>75.463861</td>\n",
       "      <td>183.656250</td>\n",
       "      <td>-4.97</td>\n",
       "      <td>9.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>46.412647</td>\n",
       "      <td>111.025806</td>\n",
       "      <td>68.76</td>\n",
       "      <td>5.51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>53.939328</td>\n",
       "      <td>129.841121</td>\n",
       "      <td>49.67</td>\n",
       "      <td>6.44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>70.235947</td>\n",
       "      <td>170.586592</td>\n",
       "      <td>8.31</td>\n",
       "      <td>8.46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>53.032733</td>\n",
       "      <td>127.576389</td>\n",
       "      <td>51.96</td>\n",
       "      <td>6.33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   may  minor  contact_email  vendors  share_personal  cookies  fog_index  \\\n",
       "0    1      0              1        1               1        1  75.463861   \n",
       "1    1      1              1        1               0        1  46.412647   \n",
       "2    1      0              1        1               0        1  53.939328   \n",
       "3    1      1              1        1               0        1  70.235947   \n",
       "4    1      0              1        1               0        1  53.032733   \n",
       "\n",
       "   avg_sentence_length  flesch_reading_ease  dale_chall_readability_score  \n",
       "0           183.656250                -4.97                          9.11  \n",
       "1           111.025806                68.76                          5.51  \n",
       "2           129.841121                49.67                          6.44  \n",
       "3           170.586592                 8.31                          8.46  \n",
       "4           127.576389                51.96                          6.33  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "allids = data['iD']\n",
    "features_raw = data.drop(['iD'] , axis = 1)\n",
    "features_raw = features_raw.drop([\"sell_personal\",'geo-location'],axis=1)\n",
    "# features_raw = features_raw.drop(['geo-location'],axis=1)\n",
    "features_raw = features_raw.drop(['data','information',\"us\",'privacy','personal'],axis=1)\n",
    "features_raw.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "scaler = MinMaxScaler()\n",
    "numerical = ['fog_index', 'avg_sentence_length', 'flesch_reading_ease', 'dale_chall_readability_score']\n",
    "features_log_minmax_transform = pd.DataFrame(data = features_raw)\n",
    "features_log_minmax_transform[numerical] = scaler.fit_transform(features_raw[numerical])\n",
    "alldata = features_log_minmax_transform\n",
    "preds = ET.predict(alldata)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GENERATE PREDICTIONS FOR TEST SET "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "answer = pd.DataFrame(allids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "answer[\"score\"] = preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>40043</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>40063</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>40157</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>40160</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>40163</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>40169</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>40176</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>40272</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>40288</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>40383</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>40393</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>40396</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>40399</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>40402</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>40491</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>40495</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>40498</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>40596</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>40603</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>40606</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>40705</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>40711</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>40714</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>40717</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>40723</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>40806</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>40813</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>40819</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>40931</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>40935</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>40939</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>40943</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>41041</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>41051</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>41133</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  score\n",
       "0   40043    3.0\n",
       "1   40063    4.0\n",
       "2   40157    3.0\n",
       "3   40160    4.0\n",
       "4   40163    3.0\n",
       "5   40169    3.0\n",
       "6   40176    3.0\n",
       "7   40272    4.0\n",
       "8   40288    2.0\n",
       "9   40383    3.0\n",
       "10  40393    3.0\n",
       "11  40396    3.0\n",
       "12  40399    2.0\n",
       "13  40402    3.0\n",
       "14  40491    2.0\n",
       "15  40495    3.0\n",
       "16  40498    4.0\n",
       "17  40596    2.0\n",
       "18  40603    2.0\n",
       "19  40606    3.0\n",
       "20  40705    2.0\n",
       "21  40711    2.0\n",
       "22  40714    3.0\n",
       "23  40717    3.0\n",
       "24  40723    3.0\n",
       "25  40806    3.0\n",
       "26  40813    3.0\n",
       "27  40819    3.0\n",
       "28  40931    3.0\n",
       "29  40935    5.0\n",
       "30  40939    2.0\n",
       "31  40943    3.0\n",
       "32  41041    3.0\n",
       "33  41051    3.0\n",
       "34  41133    3.0"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer.columns = [\"id\",\"score\"]\n",
    "answer.head(35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "file = open(\"hackathon_ans.csv\",\"w\") \n",
    "colsnrows = answer.to_csv(index = False)\n",
    "file.write(colsnrows)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2.])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ET.predict(features_raw.iloc[[30]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}