best_auto_features .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os \n",
    "import numpy as np \n",
    "import os\n",
    "import re \n",
    "import csv \n",
    "import math \n",
    "from sklearn import svm\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk import word_tokenize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TODO\n",
    "##### clean data some more \n",
    "##### make sure classes even in train, test data \n",
    "##### balance class split \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "535\n"
     ]
    }
   ],
   "source": [
    "\n",
    "docs = {}\n",
    "labels = {}\n",
    "#I only work if you run me in the same folder as the actual text files \n",
    "\n",
    "with open('training_labels.csv') as csv_file:\n",
    "    csv_reader = csv.reader(csv_file, delimiter=',')\n",
    "    line_count = 0\n",
    "    for row in csv_reader:\n",
    "        if line_count == 0:\n",
    "            #print (row) #these are in the order idx,id,score\n",
    "            line_count += 1\n",
    "        else:\n",
    "            #print(row)\n",
    "            id = int(row[1])\n",
    "            score= float(row[2])\n",
    "            labels[id]=score\n",
    "            line_count += 1\n",
    "    print(line_count)\n",
    "\n",
    "for root, dirs, files in os.walk(\"./\"):\n",
    "    for file in files:\n",
    "        if file.endswith(\".txt\"):\n",
    "            id = re.sub(\"[^0-9]\", \"\", file)\n",
    "            id = int(id) \n",
    "            path_file = os.path.join(root,file)\n",
    "            curdir = path_file\n",
    "            file = open(curdir, 'r') \n",
    "            text = file.read()\n",
    "            \n",
    "            docs[id] = text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "86\n",
      "348\n",
      "434\n"
     ]
    }
   ],
   "source": [
    "train_x,train_y = [],[]\n",
    "for k in labels:\n",
    "    try:\n",
    "        if docs[k] and labels[k]:\n",
    "            train_x.append(docs[k]) \n",
    "            train_y.append(labels[k])\n",
    "    except KeyError:\n",
    "        pass\n",
    "\n",
    "#these two should match \n",
    "# print (len(train_x))\n",
    "# print (len(train_y))\n",
    "\n",
    "\n",
    "indices = math.floor(len(train_x)*.2)\n",
    "indices = list(range(0,indices))\n",
    "test_x = [train_x[x] for x in indices]\n",
    "test_y = [train_y[x] for x in indices]\n",
    "train_x = train_x[indices[-1]+1:] \n",
    "train_y = train_y[indices[-1]+1:]\n",
    "print(len(test_x))\n",
    "print(len(train_x))\n",
    "print(len(test_x) + len(train_x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "# def stemming_tokenizer(text):\n",
    "#     stemmer = PorterStemmer()\n",
    "#     return [stemmer.stem(w) for w in word_tokenize(text)]\n",
    "# def vectorize(txt):\n",
    "#     count_vect = CountVectorizer(tokenizer = stemming_tokenizer, stop_words=stopwords.words('english'),max_features = 5,strip_accents ='ascii')\n",
    "    \n",
    "#     vects = count_vect.fit_transform(txt)\n",
    "#     print(vects.shape) \n",
    "#     return vects\n",
    "\n",
    "def vectorize(txt):\n",
    "    count_vect = CountVectorizer(stop_words=stopwords.words('english'),max_features = 5,strip_accents ='ascii')\n",
    "    #print(count_vect.get_feature_names)\n",
    "    vects = count_vect.fit_transform(txt)\n",
    "    print(vects.shape) \n",
    "    return vects\n",
    "\n",
    "\n",
    "\n",
    "# def vectorize(txt):\n",
    "#     vectorizer = TfidfVectorizer(stop_words='english',max_features = 10)\n",
    "#     vects = vectorizer.fit_transform(txt)\n",
    "#     print(vects.shape) \n",
    "#     return vects\n",
    "\n",
    "from sklearn.preprocessing import normalize \n",
    "def normalize_vect(v):\n",
    "    return normalize(v)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(348, 5)\n",
      "(86, 5)\n"
     ]
    }
   ],
   "source": [
    "#c = vectorize(train_x)\n",
    "\n",
    "train_x_vects = vectorize(train_x)\n",
    "test_x_vects = vectorize(test_x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SVC(C=1, cache_size=200, class_weight=None, coef0=1.0,\n",
    "  decision_function_shape='ovr', degree=3, gamma=1, kernel='rbf',\n",
    "  max_iter=-1, probability=False, random_state=None, shrinking=False,\n",
    "  tol=0.001, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVC(C=1, cache_size=200, class_weight=None, coef0=1.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma=1, kernel='rbf',\n",
       "  max_iter=-1, probability=False, random_state=None, shrinking=False,\n",
       "  tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(train_x_vects, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = model.predict(test_x_vects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5697674418604651"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(predictions == test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.46819814709775004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Erik\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score, precision_score, recall_score\n",
    "\n",
    "# print(recall_score(test_y, predictions, average = 'micro'))\n",
    "# print(precision_score(test_y, predictions, average = 'micro'))\n",
    "print(f1_score(test_y, predictions, average = 'weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Decision Trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='entropy', max_depth=11,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=2,\n",
       "            splitter='best')"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "dt  = DecisionTreeClassifier( random_state = 2, criterion='entropy', splitter = 'best', max_depth=11)\n",
    "dt.fit(train_x_vects, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "predi = dt.predict(test_x_vects, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5930232558139535"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(predi == test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6349593236221143\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Erik\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1135: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "print(f1_score(test_y, predi, average = 'weighted'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ADABOOST\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "#good params base_estimator = dt,n_estimators=5, learning_rate=1.0, algorithm='SAMME.R', random_state=1)\n",
    "\n",
    "#good params w nltk stop words model = AdaBoostClassifier(base_estimator = dt,n_estimators=14, learning_rate=.2, algorithm='SAMME.R', random_state=1)\n",
    "ada = AdaBoostClassifier(base_estimator = dt,n_estimators=30, learning_rate=1.9, algorithm='SAMME.R')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R',\n",
       "          base_estimator=DecisionTreeClassifier(class_weight=None, criterion='entropy', max_depth=11,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=2,\n",
       "            splitter='best'),\n",
       "          learning_rate=1.9, n_estimators=30, random_state=None)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ada.fit(train_x_vects, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "predi = ada.predict(test_x_vects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5348837209302325"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(predi == test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# other stuffs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#max 81.3 adaboost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# def vectorize(txt):\n",
    "#     count_vect = CountVectorizer(stop_words=stopwords.words('english'),max_features = 5,strip_accents ='ascii')\n",
    "#     #print(count_vect.get_feature_names)\n",
    "#     vects = count_vect.fit_transform(txt)\n",
    "#     print(vects.shape) \n",
    "#     return vects\n",
    "\n",
    "\n",
    "# from sklearn.tree import DecisionTreeClassifier\n",
    "# dt  = DecisionTreeClassifier( random_state = 4, criterion='entropy', splitter = 'best', max_depth=None)\n",
    "# dt.fit(train_x_vects, train_y)\n",
    "\n",
    "\n",
    "# from sklearn.ensemble import AdaBoostClassifier\n",
    "# #good params base_estimator = dt,n_estimators=5, learning_rate=1.0, algorithm='SAMME.R', random_state=1)\n",
    "\n",
    "# #good params w nltk stop words model = AdaBoostClassifier(base_estimator = dt,n_estimators=14, learning_rate=.2, algorithm='SAMME.R', random_state=1)\n",
    "# model = AdaBoostClassifier(base_estimator = dt,n_estimators=5, learning_rate=1.0, algorithm='SAMME.R', random_state=2)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forest "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "forest = RandomForestClassifier(n_estimators = 30, random_state = 4, criterion='entropy',  max_depth=13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n",
       "            max_depth=13, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=30, n_jobs=1,\n",
       "            oob_score=False, random_state=4, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest.fit(train_x_vects, train_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "predi = forest.predict(test_x_vects)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.686046511627907"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(predi == test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#max 82.55"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "# forest = RandomForestClassifier(n_estimators = 30, random_state = 4, criterion='entropy',  max_depth=13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}