quora-nlp-data-exploration.ipynb

{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport string\nimport re, string, unicodedata\nimport os\nimport time\nstart_time = time.time()\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Data Import\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\n# Shape for train and test\nprint('Shape of train:',train.shape)\nprint('Shape of test:',test.shape)\nprint(\"--- %s seconds for Data Loading ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66951323931f52749a70c76bc06a4dfc48f4fb05"},"cell_type":"code","source":"train[\"num_words\"] = train[\"question_text\"].apply(lambda x: len(str(x).split()))\ntest[\"num_words\"] = test[\"question_text\"].apply(lambda x: len(str(x).split()))\n#### Stats of Number of Words\nprint('maximum of num_words in train',train[\"num_words\"].max())\nprint('min of num_words in train',train[\"num_words\"].min())\nprint(\"maximum of  num_words in test\",test[\"num_words\"].max())\nprint('min of num_words in test',test[\"num_words\"].min())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79c8b3518424102549b79978fc788edabb0a93cd"},"cell_type":"code","source":"train[\"num_unique_words\"] = train[\"question_text\"].apply(lambda x: len(set(str(x).split())))\ntest[\"num_unique_words\"] = test[\"question_text\"].apply(lambda x: len(set(str(x).split())))\n#### Stats of Number of Unique Words\nprint('maximum of num_unique_words in train',train[\"num_unique_words\"].max())\nprint('mean of num_unique_words in train',train[\"num_unique_words\"].mean())\nprint(\"maximum of num_unique_words in test\",test[\"num_unique_words\"].max())\nprint('mean of num_unique_words in test',test[\"num_unique_words\"].mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"abb2cfd77c850397d1c6862ce8e596c2b6a63516"},"cell_type":"code","source":"train[\"num_chars\"] = train[\"question_text\"].apply(lambda x: len(str(x)))\ntest[\"num_chars\"] = test[\"question_text\"].apply(lambda x: len(str(x)))\n#### Stats of Number of Characters\nprint('maximum of num_chars in train',train[\"num_chars\"].max())\nprint(\"maximum of num_chars in test\",test[\"num_chars\"].max())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14800252b8704805b69aa52215f93bd8dd8f783b"},"cell_type":"code","source":"train[\"num_punctuations\"] =train['question_text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )\ntest[\"num_punctuations\"] =test['question_text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )\n#### Stats of Number of Punctuations\nprint('maximum of num_punctuations in train',train[\"num_punctuations\"].max())\nprint(\"maximum of num_punctuations in test\",test[\"num_punctuations\"].max())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"879179be7695c343470120c6e4c6b4668b02b99c"},"cell_type":"code","source":"train[\"num_words_upper\"] = train[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\ntest[\"num_words_upper\"] = test[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\n#### Stats of Number of Words in Uppercase\nprint('maximum of num_words_upper in train',train[\"num_words_upper\"].max())\nprint(\"maximum of num_words_upper in test\",test[\"num_words_upper\"].max())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ec43785f116ea1fee2ce87cf18be6f6767f9d8a"},"cell_type":"code","source":"import seaborn as sns\ncolor = sns.color_palette()\n\n%matplotlib inline\n\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\n\ncnt_srs = train['target'].value_counts()\n## target distribution ##\nlabels = (np.array(cnt_srs.index))\nsizes = (np.array((cnt_srs / cnt_srs.sum())*100))\n\ntrace = go.Pie(labels=labels, values=sizes)\nlayout = go.Layout(\n    title='Target distribution',\n    font=dict(size=12),\n    width=500,\n    height=500,\n)\ndata = [trace]\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig, filename=\"usertype\")\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4ac5411dbf660412d7979c4f4ba98dc477486a5b"},"cell_type":"markdown","source":"# Boxplots"},{"metadata":{"trusted":true,"_uuid":"9f405216af10a83b1344406d3e697dda4b41fa70"},"cell_type":"code","source":"import matplotlib\nfrom matplotlib import pyplot  as plt\n\n\n\nfig, ax = plt.subplots(figsize=(15,2))\nsns.boxplot(x=\"num_words\", y=\"target\", data=train, ax=ax, orient='h')\nax.set_xlabel('Word Count', size=10)\nax.set_ylabel('Target', size=10)\nax.set_title('Word Count distribution', size=12)\nplt.gca().xaxis.grid(True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94050923a91b6e4d0bc9e320b0a9a67c7822deef"},"cell_type":"code","source":"import matplotlib\nfrom matplotlib import pyplot  as plt\n\n\nfig, ax = plt.subplots(figsize=(15,2))\nsns.boxplot(x=\"num_unique_words\", y=\"target\", data=train, ax=ax, orient='h')\nax.set_xlabel('Unique word count', size=10)\nax.set_ylabel('Target', size=10)\nax.set_title('Unique word count distribution', size=12)\nplt.gca().xaxis.grid(True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"712bb15ae275504471d73694f5ad0b526f7691b0"},"cell_type":"code","source":"import matplotlib\nfrom matplotlib import pyplot  as plt\n\n\n\nfig, ax = plt.subplots(figsize=(15,2))\nsns.boxplot(x=\"num_chars\", y=\"target\", data=train, ax=ax, orient='h')\nax.set_xlabel('Character Count', size=10)\nax.set_ylabel('Target', size=10)\nax.set_title('Character count distribution', size=12)\nplt.gca().xaxis.grid(True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9d90ba79f7bf557ed658f7715137b4474678d04"},"cell_type":"code","source":"import matplotlib\nfrom matplotlib import pyplot  as plt\n\n\nfig, ax = plt.subplots(figsize=(15,2))\nsns.boxplot(x=\"num_punctuations\", y=\"target\", data=train, ax=ax, orient='h')\nax.set_xlabel('Punctuation Count', size=10)\nax.set_ylabel('Target', size=10)\nax.set_title('Punctuation count distribution', size=12)\nplt.gca().xaxis.grid(True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afe20ec853e8604c09ddfca3ce9ff03d357d3986"},"cell_type":"code","source":"import matplotlib\nfrom matplotlib import pyplot  as plt\n\n\nfig, ax = plt.subplots(figsize=(15,2))\nsns.boxplot(x=\"num_words_upper\", y=\"target\", data=train, ax=ax, orient='h')\nax.set_xlabel('Upper case count', size=10)\nax.set_ylabel('Target', size=10)\nax.set_title('Upper case count distribution', size=12)\nplt.gca().xaxis.grid(True)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"38dde2923ef904dcf4c45a2c99d9cfd2aadec7c8"},"cell_type":"markdown","source":"# Wordclouds"},{"metadata":{"trusted":true,"_uuid":"6c15e3a1855a6564d6e3059b2a3f5b4c88df0fcd"},"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\n\ndef generate_wordcloud(text): \n    wordcloud = WordCloud(relative_scaling = 1.0,stopwords = set(STOPWORDS), max_words= 15, background_color ='white').generate(text)\n    fig,ax = plt.subplots(1,1,figsize=(10,10))\n    ax.imshow(wordcloud, interpolation='bilinear')\n    ax.axis(\"off\")\n    ax.margins(x=0, y=0)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36cdf7a517286c462a90681bd7e9df731eed7026"},"cell_type":"code","source":"# Sincere\ngenerate_wordcloud(\" \".join(train.question_text[train.target == 0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8067837e6ebac0f08857c65bde1ea936088157d5"},"cell_type":"code","source":"# Insincere\ngenerate_wordcloud(\" \".join(train.question_text[train.target == 1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2a7bc151ecb4ffe9aeafeaa5ede71041751f057"},"cell_type":"markdown","source":"Weights"},{"metadata":{"trusted":true,"_uuid":"0f34934f58b0d6ef4680c97896c14cebd9b38114"},"cell_type":"code","source":"train['Dataset'] = 'train'\ntest['Dataset'] = 'test'\n\nall_data =  pd.concat([train, test], axis= 0, ignore_index= True)\n\nimport nltk\n#!pip install inflect #Make sure the Kernal has Internet Connected (Check Settings)\n#import inflect\nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import sent_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer\nfrom nltk.stem.wordnet import WordNetLemmatizer \n\n#lower case\nall_data['question_text'] = all_data['question_text'].apply(lambda x: \" \".join(x.lower() for x in x.split()))\n#Removing Punctuation\nall_data['question_text'] = all_data['question_text'].str.replace('[^\\w\\s]','')\n#Removing numbers\nall_data['question_text'] = all_data['question_text'].str.replace('[0-9]','')\n#Remooving stop words and words with length <=2\nfrom nltk.corpus import stopwords\nstop = stopwords.words('english')\nall_data['question_text'] = all_data['question_text'].apply(lambda x: \" \".join(x for x in x.split() if x not in stop and len(x)>2))\n# Lemmatize\nfrom nltk.stem import WordNetLemmatizer\nwl = WordNetLemmatizer()\nall_data['question_text'] = all_data['question_text'].apply(lambda x: \" \".join(wl.lemmatize(x,'v') for x in x.split()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"165f0ddfb444f5c9850d524ef9301ac72c311d89"},"cell_type":"code","source":"from sklearn.metrics import f1_score\nfrom sklearn.model_selection import KFold\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import cross_val_score\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"925d08395dfaa4ae983aff5cebdd741ad793bc80"},"cell_type":"code","source":"print(test.shape)\nprint(train.shape)\ntest = all_data[all_data.Dataset == 'test']\nprint(test.shape)\ntrain = all_data[all_data.Dataset == 'train']\nprint(train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2f1c83af45a51b4d743ad757781edf160e2621d"},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nword_vectorizer = TfidfVectorizer(\n    sublinear_tf=True,\n    strip_accents='unicode',\n    analyzer='word',\n    token_pattern=r'\\w{1,}',\n    stop_words='english',\n    ngram_range=(1, 3),\n    max_features=10000)\n\nword_vectorizer.fit(all_data.question_text)\ndel all_data\ntrain_word_features = word_vectorizer.transform(train.question_text)\ntest_word_features = word_vectorizer.transform(test.question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4abb23c193efe20d3eb8f0d63ca17a73745d334c"},"cell_type":"code","source":"train_target = train['target'].values\n\nkf = KFold(n_splits=5, shuffle=True, random_state=187)\ntest_pred_tf = 0\noof_pred_tf = np.zeros([train.shape[0],])\n\nfor i, (train_index, val_index) in tqdm(enumerate(kf.split(train))):\n    x_train, x_val = train_word_features[train_index,:], train_word_features[val_index,:]\n    y_train, y_val = train_target[train_index], train_target[val_index]\n    classifier = LogisticRegression(class_weight = \"balanced\", C=0.5, solver='sag')\n    classifier.fit(x_train, y_train)\n    val_preds = classifier.predict_proba(x_val)[:,1]\n    preds = classifier.predict_proba(test_word_features)[:,1]\n    test_pred_tf += 0.2*preds\n    oof_pred_tf[val_index] = val_preds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9303babe3d953223e9c33cc0461a812cd5b2585"},"cell_type":"code","source":"import eli5\neli5.show_weights(classifier, vec=word_vectorizer, top=100, feature_filter=lambda x: x != '<BIAS>')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"492404981d8eaace2d8e324ea397017cceb6e843"},"cell_type":"code","source":"import gensim\nfrom gensim import corpora\n#Creating a dictionary of words\ndct = corpora.Dictionary(  nltk.word_tokenize(i) for i in  train.question_text )\ntrain[\"question_text_tokens\"] = list(map(nltk.word_tokenize, train.question_text))\n# check threshold\ndct.filter_extremes(no_below=20, no_above=0.5)\n# Reindexes the remaining words after filtering\ndct.compactify()\nprint(\"Left with {} words.\".format(len(dct.values())))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46e2db33e2d4004357d79467d71f986353a8676c"},"cell_type":"code","source":"#Make a BOW for every document\ndef document_to_bow(df):\n    df['bow'] = list(map(lambda doc: dct.doc2bow(doc), df.question_text_tokens))\n    \ndocument_to_bow(train)\n\n## LDA Topic Model\n# model imports\nfrom gensim.models.ldamulticore import LdaMulticore\ncorpus = train.bow\nnum_topics = 100\n#A multicore approach to decrease training time\nLDAmodel = LdaMulticore(corpus=corpus,\n                        id2word=dct,\n                        num_topics=num_topics,\n                        workers=4,\n                        chunksize=4000,\n                        passes=7,\n                        alpha='asymmetric')\nprint(\"--- %s seconds for LDA ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a93d73b0a17c5e7d6f04fb20bb748fb7a3a9126d"},"cell_type":"code","source":"def document_to_lda_features(lda_model, document):\n    \"\"\" Transforms a bag of words document to features.\n    It returns the proportion of how much each topic was\n    present in the document.\n    \"\"\"\n    topic_importances = LDAmodel.get_document_topics(document, minimum_probability=0)\n    topic_importances = np.array(topic_importances)\n    return topic_importances[:,1]\n\ntrain['lda_features'] = list(map(lambda doc:\n                                      document_to_lda_features(LDAmodel, doc),\n                                      train.bow))\nprint(\"--- %s seconds for LDA to Features ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29fb16ecb679b67d896c0b3f88a8e6bcea6ce5a7"},"cell_type":"code","source":"Insincere_topic_distribution = train.loc[train.target == 1, 'lda_features'].mean()\nSincere_topic_distribution = train.loc[train.target == 0, 'lda_features'].mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44fddec1c3833bb3ed5abfecd2e8db2443e3d231"},"cell_type":"code","source":"### Visualize Topics for different Classes\nimport matplotlib.pyplot as plt\nfig, [ax1,ax2] = plt.subplots(2,1,figsize=(20,20))\n\nnr_top_bars = 5\n\nax1.set_title(\"Insincere topic distributions\", fontsize=16)\nax2.set_title(\"Sincere topic distributions\", fontsize=16)\n\nfor ax, distribution, color in zip([ax1,ax2],\n                                   [Insincere_topic_distribution,Sincere_topic_distribution],\n                                   ['b','r']):\n    # Individual distribution barplots\n    ax.bar(range(len(distribution)), distribution, alpha=0.7)\n    rects = ax.patches\n    for i in np.argsort(distribution)[-nr_top_bars:]:\n        rects[i].set_color(color)\n        rects[i].set_alpha(1)\n    # General plotting adjustments\n    ax.set_xlim(-1, 150)\n    ax.set_xticks(range(20,149,20))\n    ax.set_xticklabels(range(20,149,20), fontsize=16)\n    ax.set_ylim(0,0.02)\n    ax.set_yticks([0,0.01,0.02])\n    ax.set_yticklabels([0,0.01,0.02], fontsize=16)\n\nfig.tight_layout(h_pad=3.)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c7bbb5867c25bf7b5f449aa0cacaa0a96b9d1eda"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}