vision-transformer-vit-tutorial-baseline.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2020-12-02T18:32:11.628947Z",
     "iopub.status.busy": "2020-12-02T18:32:11.628076Z",
     "iopub.status.idle": "2020-12-02T18:32:24.574113Z",
     "shell.execute_reply": "2020-12-02T18:32:24.573413Z"
    },
    "papermill": {
     "duration": 12.981293,
     "end_time": "2020-12-02T18:32:24.574247",
     "exception": false,
     "start_time": "2020-12-02T18:32:11.592954",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting nb_black\r\n",
      "  Downloading nb_black-1.0.7.tar.gz (4.8 kB)\r\n",
      "Requirement already satisfied: ipython in /opt/conda/lib/python3.7/site-packages (from nb_black) (7.13.0)\r\n",
      "Requirement already satisfied: black>='19.3' in /opt/conda/lib/python3.7/site-packages (from nb_black) (19.10b0)\r\n",
      "Requirement already satisfied: jedi>=0.10 in /opt/conda/lib/python3.7/site-packages (from ipython->nb_black) (0.15.2)\r\n",
      "Requirement already satisfied: traitlets>=4.2 in /opt/conda/lib/python3.7/site-packages (from ipython->nb_black) (4.3.3)\r\n",
      "Requirement already satisfied: pickleshare in /opt/conda/lib/python3.7/site-packages (from ipython->nb_black) (0.7.5)\r\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from ipython->nb_black) (3.0.5)\r\n",
      "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /opt/conda/lib/python3.7/site-packages (from ipython->nb_black) (4.8.0)\r\n",
      "Requirement already satisfied: pygments in /opt/conda/lib/python3.7/site-packages (from ipython->nb_black) (2.6.1)\r\n",
      "Requirement already satisfied: backcall in /opt/conda/lib/python3.7/site-packages (from ipython->nb_black) (0.1.0)\r\n",
      "Requirement already satisfied: setuptools>=18.5 in /opt/conda/lib/python3.7/site-packages (from ipython->nb_black) (46.1.3.post20200325)\r\n",
      "Requirement already satisfied: decorator in /opt/conda/lib/python3.7/site-packages (from ipython->nb_black) (4.4.2)\r\n",
      "Requirement already satisfied: appdirs in /opt/conda/lib/python3.7/site-packages (from black>='19.3'->nb_black) (1.4.3)\r\n",
      "Requirement already satisfied: attrs>=18.1.0 in /opt/conda/lib/python3.7/site-packages (from black>='19.3'->nb_black) (19.3.0)\r\n",
      "Requirement already satisfied: click>=6.5 in /opt/conda/lib/python3.7/site-packages (from black>='19.3'->nb_black) (7.1.1)\r\n",
      "Requirement already satisfied: toml>=0.9.4 in /opt/conda/lib/python3.7/site-packages (from black>='19.3'->nb_black) (0.10.0)\r\n",
      "Requirement already satisfied: typed-ast>=1.4.0 in /opt/conda/lib/python3.7/site-packages (from black>='19.3'->nb_black) (1.4.1)\r\n",
      "Requirement already satisfied: regex in /opt/conda/lib/python3.7/site-packages (from black>='19.3'->nb_black) (2020.4.4)\r\n",
      "Requirement already satisfied: pathspec<1,>=0.6 in /opt/conda/lib/python3.7/site-packages (from black>='19.3'->nb_black) (0.8.0)\r\n",
      "Requirement already satisfied: parso>=0.5.2 in /opt/conda/lib/python3.7/site-packages (from jedi>=0.10->ipython->nb_black) (0.5.2)\r\n",
      "Requirement already satisfied: ipython-genutils in /opt/conda/lib/python3.7/site-packages (from traitlets>=4.2->ipython->nb_black) (0.2.0)\r\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.7/site-packages (from traitlets>=4.2->ipython->nb_black) (1.14.0)\r\n",
      "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.7/site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython->nb_black) (0.1.9)\r\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.7/site-packages (from pexpect; sys_platform != \"win32\"->ipython->nb_black) (0.6.0)\r\n",
      "Building wheels for collected packages: nb-black\r\n",
      "  Building wheel for nb-black (setup.py) ... \u001b[?25l-\b \b\\\b \bdone\r\n",
      "\u001b[?25h  Created wheel for nb-black: filename=nb_black-1.0.7-py3-none-any.whl size=5280 sha256=df9d43f8b50bb748b70902d77c44c28c89842ee1c713afc0c6e4bad04efa5cc9\r\n",
      "  Stored in directory: /root/.cache/pip/wheels/1e/b2/88/51c66d23ea5fd0d40ed50997555e15d981d92671376a9a412a\r\n",
      "Successfully built nb-black\r\n",
      "Installing collected packages: nb-black\r\n",
      "Successfully installed nb-black-1.0.7\r\n",
      "\u001b[33mWARNING: You are using pip version 20.1.1; however, version 20.3 is available.\r\n",
      "You should consider upgrading via the '/opt/conda/bin/python3.7 -m pip install --upgrade pip' command.\u001b[0m\r\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 1;\n",
       "                var nbb_unformatted_code = \"!cp -r ../input/vittutorialillustrations/* ./ \\n\\n!pip install nb_black\\n%load_ext nb_black\";\n",
       "                var nbb_formatted_code = \"!cp -r ../input/vittutorialillustrations/* ./ \\n\\n!pip install nb_black\\n%load_ext nb_black\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "!cp -r ../input/vittutorialillustrations/* ./ \n",
    "\n",
    "!pip install nb_black\n",
    "%load_ext nb_black"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.025221,
     "end_time": "2020-12-02T18:32:24.625949",
     "exception": false,
     "start_time": "2020-12-02T18:32:24.600728",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Introduction\n",
    "\n",
    "This notebook essentially has two parts as seen from the title. \n",
    "\n",
    "<a href=\"#Vision-Transformers:-A-gentle-introduction\">1. Vision Transformer: A gentle introduction</a> <br>\n",
    "<a href=\"#Vision-Transformer-Implementation-in-PyTorch\">2. Implementation in PyTorch</a>\n",
    "\n",
    "**I'll briefly try to explain the fundamental ideas behind Vision Transformers and how it works before getting into the implementation of ViT in PyTorch for this competition.**\n",
    "\n",
    "So if you are only interested in code, you can feel free to skip straight ahead to the second section of this notebook. The implementation isn't a whole lot different, thanks to [rwightman/pytorch-image-models](https://github.com/rwightman/pytorch-image-models) library which contains all the model implementations including the pretrained weights for us to use. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.025184,
     "end_time": "2020-12-02T18:32:24.676852",
     "exception": false,
     "start_time": "2020-12-02T18:32:24.651668",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# <font size=4 color='blue'>If you find this notebook useful, leave an upvote, that motivates me to write more such notebooks.</font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.025289,
     "end_time": "2020-12-02T18:32:24.727738",
     "exception": false,
     "start_time": "2020-12-02T18:32:24.702449",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Vision Transformers: A gentle introduction\n",
    "\n",
    "Vision Transformers were first introduced in the paper [AN IMAGE IS WORTH 16X16 WORDS:\n",
    "TRANSFORMERS FOR IMAGE RECOGNITION AT SCALE](https://arxiv.org/pdf/2010.11929.pdf) by the Google Brain team in late October 2020. \n",
    "\n",
    "To understand how ViT works, obviously you must have prior knowledge about how Transformers work and what problems it solved. I'll briefly introduce you to how transformers work before getting into the details of the topic at hand - ViT.\n",
    "\n",
    "![ViT-Illustration](vision-transformer.png)\n",
    "\n",
    "If you are new to NLP and interested in learning more about the transformer models and get a fair intuition of how they actually work, I recomment checking out the fantastic blog posts of [Jay Allamar](https://jalammar.github.io/). (The image above was also inspired from one of his blog posts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.025597,
     "end_time": "2020-12-02T18:32:24.780502",
     "exception": false,
     "start_time": "2020-12-02T18:32:24.754905",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Transformers: A brief overview\n",
    "\n",
    "> **If you already understand Transformers, feel free to skip ahead to the next section.**\n",
    "\n",
    "Transformer models well and truly revolutionized Natural Language Processing as we know. When they were first introduced they broke multiple NLP records and were pushing the then State of the Art. Now, they have become a de-facto standard for modern NLP tasks and they bring spectacular performance gains when compared to the previous generation of models like LSTMs and GRUs.  \n",
    "\n",
    "By far the most important paper that transformed the NLP landscape is the [\"Attention is all you need\"](https://arxiv.org/pdf/1706.03762.pdf) paper. The transformer architecture was introduced in this paper. \n",
    "\n",
    "### **Motivations:**\n",
    "\n",
    "The existing models at that time for sequence and NLP tasks mostly involved RNNs. **The problem with these networks were that they couldn't capture long term dependencies.** \n",
    "\n",
    "LSTMs and GRUs - variants of RNNs were capable of capturing the dependencies but it is also limited. \n",
    "\n",
    "So, the main inspiration behind the transformer was to get rid of this recurrence and still end up capturing almost all the dependencies, to be precise global dependencies, yes the reference window of tranformers is full-range. This was achieved using a variant of attention mechanism called *self-attention* (multi-headed) which is very important for their success. One other advantage of Tranformer models are that they are highly parallelizable.  \n",
    "\n",
    "\n",
    "### Transformer Architecture\n",
    "**Note: The architecture diagrams are annotated with the corresponding step in the explanation.**\n",
    "\n",
    "![TranformerArchitecture](transformer-arch.png)\n",
    "\n",
    "- Transformer has two parts, the decoder which is on the left side on the above diagram and the encoder which is on the right. \n",
    "- Imagine we are doing machine translation for now.\n",
    "- The encoder takes the input data (sentence), and produces an intermediate representation of the input. \n",
    "- The decoder decodes this intermediate representation step by step and generates the output. The difference however is in how it is doing this. \n",
    "- Understanding the Encoder section is enough for ViT. \n",
    "\n",
    "> **Note: The explanations here are more about the intuition behind the architectures. For more mathematical details check out the respective research papers instead.**\n",
    "\n",
    "### Tranformers: Step by step overview\n",
    "**(1)** The input data first gets embedded into a vector. The embedding layer helps us grab a learned vector representation for each word.\n",
    "\n",
    "**(2)** In the next stage a positional encoding is injected into the input embeddings. This is because a transformer has no idea about the order of the sequence that is being passed as input - for example a sentence.\n",
    "\n",
    "**(3)** Now the multi-headed attention is where things get a little different. \n",
    "\n",
    "**Multi-headed-attention architecture:**\n",
    "![multi-headed-attn](multi-headed-attention.png)\n",
    "\n",
    "**(4)** Multi-Headed Attention consists of three learnable vectors. Query, Key and Value vectors. The motivation of this reportedly comes from information retrival where you search (query) and the search engine compares your query with a key and responds with a value. \n",
    "\n",
    "**(5)** The Q and K representations undergo a dot product matrix multiplication to produce a score matrix which represents how much a word has to attend to every other word. Higher score means more attention and vice-versa. \n",
    "\n",
    "**(6)** Then the Score matrix is scaled down according to the dimensions of the Q and K vectors. This is to ensure more stable gradients as multiplication can have exploding effects. \n",
    "\n",
    "(We'll discuss the mask part when we reach the decoder section)\n",
    "\n",
    "**(7)** Next the Score matrix is softmaxed to turn attention scores into probabilities. Obviously higher scores are heightened and lower scores are depressed. This ensures the model to be confident on which words to attend to. \n",
    "\n",
    "**(8)** Then the resultant matrix with probabilites is multiplied with the value vector. This will make the higher probaility scores the model has learned to be more important. The low scoring words will effectively drown out to become irrelevant. \n",
    "\n",
    "**(9)** Then, the concatenated output of QK and V vectors are fed into the Linear layer to process further. \n",
    "\n",
    "**(10)** Self-Attention is performed for each word in the sequence. Since one doesn't depend on the other a copy of the self attention module can be used to process everything simultaneously making this **multi-headed**. \n",
    "\n",
    "**(11)** Then the output value vectors are concatenated and added to the residual connection coming from the input layer and then the resultant respresentation is passed into a *LayerNorm* for normalization. (Residual connection help gradients flow through the network and LayernNorm helps reduce the training time by a small fraction and stabilize the network)\n",
    "\n",
    "**(12)** Further, the output is passed into a point-wise feed forward network to obtain an even richer representation.  \n",
    "\n",
    "**(13)** The outputs are again Layer-normed and residuals are added from the previous layer. \n",
    "\n",
    "\n",
    "**Note: This wraps up the encoder section and trust me this is enough to fully understand Vision Transformer. I'll be largely leaving it up to you to understand the decoder part as it is very similar to the encoding layer.**\n",
    "\n",
    "**(14)** The output from the encoder along with the inputs (if any) from the previous time steps/words are fed into the decoder where the outputs undergo masked-multi headed attention before being fed into the next attention layer along with the output from encoder. \n",
    "\n",
    "**(15)** Masked multi headed attention is necessary because the network shouldn't have any visibility into the words that are to come later in the sequence while decoding, to ensure there is no leak. This is done by masking the entries of words that come later in the series in the Score matrix. Current and previous words in the sequence are added with 1 and the future word scores are added with `-inf`. This ensures the future words in the series get drowned out into 0 when performing softmax to obtain the probabilities, while the rest are retained. \n",
    "\n",
    "**(16)** There are residual connections here as well, to improve the flow of gradients. Finally the output is sent to a Linear layer and softmaxed to obtain the outputs in probabilities. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.026119,
     "end_time": "2020-12-02T18:32:24.833111",
     "exception": false,
     "start_time": "2020-12-02T18:32:24.806992",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## How Vision Tranformers works?\n",
    "\n",
    "Now that we have covered transformers' internal working at a high level, we are finally ready to tackle Vision Tranformers. \n",
    "\n",
    "Applying Transformers on images was always going to be a challenge for the following reasons,\n",
    "- Unlike words/sentences/paragraphs, images contain much much more information in them basically in form of pixels. \n",
    "- It would be very hard, even with current hardware to attend to every other pixel in the image. \n",
    "- Instead, a popular alternative was to use localized attention. \n",
    "- In fact CNNs do something very similar through convolutions and the receptive field essentially grows bigger as we go deeper into the model's layers, but Tranformers were always going to be computationally more expensive than CNNs because of the' nature of Transformers. And of course, we know how incredibly much CNNs have contributed to the current advancements in Computer Vision.\n",
    "\n",
    "Google researchers have proposed something different in their paper than can possibly be the next big step in Computer Vision. They show that the reliance on CNNs may not be necessary anymore. So, let's dive right in and explore more about Vision Transformers.\n",
    "\n",
    "\n",
    "### Vision Transformer Architecture\n",
    "\n",
    "![vit-architecture](vit-arch.png)\n",
    "\n",
    "**(1)** They are only using the Encoder part of the transformer but the difference is in how they are feeding the images into the network. \n",
    "\n",
    "**(2)** They are breaking down the image into fixed size patches. So one of these patches can be of dimension 16x16 or 32x32 as proposed in the paper. More patches means more simpler it is to train these networks as the patches themselves get smaller. Hence we have that in the title - \"An Image is worth 16x16 words\". \n",
    "\n",
    "**(3)** The patches are then unrolled (flattened) and sent for further processing into the network.\n",
    "\n",
    "**(4)** Unlike NNs here the model has no idea whatsoever about the position of the samples in the sequence, here each sample is a patch from the input image. So the image is fed **along with a positional embedding vector** and into the encoder. One thing to note here is the positional embeddings are also learnable so you don't actually feed hard-coded vectors w.r.t to their positions. \n",
    "\n",
    "**(5)** There is also a special token at the start just like BERT.\n",
    "\n",
    "**(6)** So each image patch is first unrolled (flattened) into a big vector and gets multiplied with an embedding matrix which is also learnable, creating embedded patches. And these embedded patches are combined with the positional embedding vector and that gets fed into the Tranformer. \n",
    "\n",
    "> **Note: From here everything is just the same as a standard transformer**\n",
    "\n",
    "**(7)** With the only difference being, instead of a decoder the output from the encoder is passed directly into a Feed Forward Neural Network to obtain the classification output. \n",
    "\n",
    "### Things to note:\n",
    "- The paper ALMOST completely neglects Convolutions. \n",
    "- They are however using a couple of variants of ViT in which Convolutional embeddings of image patches are used. But that doesn't seem to impact performance much.  \n",
    "- Vision Transformers, at the time of writing this are topping Image Classification benchmarks on ImageNet. \n",
    "\n",
    "<img src=\"benchmarks-chart.png\" width=\"700\">\n",
    "<!-- ![BenchmarksChart](benchmarks-charpng) -->\n",
    "\n",
    "- There are a lot more interesting things in this paper but the one thing that stands out for me and potentially shows the power of transformers over CNNs is illustrated in the image below which shows the attention distance with respect to the layers. \n",
    "\n",
    "\n",
    "<img src=\"attn-distance.png\" width=\"300\" height=\"300\">\n",
    "<br>\n",
    "\n",
    "- The graph above suggests that the Transformers are already capable of paying attention to  regions that are far apart right from the starting layers of the network which is a pretty significant gain the Transformers bring over CNNs which has a finite receptive field at the start."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.029479,
     "end_time": "2020-12-02T18:32:24.888750",
     "exception": false,
     "start_time": "2020-12-02T18:32:24.859271",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# <font size=4 color='blue'>If you find this notebook useful, leave an upvote, that motivates me to write more such notebooks.</font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.028891,
     "end_time": "2020-12-02T18:32:24.947471",
     "exception": false,
     "start_time": "2020-12-02T18:32:24.918580",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Vision Transformer Implementation in PyTorch\n",
    "Now that you understand vision transformers, let's build a baseline model for [this competition](https://www.kaggle.com/c/cassava-leaf-disease-classification)\n",
    "\n",
    "First lets install torch-xla to be able to use the TPU and torch-image-models (timm). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_kg_hide-input": true,
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2020-12-02T18:32:25.025299Z",
     "iopub.status.busy": "2020-12-02T18:32:25.024308Z",
     "iopub.status.idle": "2020-12-02T18:34:10.700354Z",
     "shell.execute_reply": "2020-12-02T18:34:10.699640Z"
    },
    "papermill": {
     "duration": 105.718607,
     "end_time": "2020-12-02T18:34:10.700500",
     "exception": false,
     "start_time": "2020-12-02T18:32:24.981893",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\r\n",
      "                                 Dload  Upload   Total   Spent    Left  Speed\r\n",
      "100  5116  100  5116    0     0  16088      0 --:--:-- --:--:-- --:--:-- 16037\r\n",
      "Updating... This may take around 2 minutes.\r\n",
      "Updating TPU runtime to pytorch-1.7 ...\r\n",
      "Found existing installation: torch 1.5.0\r\n",
      "Uninstalling torch-1.5.0:\r\n",
      "  Successfully uninstalled torch-1.5.0\r\n",
      "Found existing installation: torchvision 0.6.0a0+35d732a\r\n",
      "Uninstalling torchvision-0.6.0a0+35d732a:\r\n",
      "Done updating TPU runtime\r\n",
      "  Successfully uninstalled torchvision-0.6.0a0+35d732a\r\n",
      "Copying gs://tpu-pytorch/wheels/torch-1.7-cp37-cp37m-linux_x86_64.whl...\r\n",
      "\r\n",
      "Operation completed over 1 objects/114.2 MiB.                                    \r\n",
      "Copying gs://tpu-pytorch/wheels/torch_xla-1.7-cp37-cp37m-linux_x86_64.whl...\r\n",
      "\r\n",
      "Operation completed over 1 objects/127.4 MiB.                                    \r\n",
      "Copying gs://tpu-pytorch/wheels/torchvision-1.7-cp37-cp37m-linux_x86_64.whl...\r\n",
      "\r\n",
      "Operation completed over 1 objects/3.1 MiB.                                      \r\n",
      "Processing ./torch-1.7-cp37-cp37m-linux_x86_64.whl\r\n",
      "Collecting dataclasses\r\n",
      "  Downloading dataclasses-0.6-py3-none-any.whl (14 kB)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from torch==1.7) (3.7.4.1)\r\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from torch==1.7) (1.18.5)\r\n",
      "Requirement already satisfied: future in /opt/conda/lib/python3.7/site-packages (from torch==1.7) (0.18.2)\r\n",
      "\u001b[31mERROR: fastai 1.0.61 requires torchvision, which is not installed.\u001b[0m\r\n",
      "\u001b[31mERROR: kornia 0.3.1 has requirement torch==1.5.0, but you'll have torch 1.7.0a0 which is incompatible.\u001b[0m\r\n",
      "\u001b[31mERROR: allennlp 1.0.0 has requirement torch<1.6.0,>=1.5.0, but you'll have torch 1.7.0a0 which is incompatible.\u001b[0m\r\n",
      "Installing collected packages: dataclasses, torch\r\n",
      "Successfully installed dataclasses-0.6 torch-1.7.0a0\r\n",
      "\u001b[33mWARNING: You are using pip version 20.1.1; however, version 20.3 is available.\r\n",
      "You should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\u001b[0m\r\n",
      "Processing ./torch_xla-1.7-cp37-cp37m-linux_x86_64.whl\r\n",
      "Installing collected packages: torch-xla\r\n",
      "Successfully installed torch-xla-1.7\r\n",
      "\u001b[33mWARNING: You are using pip version 20.1.1; however, version 20.3 is available.\r\n",
      "You should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\u001b[0m\r\n",
      "Processing ./torchvision-1.7-cp37-cp37m-linux_x86_64.whl\r\n",
      "Requirement already satisfied: torch in /opt/conda/lib/python3.7/site-packages (from torchvision==1.7) (1.7.0a0)\r\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from torchvision==1.7) (1.18.5)\r\n",
      "Requirement already satisfied: pillow>=4.1.1 in /opt/conda/lib/python3.7/site-packages (from torchvision==1.7) (7.2.0)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from torch->torchvision==1.7) (3.7.4.1)\r\n",
      "Requirement already satisfied: future in /opt/conda/lib/python3.7/site-packages (from torch->torchvision==1.7) (0.18.2)\r\n",
      "Requirement already satisfied: dataclasses in /opt/conda/lib/python3.7/site-packages (from torch->torchvision==1.7) (0.6)\r\n",
      "Installing collected packages: torchvision\r\n",
      "Successfully installed torchvision-0.9.0a0+75e4a7d\r\n",
      "\u001b[33mWARNING: You are using pip version 20.1.1; however, version 20.3 is available.\r\n",
      "You should consider upgrading via the '/opt/conda/bin/python -m pip install --upgrade pip' command.\u001b[0m\r\n",
      "\r\n",
      "\r\n",
      "\r\n",
      "The following NEW packages will be installed:\r\n",
      "  libomp5\r\n",
      "0 upgraded, 1 newly installed, 0 to remove and 59 not upgraded.\r\n",
      "Need to get 234 kB of archives.\r\n",
      "After this operation, 774 kB of additional disk space will be used.\r\n",
      "Get:1 http://archive.ubuntu.com/ubuntu bionic/universe amd64 libomp5 amd64 5.0.1-1 [234 kB]\r\n",
      "Fetched 234 kB in 1s (253 kB/s)\r\n",
      "debconf: delaying package configuration, since apt-utils is not installed\r\n",
      "Selecting previously unselected package libomp5:amd64.\r\n",
      "(Reading database ... 107745 files and directories currently installed.)\r\n",
      "Preparing to unpack .../libomp5_5.0.1-1_amd64.deb ...\r\n",
      "Unpacking libomp5:amd64 (5.0.1-1) ...\r\n",
      "Setting up libomp5:amd64 (5.0.1-1) ...\r\n",
      "Processing triggers for libc-bin (2.27-3ubuntu1) ...\r\n",
      "Collecting timm\r\n",
      "  Downloading timm-0.3.1-py3-none-any.whl (247 kB)\r\n",
      "\u001b[K     |████████████████████████████████| 247 kB 3.0 MB/s \r\n",
      "\u001b[?25hRequirement already satisfied: torch>=1.0 in /opt/conda/lib/python3.7/site-packages (from timm) (1.7.0a0)\r\n",
      "Requirement already satisfied: torchvision in /opt/conda/lib/python3.7/site-packages (from timm) (0.9.0a0+75e4a7d)\r\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.7/site-packages (from torch>=1.0->timm) (3.7.4.1)\r\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from torch>=1.0->timm) (1.18.5)\r\n",
      "Requirement already satisfied: future in /opt/conda/lib/python3.7/site-packages (from torch>=1.0->timm) (0.18.2)\r\n",
      "Requirement already satisfied: dataclasses in /opt/conda/lib/python3.7/site-packages (from torch>=1.0->timm) (0.6)\r\n",
      "Requirement already satisfied: pillow>=4.1.1 in /opt/conda/lib/python3.7/site-packages (from torchvision->timm) (7.2.0)\r\n",
      "Installing collected packages: timm\r\n",
      "Successfully installed timm-0.3.1\r\n",
      "\u001b[33mWARNING: You are using pip version 20.1.1; however, version 20.3 is available.\r\n",
      "You should consider upgrading via the '/opt/conda/bin/python3.7 -m pip install --upgrade pip' command.\u001b[0m\r\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 2;\n",
       "                var nbb_unformatted_code = \"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\\n!python pytorch-xla-env-setup.py --version 1.7\\n!pip install timm\";\n",
       "                var nbb_formatted_code = \"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\\n!python pytorch-xla-env-setup.py --version 1.7\\n!pip install timm\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n",
    "!python pytorch-xla-env-setup.py --version 1.7\n",
    "!pip install timm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:10.826084Z",
     "iopub.status.busy": "2020-12-02T18:34:10.824931Z",
     "iopub.status.idle": "2020-12-02T18:34:13.015202Z",
     "shell.execute_reply": "2020-12-02T18:34:13.015867Z"
    },
    "papermill": {
     "duration": 2.258623,
     "end_time": "2020-12-02T18:34:13.016037",
     "exception": false,
     "start_time": "2020-12-02T18:34:10.757414",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 3;\n",
       "                var nbb_unformatted_code = \"import numpy as np\\nimport pandas as pd\\nimport matplotlib.pyplot as plt\\n\\nplt.style.use(\\\"ggplot\\\")\\n\\nimport torch\\nimport torch.nn as nn\\nimport torchvision.transforms as transforms\\n\\nimport torch_xla\\nimport torch_xla.core.xla_model as xm\\nimport torch_xla.distributed.xla_multiprocessing as xmp\\nimport torch_xla.distributed.parallel_loader as pl\\n\\nimport timm\\n\\nimport gc\\nimport os\\nimport time\\nimport random\\nfrom datetime import datetime\\n\\nfrom PIL import Image\\nfrom tqdm.notebook import tqdm\\nfrom sklearn import model_selection, metrics\";\n",
       "                var nbb_formatted_code = \"import numpy as np\\nimport pandas as pd\\nimport matplotlib.pyplot as plt\\n\\nplt.style.use(\\\"ggplot\\\")\\n\\nimport torch\\nimport torch.nn as nn\\nimport torchvision.transforms as transforms\\n\\nimport torch_xla\\nimport torch_xla.core.xla_model as xm\\nimport torch_xla.distributed.xla_multiprocessing as xmp\\nimport torch_xla.distributed.parallel_loader as pl\\n\\nimport timm\\n\\nimport gc\\nimport os\\nimport time\\nimport random\\nfrom datetime import datetime\\n\\nfrom PIL import Image\\nfrom tqdm.notebook import tqdm\\nfrom sklearn import model_selection, metrics\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.style.use(\"ggplot\")\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "import torch_xla\n",
    "import torch_xla.core.xla_model as xm\n",
    "import torch_xla.distributed.xla_multiprocessing as xmp\n",
    "import torch_xla.distributed.parallel_loader as pl\n",
    "\n",
    "import timm\n",
    "\n",
    "import gc\n",
    "import os\n",
    "import time\n",
    "import random\n",
    "from datetime import datetime\n",
    "\n",
    "from PIL import Image\n",
    "from tqdm.notebook import tqdm\n",
    "from sklearn import model_selection, metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:13.138754Z",
     "iopub.status.busy": "2020-12-02T18:34:13.137657Z",
     "iopub.status.idle": "2020-12-02T18:34:13.146711Z",
     "shell.execute_reply": "2020-12-02T18:34:13.145929Z"
    },
    "papermill": {
     "duration": 0.073227,
     "end_time": "2020-12-02T18:34:13.146842",
     "exception": false,
     "start_time": "2020-12-02T18:34:13.073615",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 4;\n",
       "                var nbb_unformatted_code = \"# For parallelization in TPUs\\nos.environ[\\\"XLA_USE_BF16\\\"] = \\\"1\\\"\\nos.environ[\\\"XLA_TENSOR_ALLOCATOR_MAXSIZE\\\"] = \\\"100000000\\\"\";\n",
       "                var nbb_formatted_code = \"# For parallelization in TPUs\\nos.environ[\\\"XLA_USE_BF16\\\"] = \\\"1\\\"\\nos.environ[\\\"XLA_TENSOR_ALLOCATOR_MAXSIZE\\\"] = \\\"100000000\\\"\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# For parallelization in TPUs\n",
    "os.environ[\"XLA_USE_BF16\"] = \"1\"\n",
    "os.environ[\"XLA_TENSOR_ALLOCATOR_MAXSIZE\"] = \"100000000\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:13.273861Z",
     "iopub.status.busy": "2020-12-02T18:34:13.273082Z",
     "iopub.status.idle": "2020-12-02T18:34:13.288963Z",
     "shell.execute_reply": "2020-12-02T18:34:13.288196Z"
    },
    "papermill": {
     "duration": 0.083797,
     "end_time": "2020-12-02T18:34:13.289095",
     "exception": false,
     "start_time": "2020-12-02T18:34:13.205298",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 5;\n",
       "                var nbb_unformatted_code = \"def seed_everything(seed):\\n    \\\"\\\"\\\"\\n    Seeds basic parameters for reproductibility of results\\n    \\n    Arguments:\\n        seed {int} -- Number of the seed\\n    \\\"\\\"\\\"\\n    random.seed(seed)\\n    os.environ[\\\"PYTHONHASHSEED\\\"] = str(seed)\\n    np.random.seed(seed)\\n    torch.manual_seed(seed)\\n    torch.cuda.manual_seed(seed)\\n    torch.backends.cudnn.deterministic = True\\n    torch.backends.cudnn.benchmark = False\\n\\n\\nseed_everything(1001)\";\n",
       "                var nbb_formatted_code = \"def seed_everything(seed):\\n    \\\"\\\"\\\"\\n    Seeds basic parameters for reproductibility of results\\n    \\n    Arguments:\\n        seed {int} -- Number of the seed\\n    \\\"\\\"\\\"\\n    random.seed(seed)\\n    os.environ[\\\"PYTHONHASHSEED\\\"] = str(seed)\\n    np.random.seed(seed)\\n    torch.manual_seed(seed)\\n    torch.cuda.manual_seed(seed)\\n    torch.backends.cudnn.deterministic = True\\n    torch.backends.cudnn.benchmark = False\\n\\n\\nseed_everything(1001)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def seed_everything(seed):\n",
    "    \"\"\"\n",
    "    Seeds basic parameters for reproductibility of results\n",
    "    \n",
    "    Arguments:\n",
    "        seed {int} -- Number of the seed\n",
    "    \"\"\"\n",
    "    random.seed(seed)\n",
    "    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = False\n",
    "\n",
    "\n",
    "seed_everything(1001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:13.419060Z",
     "iopub.status.busy": "2020-12-02T18:34:13.418234Z",
     "iopub.status.idle": "2020-12-02T18:34:13.429184Z",
     "shell.execute_reply": "2020-12-02T18:34:13.428386Z"
    },
    "papermill": {
     "duration": 0.080249,
     "end_time": "2020-12-02T18:34:13.429314",
     "exception": false,
     "start_time": "2020-12-02T18:34:13.349065",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 6;\n",
       "                var nbb_unformatted_code = \"# general global variables\\nDATA_PATH = \\\"../input/cassava-leaf-disease-classification\\\"\\nTRAIN_PATH = \\\"../input/cassava-leaf-disease-classification/train_images/\\\"\\nTEST_PATH = \\\"../input/cassava-leaf-disease-classification/test_images/\\\"\\nMODEL_PATH = (\\n    \\\"../input/vit-base-models-pretrained-pytorch/jx_vit_base_p16_224-80ecf9dd.pth\\\"\\n)\\n\\n# model specific global variables\\nIMG_SIZE = 224\\nBATCH_SIZE = 16\\nLR = 2e-05\\nGAMMA = 0.7\\nN_EPOCHS = 10\";\n",
       "                var nbb_formatted_code = \"# general global variables\\nDATA_PATH = \\\"../input/cassava-leaf-disease-classification\\\"\\nTRAIN_PATH = \\\"../input/cassava-leaf-disease-classification/train_images/\\\"\\nTEST_PATH = \\\"../input/cassava-leaf-disease-classification/test_images/\\\"\\nMODEL_PATH = (\\n    \\\"../input/vit-base-models-pretrained-pytorch/jx_vit_base_p16_224-80ecf9dd.pth\\\"\\n)\\n\\n# model specific global variables\\nIMG_SIZE = 224\\nBATCH_SIZE = 16\\nLR = 2e-05\\nGAMMA = 0.7\\nN_EPOCHS = 10\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# general global variables\n",
    "DATA_PATH = \"../input/cassava-leaf-disease-classification\"\n",
    "TRAIN_PATH = \"../input/cassava-leaf-disease-classification/train_images/\"\n",
    "TEST_PATH = \"../input/cassava-leaf-disease-classification/test_images/\"\n",
    "MODEL_PATH = (\n",
    "    \"../input/vit-base-models-pretrained-pytorch/jx_vit_base_p16_224-80ecf9dd.pth\"\n",
    ")\n",
    "\n",
    "# model specific global variables\n",
    "IMG_SIZE = 224\n",
    "BATCH_SIZE = 16\n",
    "LR = 2e-05\n",
    "GAMMA = 0.7\n",
    "N_EPOCHS = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:13.557875Z",
     "iopub.status.busy": "2020-12-02T18:34:13.557096Z",
     "iopub.status.idle": "2020-12-02T18:34:13.603956Z",
     "shell.execute_reply": "2020-12-02T18:34:13.603128Z"
    },
    "papermill": {
     "duration": 0.114526,
     "end_time": "2020-12-02T18:34:13.604087",
     "exception": false,
     "start_time": "2020-12-02T18:34:13.489561",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1000015157.jpg</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000201771.jpg</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100042118.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1000723321.jpg</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1000812911.jpg</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         image_id  label\n",
       "0  1000015157.jpg      0\n",
       "1  1000201771.jpg      3\n",
       "2   100042118.jpg      1\n",
       "3  1000723321.jpg      1\n",
       "4  1000812911.jpg      3"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 7;\n",
       "                var nbb_unformatted_code = \"df = pd.read_csv(os.path.join(DATA_PATH, \\\"train.csv\\\"))\\ndf.head()\";\n",
       "                var nbb_formatted_code = \"df = pd.read_csv(os.path.join(DATA_PATH, \\\"train.csv\\\"))\\ndf.head()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.read_csv(os.path.join(DATA_PATH, \"train.csv\"))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:13.744267Z",
     "iopub.status.busy": "2020-12-02T18:34:13.736833Z",
     "iopub.status.idle": "2020-12-02T18:34:13.752203Z",
     "shell.execute_reply": "2020-12-02T18:34:13.752809Z"
    },
    "papermill": {
     "duration": 0.087447,
     "end_time": "2020-12-02T18:34:13.752968",
     "exception": false,
     "start_time": "2020-12-02T18:34:13.665521",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 21397 entries, 0 to 21396\n",
      "Data columns (total 2 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   image_id  21397 non-null  object\n",
      " 1   label     21397 non-null  int64 \n",
      "dtypes: int64(1), object(1)\n",
      "memory usage: 334.5+ KB\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 8;\n",
       "                var nbb_unformatted_code = \"df.info()\";\n",
       "                var nbb_formatted_code = \"df.info()\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:13.889294Z",
     "iopub.status.busy": "2020-12-02T18:34:13.888176Z",
     "iopub.status.idle": "2020-12-02T18:34:14.103989Z",
     "shell.execute_reply": "2020-12-02T18:34:14.103314Z"
    },
    "papermill": {
     "duration": 0.288382,
     "end_time": "2020-12-02T18:34:14.104118",
     "exception": false,
     "start_time": "2020-12-02T18:34:13.815736",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f921783b6d0>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD1CAYAAABQtIIDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAYnUlEQVR4nO3dX2xT9/3/8eexHSCQEvyHkCZNhWgSVdCAsxoVMpVA8VVLEeKi0qpWIkDLivYHok0g0Kpp09RUXRKakoAUITZNvemkJqv2ZZrkGRKpFpKBRaLt2pCFdUsJTeLjBhhpHWL/LjjYP0oYwU6xCa/HnT/Hn+b9eavidT6fk9hGIpFIICIiDzxbtgsQEZHcoEAQERFAgSAiIhYFgoiIAAoEERGxKBBERARQIIiIiMWR7QIyceHChWyXgMfjYWRkJNtl5AT14jr1IUW9SMmVXpSUlNz2mnYIIiICKBBERMSiQBAREUCBICIiFgWCiIgACgQREbEoEEREBFAgiIiI5b7+w7RMTbyyMeP/xpcZzre3f5BxDSIi00E7BBERARQIIiJiUSCIiAigQBAREYsCQUREAAWCiIhYFAgiIgIoEERExKJAEBERQIEgIiIWBYKIiAAKBBERsSgQREQEUCCIiIjljh9/3dbWxpkzZygsLKSxsRGAP/zhD5w+fRqHw8GiRYvYuXMn8+bNA6Cjo4NgMIjNZqOurg6v1wtAf38/ra2txGIxqqurqaurwzAMxsfHOXjwIP39/Tz00EPs2rWLoqKi73DJIiIymTvuENauXcu+fftuGlu+fDmNjY389re/5eGHH6ajowOAgYEBQqEQTU1N7N+/nyNHjhCPxwFob29nx44dtLS0cPHiRXp6egAIBoPMmzePd955h+eee4533313utcoIiJTcMdAWLp0KQUFBTeNrVixArvdDkBlZSWmaQIQDoepqakhLy+PoqIiiouL6evrIxqNMjY2RmVlJYZhsGbNGsLhMACnTp1i7dq1AKxatYqPPvqIRCIxnWsUEZEpyPgZQjAYTB4LmaaJ2+1OXnO5XJimecu42+1Ohsj/f81utzN37lwuX76caVkiInKXMvoKzffffx+73c7TTz8NcNs7+/91xz/ZNcMwJn1vIBAgEAgA0NDQgMfjuduSb5Lp119Oh0zXkEscDseMWk+61IcU9SLlfuhF2oFw4sQJTp8+zeuvv578B9ztdhOJRJLvMU0Tl8t1y3gkEsHlct00x+12MzExwdWrV285orrB7/fj9/uTr0dGRtItP2fMhDXc4PF4ZtR60qU+pKgXKbnSi5KSktteS+vIqKenhz/96U/s2bOH2bNnJ8d9Ph+hUIjx8XGGhoYYHBykvLwcp9NJfn4+vb29JBIJuru78fl8ADz55JOcOHECgJMnT7Js2bLb7hBEROS7YyTu8AT3wIEDfPLJJ1y+fJnCwkJeeOEFOjo6uHbtWvJOvqKigldffRW4fox0/PhxbDYbW7Zsobq6GoB//vOftLW1EYvF8Hq9bN26FcMwiMViHDx4kPPnz1NQUMCuXbtYtGjRlIq/cOFCJmtn4pWNGc2fDvb2D7JdwrTJlTugbFMfUtSLlFzpxf/aIdwxEHKZAiG35Mr/8NmmPqSoFym50otpPzISEZGZR4EgIiKAAkFERCwKBBERARQIIiJiUSCIiAigQBAREYsCQUREAAWCiIhYFAgiIgIoEERExKJAEBERQIEgIiIWBYKIiAAKBBERsSgQREQEUCCIiIhFgSAiIoACQURELAoEEREBFAgiImJRIIiICACOO72hra2NM2fOUFhYSGNjIwBXrlyhubmZ4eFhFi5cyO7duykoKACgo6ODYDCIzWajrq4Or9cLQH9/P62trcRiMaqrq6mrq8MwDMbHxzl48CD9/f089NBD7Nq1i6Kiou9wySIiMpk77hDWrl3Lvn37bhrr7OykqqqKlpYWqqqq6OzsBGBgYIBQKERTUxP79+/nyJEjxONxANrb29mxYwctLS1cvHiRnp4eAILBIPPmzeOdd97hueee4913353uNYqIyBTcMRCWLl2avPu/IRwOU1tbC0BtbS3hcDg5XlNTQ15eHkVFRRQXF9PX10c0GmVsbIzKykoMw2DNmjXJOadOnWLt2rUArFq1io8++ohEIjGdaxQRkSlI6xnC6OgoTqcTAKfTyaVLlwAwTRO32518n8vlwjTNW8bdbjemad4yx263M3fuXC5fvpzeakREJG13fIZwN253Z/+/7vgnu2YYxqTvDQQCBAIBABoaGvB4PGlUmfJlRrOnR6ZryCUOh2NGrSdd6kOKepFyP/QirUAoLCwkGo3idDqJRqPMnz8fuH7nH4lEku8zTROXy3XLeCQSweVy3TTH7XYzMTHB1atXbzmiusHv9+P3+5OvR0ZG0ik/p8yENdzg8Xhm1HrSpT6kqBcpudKLkpKS215L68jI5/PR1dUFQFdXFytXrkyOh0IhxsfHGRoaYnBwkPLycpxOJ/n5+fT29pJIJOju7sbn8wHw5JNPcuLECQBOnjzJsmXLbrtDEBGR784ddwgHDhzgk08+4fLly/zwhz/khRdeYNOmTTQ3NxMMBvF4PNTX1wNQVlbG6tWrqa+vx2azsW3bNmy265mzfft22traiMVieL1eqqurAXjmmWc4ePAgP/7xjykoKGDXrl3f4XJFROR2jMR9/Cs9Fy5cyGj+xCsbp6mS9NnbP8h2CdMmV7bE2aY+pKgXKbnSi2k/MhIRkZlHgSAiIoACQURELAoEEREBFAgiImJRIIiICKBAEBERiwJBREQABYKIiFgUCCIiAigQRETEokAQERFAgSAiIhYFgoiIAAoEERGxKBBERARQIIiIiEWBICIigAJBREQsCgQREQEUCCIiYlEgiIgIoEAQERGLI5PJf/7znwkGgxiGQVlZGTt37iQWi9Hc3Mzw8DALFy5k9+7dFBQUANDR0UEwGMRms1FXV4fX6wWgv7+f1tZWYrEY1dXV1NXVYRhG5qsTEZEpS3uHYJomf/nLX2hoaKCxsZF4PE4oFKKzs5OqqipaWlqoqqqis7MTgIGBAUKhEE1NTezfv58jR44Qj8cBaG9vZ8eOHbS0tHDx4kV6enqmZ3UiIjJlGR0ZxeNxYrEYExMTxGIxnE4n4XCY2tpaAGprawmHwwCEw2FqamrIy8ujqKiI4uJi+vr6iEajjI2NUVlZiWEYrFmzJjlHRETunbSPjFwuF88//zyvvfYas2bNYsWKFaxYsYLR0VGcTicATqeTS5cuAdd3FBUVFTfNN00Tu92O2+1OjrvdbkzTTLcsERFJU9qBcOXKFcLhMK2trcydO5empia6u7tv+/5EInFX45MJBAIEAgEAGhoa8Hg8d1f0t3yZ0ezpkekaconD4ZhR60mX+pCiXqTcD71IOxDOnj1LUVER8+fPB+Cpp56it7eXwsJCotEoTqeTaDSavO52u4lEIsn5pmnicrluGY9EIrhcrkl/pt/vx+/3J1+PjIykW37OmAlruMHj8cyo9aRLfUhRL1JypRclJSW3vZb2MwSPx8O5c+f45ptvSCQSnD17ltLSUnw+H11dXQB0dXWxcuVKAHw+H6FQiPHxcYaGhhgcHKS8vByn00l+fj69vb0kEgm6u7vx+XzpliUiImlKe4dQUVHBqlWr2LNnD3a7ncWLF+P3+/n6669pbm4mGAzi8Xior68HoKysjNWrV1NfX4/NZmPbtm3YbNfzaPv27bS1tRGLxfB6vVRXV0/P6kREZMqMxN0c4ueYCxcuZDR/4pWN01RJ+uztH2S7hGmTK1vibFMfUtSLlFzpxXdyZCQiIjOLAkFERAAFgoiIWBQIIiICKBBERMSiQBAREUCBICIiFgWCiIgACgQREbEoEEREBFAgiIiIRYEgIiKAAkFERCwKBBERARQIIiJiUSCIiAigQBAREYsCQUREAAWCiIhYFAgiIgIoEERExKJAEBERAByZTP7vf//L4cOH+c9//oNhGLz22muUlJTQ3NzM8PAwCxcuZPfu3RQUFADQ0dFBMBjEZrNRV1eH1+sFoL+/n9bWVmKxGNXV1dTV1WEYRuarExGRKctoh3D06FG8Xi8HDhzgrbfeorS0lM7OTqqqqmhpaaGqqorOzk4ABgYGCIVCNDU1sX//fo4cOUI8Hgegvb2dHTt20NLSwsWLF+np6cl8ZSIiclfSDoSrV6/yj3/8g2eeeQYAh8PBvHnzCIfD1NbWAlBbW0s4HAYgHA5TU1NDXl4eRUVFFBcX09fXRzQaZWxsjMrKSgzDYM2aNck5IiJy76R9ZDQ0NMT8+fNpa2vj888/Z8mSJWzZsoXR0VGcTicATqeTS5cuAWCaJhUVFcn5LpcL0zSx2+243e7kuNvtxjTNdMsSEZE0pR0IExMTnD9/nq1bt1JRUcHRo0eTx0OTSSQSdzU+mUAgQCAQAKChoQGPx3N3RX/LlxnNnh6ZriGXOByOGbWedKkPKepFyv3Qi7QDwe1243a7k3f9q1atorOzk8LCQqLRKE6nk2g0yvz585Pvj0QiyfmmaeJyuW4Zj0QiuFyuSX+m3+/H7/cnX4+MjKRbfs6YCWu4wePxzKj1pEt9SFEvUnKlFyUlJbe9lvYzhAULFuB2u7lw4QIAZ8+e5ZFHHsHn89HV1QVAV1cXK1euBMDn8xEKhRgfH2doaIjBwUHKy8txOp3k5+fT29tLIpGgu7sbn8+XblkiIpKmjH7tdOvWrbS0tHDt2jWKiorYuXMniUSC5uZmgsEgHo+H+vp6AMrKyli9ejX19fXYbDa2bduGzXY9j7Zv305bWxuxWAyv10t1dXXmKxMRkbtiJO7mED/H3NidpGvilY3TVEn67O0fZLuEaZMrW+JsUx9S1IuUXOnFd3JkJCIiM4sCQUREAAWCiIhYFAgiIgIoEERExKJAEBERQIEgIiIWBYKIiAAKBBERsSgQREQEUCCIiIhFgSAiIoACQURELAoEEREBFAgiImJRIIiICKBAEBERiwJBREQABYKIiFgUCCIiAigQRETEokAQERFAgSAiIhZHpv+BeDzO3r17cblc7N27lytXrtDc3Mzw8DALFy5k9+7dFBQUANDR0UEwGMRms1FXV4fX6wWgv7+f1tZWYrEY1dXV1NXVYRhGpqWJiMhdyHiHcOzYMUpLS5OvOzs7qaqqoqWlhaqqKjo7OwEYGBggFArR1NTE/v37OXLkCPF4HID29nZ27NhBS0sLFy9epKenJ9OyRETkLmUUCJFIhDNnzrB+/frkWDgcpra2FoDa2lrC4XByvKamhry8PIqKiiguLqavr49oNMrY2BiVlZUYhsGaNWuSc0RE5N7J6Mjod7/7HS+99BJjY2PJsdHRUZxOJwBOp5NLly4BYJomFRUVyfe5XC5M08Rut+N2u5Pjbrcb0zQn/XmBQIBAIABAQ0MDHo8nk/L5MqPZ0yPTNeQSh8Mxo9aTLvUhRb1IuR96kXYgnD59msLCQpYsWcLHH398x/cnEom7Gp+M3+/H7/cnX4+MjEx5bq6aCWu4wePxzKj1pEt9SFEvUnKlFyUlJbe9lnYgfPbZZ5w6dYq///3vxGIxxsbGaGlpobCwkGg0itPpJBqNMn/+fOD6nX8kEknON00Tl8t1y3gkEsHlcqVbloiIpCntZwgvvvgihw8fprW1lV27dvHEE0/wk5/8BJ/PR1dXFwBdXV2sXLkSAJ/PRygUYnx8nKGhIQYHBykvL8fpdJKfn09vby+JRILu7m58Pt/0rE5ERKYs4187/bZNmzbR3NxMMBjE4/FQX18PQFlZGatXr6a+vh6bzca2bduw2a7n0fbt22lrayMWi+H1eqmurp7uskRE5A6MxN0c4ueYCxcuZDR/4pWN01RJ+uztH2S7hGmTK2ek2aY+pKgXKbnSi//1DEF/qSwiIoACQURELAoEEREBFAgiImJRIIiICKBAEBERiwJBREQABYKIiFgUCCIiAigQRETEokAQERFAgSAiIhYFgoiIAAoEERGxKBBERARQIIiIiEWBICIiwHfwFZpyf5qOb4/7MsP5M+nb40TuR9ohiIgIoEAQERGLjoxEviXT47NMj85Ax2eSHdohiIgIkMEOYWRkhNbWVr766isMw8Dv9/Pss89y5coVmpubGR4eZuHChezevZuCggIAOjo6CAaD2Gw26urq8Hq9APT399Pa2kosFqO6upq6ujoMw5ieFYqIyJSkHQh2u52XX36ZJUuWMDY2xt69e1m+fDknTpygqqqKTZs20dnZSWdnJy+99BIDAwOEQiGampqIRqP8+te/5u2338Zms9He3s6OHTuoqKjgjTfeoKenh+rq6ulcp4ikQcdnD5a0j4ycTidLliwBID8/n9LSUkzTJBwOU1tbC0BtbS3hcBiAcDhMTU0NeXl5FBUVUVxcTF9fH9FolLGxMSorKzEMgzVr1iTniIjIvTMtzxCGhoY4f/485eXljI6O4nQ6geuhcenSJQBM08TtdifnuFwuTNO8ZdztdmOa5nSUJSIidyHj3zL6+uuvaWxsZMuWLcydO/e270skEnc1PplAIEAgEACgoaEBj8dzd8V+y3RsZzOV6Rqmi3qRol6kqBfTx+Fw5PxaMgqEa9eu0djYyNNPP81TTz0FQGFhIdFoFKfTSTQaZf78+cD1O/9IJJKca5omLpfrlvFIJILL5Zr05/n9fvx+f/L1yMhIJuXnhJmwhumiXqSoFykzpRcejycn1lJSUnLba2kfGSUSCQ4fPkxpaSkbNmxIjvt8Prq6ugDo6upi5cqVyfFQKMT4+DhDQ0MMDg5SXl6O0+kkPz+f3t5eEokE3d3d+Hy+dMsSEZE0pb1D+Oyzz+ju7ubRRx/l5z//OQA/+MEP2LRpE83NzQSDQTweD/X19QCUlZWxevVq6uvrsdlsbNu2DZvteh5t376dtrY2YrEYXq9Xv2EkIpIFaQfC448/znvvvTfptddff33S8c2bN7N58+Zbxh977DEaGxvTLUVERKaB/lJZREQABYKIiFgUCCIiAigQRETEokAQERFAgSAiIhYFgoiIAAoEERGxKBBERATQdyqLiEzJg/BlQdohiIgIoEAQERGLAkFERAAFgoiIWBQIIiICKBBERMSiQBAREUCBICIiFgWCiIgACgQREbEoEEREBFAgiIiIRYEgIiJADn3aaU9PD0ePHiUej7N+/Xo2bdqU7ZJERB4oObFDiMfjHDlyhH379tHc3MyHH37IwMBAtssSEXmg5EQg9PX1UVxczKJFi3A4HNTU1BAOh7NdlojIAyUnjoxM08Ttdidfu91uzp07d8v7AoEAgUAAgIaGBkpKSjL7wf93KrP5M4l6kaJepKgXKQ9AL3Jih5BIJG4ZMwzjljG/309DQwMNDQ33oqwp2bt3b7ZLyBnqxXXqQ4p6kXI/9CInAsHtdhOJRJKvI5EITqczixWJiDx4ciIQHnvsMQYHBxkaGuLatWuEQiF8Pl+2yxIReaDkxDMEu93O1q1b+c1vfkM8HmfdunWUlZVlu6wp8fv92S4hZ6gX16kPKepFyv3QCyMx2QG+iIg8cHLiyEhERLJPgSAiIoACQURELDnxUPl+0dfXB0B5eTkDAwP09PRQUlLC9773vSxXln0HDx7kRz/6UbbLyIovvvgC0zSpqKhgzpw5yfGenh68Xm8WK5Ns+uKLLwiHw5imiWEYOJ1OfD4fjzzySLZLuy09VJ6iP/7xj/T09DAxMcHy5cs5d+4cy5Yt4+zZs6xYsYLNmzdnu8R75s0337zpdSKR4OOPP+aJJ54AYM+ePdkoKyuOHTvGX//6V0pLS/n888/ZsmULK1euBK734du9epAdP36cdevWZbuMe6Kzs5MPP/yQ73//+7hcLuD6JzLcGMvVD+/UDmGKTp48yVtvvcX4+Divvvoqhw4dYu7cuWzcuJF9+/Y9UIFgmialpaWsX78ewzBIJBL09/fz/PPPZ7u0e+5vf/sbb775JnPmzGFoaIimpiaGh4d59tlnJ/0L/AfZe++998AEwvHjx2lsbMThuPmf2A0bNlBfX69AuN/Z7XZsNhuzZ89m0aJFzJ07F4BZs2ZN+jEbM9kbb7zBsWPHeP/993n55ZdZvHgxs2bNYunSpdku7Z6Lx+PJY6KioiJ++ctf0tjYyPDw8AMZCD/72c8mHU8kEoyOjt7jarLHMAyi0SgLFy68aTwajeb0vxcKhClyOBx88803zJ49+6bPUrp69So224P1bN5ms7FhwwZWr17N73//ewoLC5mYmMh2WVmxYMEC/vWvf7F48WIA5syZw969ezl06BD//ve/s1tcFoyOjrJ//37mzZt303gikeAXv/hFlqq697Zs2cKvfvUrHn744eQHd46MjHDx4kW2bduW5epuT88Qpmh8fJy8vLxbxi9dusRXX33Fo48+moWqcsOZM2f49NNPefHFF7Ndyj0XiUSw2+0sWLDglmuffvopjz/+eBaqyp5Dhw6xbt26Sdf99ttv89Of/jQLVWVHPB6nr68P0zQBcLlclJeX5/QNpAJBREQA/R2CiIhYFAgiIgIoEERExKJAEBERQIEgIiKW/wd4aez+y6F70gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 9;\n",
       "                var nbb_unformatted_code = \"df.label.value_counts().plot(kind=\\\"bar\\\")\";\n",
       "                var nbb_formatted_code = \"df.label.value_counts().plot(kind=\\\"bar\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df.label.value_counts().plot(kind=\"bar\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:14.240631Z",
     "iopub.status.busy": "2020-12-02T18:34:14.239815Z",
     "iopub.status.idle": "2020-12-02T18:34:14.272019Z",
     "shell.execute_reply": "2020-12-02T18:34:14.271271Z"
    },
    "papermill": {
     "duration": 0.102979,
     "end_time": "2020-12-02T18:34:14.272159",
     "exception": false,
     "start_time": "2020-12-02T18:34:14.169180",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 10;\n",
       "                var nbb_unformatted_code = \"train_df, valid_df = model_selection.train_test_split(\\n    df, test_size=0.1, random_state=42, stratify=df.label.values\\n)\";\n",
       "                var nbb_formatted_code = \"train_df, valid_df = model_selection.train_test_split(\\n    df, test_size=0.1, random_state=42, stratify=df.label.values\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "train_df, valid_df = model_selection.train_test_split(\n",
    "    df, test_size=0.1, random_state=42, stratify=df.label.values\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:14.419320Z",
     "iopub.status.busy": "2020-12-02T18:34:14.418142Z",
     "iopub.status.idle": "2020-12-02T18:34:14.444009Z",
     "shell.execute_reply": "2020-12-02T18:34:14.443354Z"
    },
    "papermill": {
     "duration": 0.106308,
     "end_time": "2020-12-02T18:34:14.444148",
     "exception": false,
     "start_time": "2020-12-02T18:34:14.337840",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 11;\n",
       "                var nbb_unformatted_code = \"class CassavaDataset(torch.utils.data.Dataset):\\n    \\\"\\\"\\\"\\n    Helper Class to create the pytorch dataset\\n    \\\"\\\"\\\"\\n\\n    def __init__(self, df, data_path=DATA_PATH, mode=\\\"train\\\", transforms=None):\\n        super().__init__()\\n        self.df_data = df.values\\n        self.data_path = data_path\\n        self.transforms = transforms\\n        self.mode = mode\\n        self.data_dir = \\\"train_images\\\" if mode == \\\"train\\\" else \\\"test_images\\\"\\n\\n    def __len__(self):\\n        return len(self.df_data)\\n\\n    def __getitem__(self, index):\\n        img_name, label = self.df_data[index]\\n        img_path = os.path.join(self.data_path, self.data_dir, img_name)\\n        img = Image.open(img_path).convert(\\\"RGB\\\")\\n\\n        if self.transforms is not None:\\n            image = self.transforms(img)\\n\\n        return image, label\";\n",
       "                var nbb_formatted_code = \"class CassavaDataset(torch.utils.data.Dataset):\\n    \\\"\\\"\\\"\\n    Helper Class to create the pytorch dataset\\n    \\\"\\\"\\\"\\n\\n    def __init__(self, df, data_path=DATA_PATH, mode=\\\"train\\\", transforms=None):\\n        super().__init__()\\n        self.df_data = df.values\\n        self.data_path = data_path\\n        self.transforms = transforms\\n        self.mode = mode\\n        self.data_dir = \\\"train_images\\\" if mode == \\\"train\\\" else \\\"test_images\\\"\\n\\n    def __len__(self):\\n        return len(self.df_data)\\n\\n    def __getitem__(self, index):\\n        img_name, label = self.df_data[index]\\n        img_path = os.path.join(self.data_path, self.data_dir, img_name)\\n        img = Image.open(img_path).convert(\\\"RGB\\\")\\n\\n        if self.transforms is not None:\\n            image = self.transforms(img)\\n\\n        return image, label\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "class CassavaDataset(torch.utils.data.Dataset):\n",
    "    \"\"\"\n",
    "    Helper Class to create the pytorch dataset\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, df, data_path=DATA_PATH, mode=\"train\", transforms=None):\n",
    "        super().__init__()\n",
    "        self.df_data = df.values\n",
    "        self.data_path = data_path\n",
    "        self.transforms = transforms\n",
    "        self.mode = mode\n",
    "        self.data_dir = \"train_images\" if mode == \"train\" else \"test_images\"\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df_data)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        img_name, label = self.df_data[index]\n",
    "        img_path = os.path.join(self.data_path, self.data_dir, img_name)\n",
    "        img = Image.open(img_path).convert(\"RGB\")\n",
    "\n",
    "        if self.transforms is not None:\n",
    "            image = self.transforms(img)\n",
    "\n",
    "        return image, label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:14.590922Z",
     "iopub.status.busy": "2020-12-02T18:34:14.589829Z",
     "iopub.status.idle": "2020-12-02T18:34:14.612461Z",
     "shell.execute_reply": "2020-12-02T18:34:14.611720Z"
    },
    "papermill": {
     "duration": 0.101631,
     "end_time": "2020-12-02T18:34:14.612584",
     "exception": false,
     "start_time": "2020-12-02T18:34:14.510953",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 12;\n",
       "                var nbb_unformatted_code = \"# create image augmentations\\ntransforms_train = transforms.Compose(\\n    [\\n        transforms.Resize((IMG_SIZE, IMG_SIZE)),\\n        transforms.RandomHorizontalFlip(p=0.3),\\n        transforms.RandomVerticalFlip(p=0.3),\\n        transforms.RandomResizedCrop(IMG_SIZE),\\n        transforms.ToTensor(),\\n        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\\n    ]\\n)\\n\\ntransforms_valid = transforms.Compose(\\n    [\\n        transforms.Resize((IMG_SIZE, IMG_SIZE)),\\n        transforms.ToTensor(),\\n        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\\n    ]\\n)\";\n",
       "                var nbb_formatted_code = \"# create image augmentations\\ntransforms_train = transforms.Compose(\\n    [\\n        transforms.Resize((IMG_SIZE, IMG_SIZE)),\\n        transforms.RandomHorizontalFlip(p=0.3),\\n        transforms.RandomVerticalFlip(p=0.3),\\n        transforms.RandomResizedCrop(IMG_SIZE),\\n        transforms.ToTensor(),\\n        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\\n    ]\\n)\\n\\ntransforms_valid = transforms.Compose(\\n    [\\n        transforms.Resize((IMG_SIZE, IMG_SIZE)),\\n        transforms.ToTensor(),\\n        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\\n    ]\\n)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# create image augmentations\n",
    "transforms_train = transforms.Compose(\n",
    "    [\n",
    "        transforms.Resize((IMG_SIZE, IMG_SIZE)),\n",
    "        transforms.RandomHorizontalFlip(p=0.3),\n",
    "        transforms.RandomVerticalFlip(p=0.3),\n",
    "        transforms.RandomResizedCrop(IMG_SIZE),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n",
    "    ]\n",
    ")\n",
    "\n",
    "transforms_valid = transforms.Compose(\n",
    "    [\n",
    "        transforms.Resize((IMG_SIZE, IMG_SIZE)),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:14.755007Z",
     "iopub.status.busy": "2020-12-02T18:34:14.753893Z",
     "iopub.status.idle": "2020-12-02T18:34:14.765112Z",
     "shell.execute_reply": "2020-12-02T18:34:14.764319Z"
    },
    "papermill": {
     "duration": 0.08523,
     "end_time": "2020-12-02T18:34:14.765245",
     "exception": false,
     "start_time": "2020-12-02T18:34:14.680015",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Available Vision Transformer Models: \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['vit_base_patch16_224',\n",
       " 'vit_base_patch16_384',\n",
       " 'vit_base_patch32_384',\n",
       " 'vit_base_resnet26d_224',\n",
       " 'vit_base_resnet50d_224',\n",
       " 'vit_huge_patch16_224',\n",
       " 'vit_huge_patch32_384',\n",
       " 'vit_large_patch16_224',\n",
       " 'vit_large_patch16_384',\n",
       " 'vit_large_patch32_384',\n",
       " 'vit_small_patch16_224',\n",
       " 'vit_small_resnet26d_224',\n",
       " 'vit_small_resnet50d_s3_224']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 13;\n",
       "                var nbb_unformatted_code = \"print(\\\"Available Vision Transformer Models: \\\")\\ntimm.list_models(\\\"vit*\\\")\";\n",
       "                var nbb_formatted_code = \"print(\\\"Available Vision Transformer Models: \\\")\\ntimm.list_models(\\\"vit*\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Available Vision Transformer Models: \")\n",
    "timm.list_models(\"vit*\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:14.936128Z",
     "iopub.status.busy": "2020-12-02T18:34:14.924554Z",
     "iopub.status.idle": "2020-12-02T18:34:15.031764Z",
     "shell.execute_reply": "2020-12-02T18:34:15.030784Z"
    },
    "papermill": {
     "duration": 0.197739,
     "end_time": "2020-12-02T18:34:15.031965",
     "exception": false,
     "start_time": "2020-12-02T18:34:14.834226",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 14;\n",
       "                var nbb_unformatted_code = \"class ViTBase16(nn.Module):\\n    def __init__(self, n_classes, pretrained=False):\\n\\n        super(ViTBase16, self).__init__()\\n\\n        self.model = timm.create_model(\\\"vit_base_patch16_224\\\", pretrained=False)\\n        if pretrained:\\n            self.model.load_state_dict(torch.load(MODEL_PATH))\\n\\n        self.model.head = nn.Linear(self.model.head.in_features, n_classes)\\n\\n    def forward(self, x):\\n        x = self.model(x)\\n        return x\\n\\n    def train_one_epoch(self, train_loader, criterion, optimizer, device):\\n        # keep track of training loss\\n        epoch_loss = 0.0\\n        epoch_accuracy = 0.0\\n\\n        ###################\\n        # train the model #\\n        ###################\\n        self.model.train()\\n        for i, (data, target) in enumerate(train_loader):\\n            # move tensors to GPU if CUDA is available\\n            if device.type == \\\"cuda\\\":\\n                data, target = data.cuda(), target.cuda()\\n            elif device.type == \\\"xla\\\":\\n                data = data.to(device, dtype=torch.float32)\\n                target = target.to(device, dtype=torch.int64)\\n\\n            # clear the gradients of all optimized variables\\n            optimizer.zero_grad()\\n            # forward pass: compute predicted outputs by passing inputs to the model\\n            output = self.forward(data)\\n            # calculate the batch loss\\n            loss = criterion(output, target)\\n            # backward pass: compute gradient of the loss with respect to model parameters\\n            loss.backward()\\n            # Calculate Accuracy\\n            accuracy = (output.argmax(dim=1) == target).float().mean()\\n            # update training loss and accuracy\\n            epoch_loss += loss\\n            epoch_accuracy += accuracy\\n\\n            # perform a single optimization step (parameter update)\\n            if device.type == \\\"xla\\\":\\n                xm.optimizer_step(optimizer)\\n\\n                if i % 20 == 0:\\n                    xm.master_print(f\\\"\\\\tBATCH {i+1}/{len(train_loader)} - LOSS: {loss}\\\")\\n\\n            else:\\n                optimizer.step()\\n\\n        return epoch_loss / len(train_loader), epoch_accuracy / len(train_loader)\\n\\n    def validate_one_epoch(self, valid_loader, criterion, device):\\n        # keep track of validation loss\\n        valid_loss = 0.0\\n        valid_accuracy = 0.0\\n\\n        ######################\\n        # validate the model #\\n        ######################\\n        self.model.eval()\\n        for data, target in valid_loader:\\n            # move tensors to GPU if CUDA is available\\n            if device.type == \\\"cuda\\\":\\n                data, target = data.cuda(), target.cuda()\\n            elif device.type == \\\"xla\\\":\\n                data = data.to(device, dtype=torch.float32)\\n                target = target.to(device, dtype=torch.int64)\\n\\n            with torch.no_grad():\\n                # forward pass: compute predicted outputs by passing inputs to the model\\n                output = self.model(data)\\n                # calculate the batch loss\\n                loss = criterion(output, target)\\n                # Calculate Accuracy\\n                accuracy = (output.argmax(dim=1) == target).float().mean()\\n                # update average validation loss and accuracy\\n                valid_loss += loss\\n                valid_accuracy += accuracy\\n\\n        return valid_loss / len(valid_loader), valid_accuracy / len(valid_loader)\";\n",
       "                var nbb_formatted_code = \"class ViTBase16(nn.Module):\\n    def __init__(self, n_classes, pretrained=False):\\n\\n        super(ViTBase16, self).__init__()\\n\\n        self.model = timm.create_model(\\\"vit_base_patch16_224\\\", pretrained=False)\\n        if pretrained:\\n            self.model.load_state_dict(torch.load(MODEL_PATH))\\n\\n        self.model.head = nn.Linear(self.model.head.in_features, n_classes)\\n\\n    def forward(self, x):\\n        x = self.model(x)\\n        return x\\n\\n    def train_one_epoch(self, train_loader, criterion, optimizer, device):\\n        # keep track of training loss\\n        epoch_loss = 0.0\\n        epoch_accuracy = 0.0\\n\\n        ###################\\n        # train the model #\\n        ###################\\n        self.model.train()\\n        for i, (data, target) in enumerate(train_loader):\\n            # move tensors to GPU if CUDA is available\\n            if device.type == \\\"cuda\\\":\\n                data, target = data.cuda(), target.cuda()\\n            elif device.type == \\\"xla\\\":\\n                data = data.to(device, dtype=torch.float32)\\n                target = target.to(device, dtype=torch.int64)\\n\\n            # clear the gradients of all optimized variables\\n            optimizer.zero_grad()\\n            # forward pass: compute predicted outputs by passing inputs to the model\\n            output = self.forward(data)\\n            # calculate the batch loss\\n            loss = criterion(output, target)\\n            # backward pass: compute gradient of the loss with respect to model parameters\\n            loss.backward()\\n            # Calculate Accuracy\\n            accuracy = (output.argmax(dim=1) == target).float().mean()\\n            # update training loss and accuracy\\n            epoch_loss += loss\\n            epoch_accuracy += accuracy\\n\\n            # perform a single optimization step (parameter update)\\n            if device.type == \\\"xla\\\":\\n                xm.optimizer_step(optimizer)\\n\\n                if i % 20 == 0:\\n                    xm.master_print(f\\\"\\\\tBATCH {i+1}/{len(train_loader)} - LOSS: {loss}\\\")\\n\\n            else:\\n                optimizer.step()\\n\\n        return epoch_loss / len(train_loader), epoch_accuracy / len(train_loader)\\n\\n    def validate_one_epoch(self, valid_loader, criterion, device):\\n        # keep track of validation loss\\n        valid_loss = 0.0\\n        valid_accuracy = 0.0\\n\\n        ######################\\n        # validate the model #\\n        ######################\\n        self.model.eval()\\n        for data, target in valid_loader:\\n            # move tensors to GPU if CUDA is available\\n            if device.type == \\\"cuda\\\":\\n                data, target = data.cuda(), target.cuda()\\n            elif device.type == \\\"xla\\\":\\n                data = data.to(device, dtype=torch.float32)\\n                target = target.to(device, dtype=torch.int64)\\n\\n            with torch.no_grad():\\n                # forward pass: compute predicted outputs by passing inputs to the model\\n                output = self.model(data)\\n                # calculate the batch loss\\n                loss = criterion(output, target)\\n                # Calculate Accuracy\\n                accuracy = (output.argmax(dim=1) == target).float().mean()\\n                # update average validation loss and accuracy\\n                valid_loss += loss\\n                valid_accuracy += accuracy\\n\\n        return valid_loss / len(valid_loader), valid_accuracy / len(valid_loader)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "class ViTBase16(nn.Module):\n",
    "    def __init__(self, n_classes, pretrained=False):\n",
    "\n",
    "        super(ViTBase16, self).__init__()\n",
    "\n",
    "        self.model = timm.create_model(\"vit_base_patch16_224\", pretrained=False)\n",
    "        if pretrained:\n",
    "            self.model.load_state_dict(torch.load(MODEL_PATH))\n",
    "\n",
    "        self.model.head = nn.Linear(self.model.head.in_features, n_classes)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.model(x)\n",
    "        return x\n",
    "\n",
    "    def train_one_epoch(self, train_loader, criterion, optimizer, device):\n",
    "        # keep track of training loss\n",
    "        epoch_loss = 0.0\n",
    "        epoch_accuracy = 0.0\n",
    "\n",
    "        ###################\n",
    "        # train the model #\n",
    "        ###################\n",
    "        self.model.train()\n",
    "        for i, (data, target) in enumerate(train_loader):\n",
    "            # move tensors to GPU if CUDA is available\n",
    "            if device.type == \"cuda\":\n",
    "                data, target = data.cuda(), target.cuda()\n",
    "            elif device.type == \"xla\":\n",
    "                data = data.to(device, dtype=torch.float32)\n",
    "                target = target.to(device, dtype=torch.int64)\n",
    "\n",
    "            # clear the gradients of all optimized variables\n",
    "            optimizer.zero_grad()\n",
    "            # forward pass: compute predicted outputs by passing inputs to the model\n",
    "            output = self.forward(data)\n",
    "            # calculate the batch loss\n",
    "            loss = criterion(output, target)\n",
    "            # backward pass: compute gradient of the loss with respect to model parameters\n",
    "            loss.backward()\n",
    "            # Calculate Accuracy\n",
    "            accuracy = (output.argmax(dim=1) == target).float().mean()\n",
    "            # update training loss and accuracy\n",
    "            epoch_loss += loss\n",
    "            epoch_accuracy += accuracy\n",
    "\n",
    "            # perform a single optimization step (parameter update)\n",
    "            if device.type == \"xla\":\n",
    "                xm.optimizer_step(optimizer)\n",
    "\n",
    "                if i % 20 == 0:\n",
    "                    xm.master_print(f\"\\tBATCH {i+1}/{len(train_loader)} - LOSS: {loss}\")\n",
    "\n",
    "            else:\n",
    "                optimizer.step()\n",
    "\n",
    "        return epoch_loss / len(train_loader), epoch_accuracy / len(train_loader)\n",
    "\n",
    "    def validate_one_epoch(self, valid_loader, criterion, device):\n",
    "        # keep track of validation loss\n",
    "        valid_loss = 0.0\n",
    "        valid_accuracy = 0.0\n",
    "\n",
    "        ######################\n",
    "        # validate the model #\n",
    "        ######################\n",
    "        self.model.eval()\n",
    "        for data, target in valid_loader:\n",
    "            # move tensors to GPU if CUDA is available\n",
    "            if device.type == \"cuda\":\n",
    "                data, target = data.cuda(), target.cuda()\n",
    "            elif device.type == \"xla\":\n",
    "                data = data.to(device, dtype=torch.float32)\n",
    "                target = target.to(device, dtype=torch.int64)\n",
    "\n",
    "            with torch.no_grad():\n",
    "                # forward pass: compute predicted outputs by passing inputs to the model\n",
    "                output = self.model(data)\n",
    "                # calculate the batch loss\n",
    "                loss = criterion(output, target)\n",
    "                # Calculate Accuracy\n",
    "                accuracy = (output.argmax(dim=1) == target).float().mean()\n",
    "                # update average validation loss and accuracy\n",
    "                valid_loss += loss\n",
    "                valid_accuracy += accuracy\n",
    "\n",
    "        return valid_loss / len(valid_loader), valid_accuracy / len(valid_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:15.199425Z",
     "iopub.status.busy": "2020-12-02T18:34:15.198635Z",
     "iopub.status.idle": "2020-12-02T18:34:15.236790Z",
     "shell.execute_reply": "2020-12-02T18:34:15.236128Z"
    },
    "papermill": {
     "duration": 0.131593,
     "end_time": "2020-12-02T18:34:15.236935",
     "exception": false,
     "start_time": "2020-12-02T18:34:15.105342",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 15;\n",
       "                var nbb_unformatted_code = \"def fit_tpu(\\n    model, epochs, device, criterion, optimizer, train_loader, valid_loader=None\\n):\\n\\n    valid_loss_min = np.Inf  # track change in validation loss\\n\\n    # keeping track of losses as it happen\\n    train_losses = []\\n    valid_losses = []\\n    train_accs = []\\n    valid_accs = []\\n\\n    for epoch in range(1, epochs + 1):\\n        gc.collect()\\n        para_train_loader = pl.ParallelLoader(train_loader, [device])\\n\\n        xm.master_print(f\\\"{'='*50}\\\")\\n        xm.master_print(f\\\"EPOCH {epoch} - TRAINING...\\\")\\n        train_loss, train_acc = model.train_one_epoch(\\n            para_train_loader.per_device_loader(device), criterion, optimizer, device\\n        )\\n        xm.master_print(\\n            f\\\"\\\\n\\\\t[TRAIN] EPOCH {epoch} - LOSS: {train_loss}, ACCURACY: {train_acc}\\\\n\\\"\\n        )\\n        train_losses.append(train_loss)\\n        train_accs.append(train_acc)\\n        gc.collect()\\n\\n        if valid_loader is not None:\\n            gc.collect()\\n            para_valid_loader = pl.ParallelLoader(valid_loader, [device])\\n            xm.master_print(f\\\"EPOCH {epoch} - VALIDATING...\\\")\\n            valid_loss, valid_acc = model.validate_one_epoch(\\n                para_valid_loader.per_device_loader(device), criterion, device\\n            )\\n            xm.master_print(f\\\"\\\\t[VALID] LOSS: {valid_loss}, ACCURACY: {valid_acc}\\\\n\\\")\\n            valid_losses.append(valid_loss)\\n            valid_accs.append(valid_acc)\\n            gc.collect()\\n\\n            # save model if validation loss has decreased\\n            if valid_loss <= valid_loss_min and epoch != 1:\\n                xm.master_print(\\n                    \\\"Validation loss decreased ({:.4f} --> {:.4f}).  Saving model ...\\\".format(\\n                        valid_loss_min, valid_loss\\n                    )\\n                )\\n            #                 xm.save(model.state_dict(), 'best_model.pth')\\n\\n            valid_loss_min = valid_loss\\n\\n    return {\\n        \\\"train_loss\\\": train_losses,\\n        \\\"valid_losses\\\": valid_losses,\\n        \\\"train_acc\\\": train_accs,\\n        \\\"valid_acc\\\": valid_accs,\\n    }\";\n",
       "                var nbb_formatted_code = \"def fit_tpu(\\n    model, epochs, device, criterion, optimizer, train_loader, valid_loader=None\\n):\\n\\n    valid_loss_min = np.Inf  # track change in validation loss\\n\\n    # keeping track of losses as it happen\\n    train_losses = []\\n    valid_losses = []\\n    train_accs = []\\n    valid_accs = []\\n\\n    for epoch in range(1, epochs + 1):\\n        gc.collect()\\n        para_train_loader = pl.ParallelLoader(train_loader, [device])\\n\\n        xm.master_print(f\\\"{'='*50}\\\")\\n        xm.master_print(f\\\"EPOCH {epoch} - TRAINING...\\\")\\n        train_loss, train_acc = model.train_one_epoch(\\n            para_train_loader.per_device_loader(device), criterion, optimizer, device\\n        )\\n        xm.master_print(\\n            f\\\"\\\\n\\\\t[TRAIN] EPOCH {epoch} - LOSS: {train_loss}, ACCURACY: {train_acc}\\\\n\\\"\\n        )\\n        train_losses.append(train_loss)\\n        train_accs.append(train_acc)\\n        gc.collect()\\n\\n        if valid_loader is not None:\\n            gc.collect()\\n            para_valid_loader = pl.ParallelLoader(valid_loader, [device])\\n            xm.master_print(f\\\"EPOCH {epoch} - VALIDATING...\\\")\\n            valid_loss, valid_acc = model.validate_one_epoch(\\n                para_valid_loader.per_device_loader(device), criterion, device\\n            )\\n            xm.master_print(f\\\"\\\\t[VALID] LOSS: {valid_loss}, ACCURACY: {valid_acc}\\\\n\\\")\\n            valid_losses.append(valid_loss)\\n            valid_accs.append(valid_acc)\\n            gc.collect()\\n\\n            # save model if validation loss has decreased\\n            if valid_loss <= valid_loss_min and epoch != 1:\\n                xm.master_print(\\n                    \\\"Validation loss decreased ({:.4f} --> {:.4f}).  Saving model ...\\\".format(\\n                        valid_loss_min, valid_loss\\n                    )\\n                )\\n            #                 xm.save(model.state_dict(), 'best_model.pth')\\n\\n            valid_loss_min = valid_loss\\n\\n    return {\\n        \\\"train_loss\\\": train_losses,\\n        \\\"valid_losses\\\": valid_losses,\\n        \\\"train_acc\\\": train_accs,\\n        \\\"valid_acc\\\": valid_accs,\\n    }\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def fit_tpu(\n",
    "    model, epochs, device, criterion, optimizer, train_loader, valid_loader=None\n",
    "):\n",
    "\n",
    "    valid_loss_min = np.Inf  # track change in validation loss\n",
    "\n",
    "    # keeping track of losses as it happen\n",
    "    train_losses = []\n",
    "    valid_losses = []\n",
    "    train_accs = []\n",
    "    valid_accs = []\n",
    "\n",
    "    for epoch in range(1, epochs + 1):\n",
    "        gc.collect()\n",
    "        para_train_loader = pl.ParallelLoader(train_loader, [device])\n",
    "\n",
    "        xm.master_print(f\"{'='*50}\")\n",
    "        xm.master_print(f\"EPOCH {epoch} - TRAINING...\")\n",
    "        train_loss, train_acc = model.train_one_epoch(\n",
    "            para_train_loader.per_device_loader(device), criterion, optimizer, device\n",
    "        )\n",
    "        xm.master_print(\n",
    "            f\"\\n\\t[TRAIN] EPOCH {epoch} - LOSS: {train_loss}, ACCURACY: {train_acc}\\n\"\n",
    "        )\n",
    "        train_losses.append(train_loss)\n",
    "        train_accs.append(train_acc)\n",
    "        gc.collect()\n",
    "\n",
    "        if valid_loader is not None:\n",
    "            gc.collect()\n",
    "            para_valid_loader = pl.ParallelLoader(valid_loader, [device])\n",
    "            xm.master_print(f\"EPOCH {epoch} - VALIDATING...\")\n",
    "            valid_loss, valid_acc = model.validate_one_epoch(\n",
    "                para_valid_loader.per_device_loader(device), criterion, device\n",
    "            )\n",
    "            xm.master_print(f\"\\t[VALID] LOSS: {valid_loss}, ACCURACY: {valid_acc}\\n\")\n",
    "            valid_losses.append(valid_loss)\n",
    "            valid_accs.append(valid_acc)\n",
    "            gc.collect()\n",
    "\n",
    "            # save model if validation loss has decreased\n",
    "            if valid_loss <= valid_loss_min and epoch != 1:\n",
    "                xm.master_print(\n",
    "                    \"Validation loss decreased ({:.4f} --> {:.4f}).  Saving model ...\".format(\n",
    "                        valid_loss_min, valid_loss\n",
    "                    )\n",
    "                )\n",
    "            #                 xm.save(model.state_dict(), 'best_model.pth')\n",
    "\n",
    "            valid_loss_min = valid_loss\n",
    "\n",
    "    return {\n",
    "        \"train_loss\": train_losses,\n",
    "        \"valid_losses\": valid_losses,\n",
    "        \"train_acc\": train_accs,\n",
    "        \"valid_acc\": valid_accs,\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:15.386773Z",
     "iopub.status.busy": "2020-12-02T18:34:15.385757Z",
     "iopub.status.idle": "2020-12-02T18:34:17.883884Z",
     "shell.execute_reply": "2020-12-02T18:34:17.882840Z"
    },
    "papermill": {
     "duration": 2.57545,
     "end_time": "2020-12-02T18:34:17.884083",
     "exception": false,
     "start_time": "2020-12-02T18:34:15.308633",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 16;\n",
       "                var nbb_unformatted_code = \"model = ViTBase16(n_classes=5, pretrained=True)\";\n",
       "                var nbb_formatted_code = \"model = ViTBase16(n_classes=5, pretrained=True)\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = ViTBase16(n_classes=5, pretrained=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:18.110856Z",
     "iopub.status.busy": "2020-12-02T18:34:18.107691Z",
     "iopub.status.idle": "2020-12-02T18:34:18.154359Z",
     "shell.execute_reply": "2020-12-02T18:34:18.154988Z"
    },
    "papermill": {
     "duration": 0.156612,
     "end_time": "2020-12-02T18:34:18.155150",
     "exception": false,
     "start_time": "2020-12-02T18:34:17.998538",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 17;\n",
       "                var nbb_unformatted_code = \"def _run():\\n    train_dataset = CassavaDataset(train_df, transforms=transforms_train)\\n    valid_dataset = CassavaDataset(valid_df, transforms=transforms_valid)\\n\\n    train_sampler = torch.utils.data.distributed.DistributedSampler(\\n        train_dataset,\\n        num_replicas=xm.xrt_world_size(),\\n        rank=xm.get_ordinal(),\\n        shuffle=True,\\n    )\\n\\n    valid_sampler = torch.utils.data.distributed.DistributedSampler(\\n        valid_dataset,\\n        num_replicas=xm.xrt_world_size(),\\n        rank=xm.get_ordinal(),\\n        shuffle=False,\\n    )\\n\\n    train_loader = torch.utils.data.DataLoader(\\n        dataset=train_dataset,\\n        batch_size=BATCH_SIZE,\\n        sampler=train_sampler,\\n        drop_last=True,\\n        num_workers=8,\\n    )\\n\\n    valid_loader = torch.utils.data.DataLoader(\\n        dataset=valid_dataset,\\n        batch_size=BATCH_SIZE,\\n        sampler=valid_sampler,\\n        drop_last=True,\\n        num_workers=8,\\n    )\\n\\n    criterion = nn.CrossEntropyLoss()\\n    #     device = torch.device(\\\"cuda\\\" if torch.cuda.is_available() else \\\"cpu\\\")\\n    device = xm.xla_device()\\n    model.to(device)\\n\\n    lr = LR * xm.xrt_world_size()\\n    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\\n\\n    xm.master_print(f\\\"INITIALIZING TRAINING ON {xm.xrt_world_size()} TPU CORES\\\")\\n    start_time = datetime.now()\\n    xm.master_print(f\\\"Start Time: {start_time}\\\")\\n\\n    logs = fit_tpu(\\n        model=model,\\n        epochs=N_EPOCHS,\\n        device=device,\\n        criterion=criterion,\\n        optimizer=optimizer,\\n        train_loader=train_loader,\\n        valid_loader=valid_loader,\\n    )\\n\\n    xm.master_print(f\\\"Execution time: {datetime.now() - start_time}\\\")\\n\\n    xm.master_print(\\\"Saving Model\\\")\\n    xm.save(\\n        model.state_dict(), f'model_5e_{datetime.now().strftime(\\\"%Y%m%d-%H%M\\\")}.pth'\\n    )\";\n",
       "                var nbb_formatted_code = \"def _run():\\n    train_dataset = CassavaDataset(train_df, transforms=transforms_train)\\n    valid_dataset = CassavaDataset(valid_df, transforms=transforms_valid)\\n\\n    train_sampler = torch.utils.data.distributed.DistributedSampler(\\n        train_dataset,\\n        num_replicas=xm.xrt_world_size(),\\n        rank=xm.get_ordinal(),\\n        shuffle=True,\\n    )\\n\\n    valid_sampler = torch.utils.data.distributed.DistributedSampler(\\n        valid_dataset,\\n        num_replicas=xm.xrt_world_size(),\\n        rank=xm.get_ordinal(),\\n        shuffle=False,\\n    )\\n\\n    train_loader = torch.utils.data.DataLoader(\\n        dataset=train_dataset,\\n        batch_size=BATCH_SIZE,\\n        sampler=train_sampler,\\n        drop_last=True,\\n        num_workers=8,\\n    )\\n\\n    valid_loader = torch.utils.data.DataLoader(\\n        dataset=valid_dataset,\\n        batch_size=BATCH_SIZE,\\n        sampler=valid_sampler,\\n        drop_last=True,\\n        num_workers=8,\\n    )\\n\\n    criterion = nn.CrossEntropyLoss()\\n    #     device = torch.device(\\\"cuda\\\" if torch.cuda.is_available() else \\\"cpu\\\")\\n    device = xm.xla_device()\\n    model.to(device)\\n\\n    lr = LR * xm.xrt_world_size()\\n    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\\n\\n    xm.master_print(f\\\"INITIALIZING TRAINING ON {xm.xrt_world_size()} TPU CORES\\\")\\n    start_time = datetime.now()\\n    xm.master_print(f\\\"Start Time: {start_time}\\\")\\n\\n    logs = fit_tpu(\\n        model=model,\\n        epochs=N_EPOCHS,\\n        device=device,\\n        criterion=criterion,\\n        optimizer=optimizer,\\n        train_loader=train_loader,\\n        valid_loader=valid_loader,\\n    )\\n\\n    xm.master_print(f\\\"Execution time: {datetime.now() - start_time}\\\")\\n\\n    xm.master_print(\\\"Saving Model\\\")\\n    xm.save(\\n        model.state_dict(), f'model_5e_{datetime.now().strftime(\\\"%Y%m%d-%H%M\\\")}.pth'\\n    )\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def _run():\n",
    "    train_dataset = CassavaDataset(train_df, transforms=transforms_train)\n",
    "    valid_dataset = CassavaDataset(valid_df, transforms=transforms_valid)\n",
    "\n",
    "    train_sampler = torch.utils.data.distributed.DistributedSampler(\n",
    "        train_dataset,\n",
    "        num_replicas=xm.xrt_world_size(),\n",
    "        rank=xm.get_ordinal(),\n",
    "        shuffle=True,\n",
    "    )\n",
    "\n",
    "    valid_sampler = torch.utils.data.distributed.DistributedSampler(\n",
    "        valid_dataset,\n",
    "        num_replicas=xm.xrt_world_size(),\n",
    "        rank=xm.get_ordinal(),\n",
    "        shuffle=False,\n",
    "    )\n",
    "\n",
    "    train_loader = torch.utils.data.DataLoader(\n",
    "        dataset=train_dataset,\n",
    "        batch_size=BATCH_SIZE,\n",
    "        sampler=train_sampler,\n",
    "        drop_last=True,\n",
    "        num_workers=8,\n",
    "    )\n",
    "\n",
    "    valid_loader = torch.utils.data.DataLoader(\n",
    "        dataset=valid_dataset,\n",
    "        batch_size=BATCH_SIZE,\n",
    "        sampler=valid_sampler,\n",
    "        drop_last=True,\n",
    "        num_workers=8,\n",
    "    )\n",
    "\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    #     device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "    device = xm.xla_device()\n",
    "    model.to(device)\n",
    "\n",
    "    lr = LR * xm.xrt_world_size()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=lr)\n",
    "\n",
    "    xm.master_print(f\"INITIALIZING TRAINING ON {xm.xrt_world_size()} TPU CORES\")\n",
    "    start_time = datetime.now()\n",
    "    xm.master_print(f\"Start Time: {start_time}\")\n",
    "\n",
    "    logs = fit_tpu(\n",
    "        model=model,\n",
    "        epochs=N_EPOCHS,\n",
    "        device=device,\n",
    "        criterion=criterion,\n",
    "        optimizer=optimizer,\n",
    "        train_loader=train_loader,\n",
    "        valid_loader=valid_loader,\n",
    "    )\n",
    "\n",
    "    xm.master_print(f\"Execution time: {datetime.now() - start_time}\")\n",
    "\n",
    "    xm.master_print(\"Saving Model\")\n",
    "    xm.save(\n",
    "        model.state_dict(), f'model_5e_{datetime.now().strftime(\"%Y%m%d-%H%M\")}.pth'\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-12-02T18:34:18.313245Z",
     "iopub.status.busy": "2020-12-02T18:34:18.312128Z",
     "iopub.status.idle": "2020-12-02T19:08:15.586913Z",
     "shell.execute_reply": "2020-12-02T19:08:15.587804Z"
    },
    "papermill": {
     "duration": 2037.357951,
     "end_time": "2020-12-02T19:08:15.588226",
     "exception": false,
     "start_time": "2020-12-02T18:34:18.230275",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INITIALIZING TRAINING ON 8 TPU CORES\n",
      "Start Time: 2020-12-02 18:34:26.339168\n",
      "==================================================\n",
      "EPOCH 1 - TRAINING...\n",
      "\tBATCH 1/150 - LOSS: 1.609375\n",
      "\tBATCH 21/150 - LOSS: 0.7734375\n",
      "\tBATCH 41/150 - LOSS: 0.33984375\n",
      "\tBATCH 61/150 - LOSS: 0.333984375\n",
      "\tBATCH 81/150 - LOSS: 0.314453125\n",
      "\tBATCH 101/150 - LOSS: 0.765625\n",
      "\tBATCH 121/150 - LOSS: 0.439453125\n",
      "\tBATCH 141/150 - LOSS: 0.451171875\n",
      "\n",
      "\t[TRAIN] EPOCH 1 - LOSS: 0.59765625, ACCURACY: 0.83203125\n",
      "\n",
      "EPOCH 1 - VALIDATING...\n",
      "\t[VALID] LOSS: 0.419921875, ACCURACY: 0.84375\n",
      "\n",
      "==================================================\n",
      "EPOCH 2 - TRAINING...\n",
      "\tBATCH 1/150 - LOSS: 0.458984375\n",
      "\tBATCH 21/150 - LOSS: 0.2734375\n",
      "\tBATCH 41/150 - LOSS: 0.1396484375\n",
      "\tBATCH 61/150 - LOSS: 0.177734375\n",
      "\tBATCH 81/150 - LOSS: 0.14453125\n",
      "\tBATCH 101/150 - LOSS: 0.5703125\n",
      "\tBATCH 121/150 - LOSS: 0.32421875\n",
      "\tBATCH 141/150 - LOSS: 0.60546875\n",
      "\n",
      "\t[TRAIN] EPOCH 2 - LOSS: 0.41796875, ACCURACY: 0.9140625\n",
      "\n",
      "EPOCH 2 - VALIDATING...\n",
      "\t[VALID] LOSS: 0.4140625, ACCURACY: 0.83984375\n",
      "\n",
      "Validation loss decreased (0.4199 --> 0.4141).  Saving model ...\n",
      "==================================================\n",
      "EPOCH 3 - TRAINING...\n",
      "\tBATCH 1/150 - LOSS: 0.20703125\n",
      "\tBATCH 21/150 - LOSS: 0.5625\n",
      "\tBATCH 41/150 - LOSS: 0.15625\n",
      "\tBATCH 61/150 - LOSS: 0.2373046875\n",
      "\tBATCH 81/150 - LOSS: 0.208984375\n",
      "\tBATCH 101/150 - LOSS: 0.435546875\n",
      "\tBATCH 121/150 - LOSS: 0.28515625\n",
      "\tBATCH 141/150 - LOSS: 0.337890625\n",
      "\n",
      "\t[TRAIN] EPOCH 3 - LOSS: 0.39453125, ACCURACY: 0.9140625\n",
      "\n",
      "EPOCH 3 - VALIDATING...\n",
      "\t[VALID] LOSS: 0.404296875, ACCURACY: 0.83984375\n",
      "\n",
      "Validation loss decreased (0.4141 --> 0.4043).  Saving model ...\n",
      "==================================================\n",
      "EPOCH 4 - TRAINING...\n",
      "\tBATCH 1/150 - LOSS: 0.17578125\n",
      "\tBATCH 21/150 - LOSS: 0.15625\n",
      "\tBATCH 41/150 - LOSS: 0.154296875\n",
      "\tBATCH 61/150 - LOSS: 0.11865234375\n",
      "\tBATCH 81/150 - LOSS: 0.1298828125\n",
      "\tBATCH 101/150 - LOSS: 0.36328125\n",
      "\tBATCH 121/150 - LOSS: 0.208984375\n",
      "\tBATCH 141/150 - LOSS: 0.333984375\n",
      "\n",
      "\t[TRAIN] EPOCH 4 - LOSS: 0.369140625, ACCURACY: 0.921875\n",
      "\n",
      "EPOCH 4 - VALIDATING...\n",
      "\t[VALID] LOSS: 0.3984375, ACCURACY: 0.859375\n",
      "\n",
      "Validation loss decreased (0.4043 --> 0.3984).  Saving model ...\n",
      "==================================================\n",
      "EPOCH 5 - TRAINING...\n",
      "\tBATCH 1/150 - LOSS: 0.5390625\n",
      "\tBATCH 21/150 - LOSS: 0.1865234375\n",
      "\tBATCH 41/150 - LOSS: 0.263671875\n",
      "\tBATCH 61/150 - LOSS: 0.1328125\n",
      "\tBATCH 81/150 - LOSS: 0.10498046875\n",
      "\tBATCH 101/150 - LOSS: 0.390625\n",
      "\tBATCH 121/150 - LOSS: 0.2890625\n",
      "\tBATCH 141/150 - LOSS: 0.318359375\n",
      "\n",
      "\t[TRAIN] EPOCH 5 - LOSS: 0.357421875, ACCURACY: 0.921875\n",
      "\n",
      "EPOCH 5 - VALIDATING...\n",
      "\t[VALID] LOSS: 0.416015625, ACCURACY: 0.84375\n",
      "\n",
      "==================================================\n",
      "EPOCH 6 - TRAINING...\n",
      "\tBATCH 1/150 - LOSS: 0.51953125\n",
      "\tBATCH 21/150 - LOSS: 0.27734375\n",
      "\tBATCH 41/150 - LOSS: 0.2138671875\n",
      "\tBATCH 61/150 - LOSS: 0.1630859375\n",
      "\tBATCH 81/150 - LOSS: 0.19140625\n",
      "\tBATCH 101/150 - LOSS: 0.60546875\n",
      "\tBATCH 121/150 - LOSS: 0.271484375\n",
      "\tBATCH 141/150 - LOSS: 0.1943359375\n",
      "\n",
      "\t[TRAIN] EPOCH 6 - LOSS: 0.34375, ACCURACY: 0.92578125\n",
      "\n",
      "EPOCH 6 - VALIDATING...\n",
      "\t[VALID] LOSS: 0.41796875, ACCURACY: 0.8515625\n",
      "\n",
      "==================================================\n",
      "EPOCH 7 - TRAINING...\n",
      "\tBATCH 1/150 - LOSS: 0.3125\n",
      "\tBATCH 21/150 - LOSS: 0.30078125\n",
      "\tBATCH 41/150 - LOSS: 0.2041015625\n",
      "\tBATCH 61/150 - LOSS: 0.3515625\n",
      "\tBATCH 81/150 - LOSS: 0.0849609375\n",
      "\tBATCH 101/150 - LOSS: 0.408203125\n",
      "\tBATCH 121/150 - LOSS: 0.146484375\n",
      "\tBATCH 141/150 - LOSS: 0.2001953125\n",
      "\n",
      "\t[TRAIN] EPOCH 7 - LOSS: 0.30078125, ACCURACY: 0.94140625\n",
      "\n",
      "EPOCH 7 - VALIDATING...\n",
      "\t[VALID] LOSS: 0.40625, ACCURACY: 0.85546875\n",
      "\n",
      "Validation loss decreased (0.4180 --> 0.4062).  Saving model ...\n",
      "==================================================\n",
      "EPOCH 8 - TRAINING...\n",
      "\tBATCH 1/150 - LOSS: 0.16015625\n",
      "\tBATCH 21/150 - LOSS: 0.181640625\n",
      "\tBATCH 41/150 - LOSS: 0.083984375\n",
      "\tBATCH 61/150 - LOSS: 0.142578125\n",
      "\tBATCH 81/150 - LOSS: 0.11962890625\n",
      "\tBATCH 101/150 - LOSS: 0.2197265625\n",
      "\tBATCH 121/150 - LOSS: 0.1875\n",
      "\tBATCH 141/150 - LOSS: 0.33984375\n",
      "\n",
      "\t[TRAIN] EPOCH 8 - LOSS: 0.28515625, ACCURACY: 0.94140625\n",
      "\n",
      "EPOCH 8 - VALIDATING...\n",
      "\t[VALID] LOSS: 0.40625, ACCURACY: 0.87109375\n",
      "\n",
      "Validation loss decreased (0.4062 --> 0.4062).  Saving model ...\n",
      "==================================================\n",
      "EPOCH 9 - TRAINING...\n",
      "\tBATCH 1/150 - LOSS: 0.435546875\n",
      "\tBATCH 21/150 - LOSS: 0.39453125\n",
      "\tBATCH 41/150 - LOSS: 0.212890625\n",
      "\tBATCH 61/150 - LOSS: 0.0830078125\n",
      "\tBATCH 81/150 - LOSS: 0.07958984375\n",
      "\tBATCH 101/150 - LOSS: 0.34765625\n",
      "\tBATCH 121/150 - LOSS: 0.279296875\n",
      "\tBATCH 141/150 - LOSS: 0.2470703125\n",
      "\n",
      "\t[TRAIN] EPOCH 9 - LOSS: 0.2890625, ACCURACY: 0.9453125\n",
      "\n",
      "EPOCH 9 - VALIDATING...\n",
      "\t[VALID] LOSS: 0.41796875, ACCURACY: 0.87109375\n",
      "\n",
      "==================================================\n",
      "EPOCH 10 - TRAINING...\n",
      "\tBATCH 1/150 - LOSS: 0.38671875\n",
      "\tBATCH 21/150 - LOSS: 0.2119140625\n",
      "\tBATCH 41/150 - LOSS: 0.06884765625\n",
      "\tBATCH 61/150 - LOSS: 0.1806640625\n",
      "\tBATCH 81/150 - LOSS: 0.0546875\n",
      "\tBATCH 101/150 - LOSS: 0.19140625\n",
      "\tBATCH 121/150 - LOSS: 0.197265625\n",
      "\tBATCH 141/150 - LOSS: 0.328125\n",
      "\n",
      "\t[TRAIN] EPOCH 10 - LOSS: 0.28125, ACCURACY: 0.9453125\n",
      "\n",
      "EPOCH 10 - VALIDATING...\n",
      "\t[VALID] LOSS: 0.453125, ACCURACY: 0.83984375\n",
      "\n",
      "Execution time: 0:33:39.824880\n",
      "Saving Model\n"
     ]
    },
    {
     "data": {
      "application/javascript": [
       "\n",
       "            setTimeout(function() {\n",
       "                var nbb_cell_id = 18;\n",
       "                var nbb_unformatted_code = \"# Start training processes\\ndef _mp_fn(rank, flags):\\n    torch.set_default_tensor_type(\\\"torch.FloatTensor\\\")\\n    a = _run()\\n\\n\\n# _run()\\nFLAGS = {}\\nxmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method=\\\"fork\\\")\";\n",
       "                var nbb_formatted_code = \"# Start training processes\\ndef _mp_fn(rank, flags):\\n    torch.set_default_tensor_type(\\\"torch.FloatTensor\\\")\\n    a = _run()\\n\\n\\n# _run()\\nFLAGS = {}\\nxmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method=\\\"fork\\\")\";\n",
       "                var nbb_cells = Jupyter.notebook.get_cells();\n",
       "                for (var i = 0; i < nbb_cells.length; ++i) {\n",
       "                    if (nbb_cells[i].input_prompt_number == nbb_cell_id) {\n",
       "                        if (nbb_cells[i].get_text() == nbb_unformatted_code) {\n",
       "                             nbb_cells[i].set_text(nbb_formatted_code);\n",
       "                        }\n",
       "                        break;\n",
       "                    }\n",
       "                }\n",
       "            }, 500);\n",
       "            "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Start training processes\n",
    "def _mp_fn(rank, flags):\n",
    "    torch.set_default_tensor_type(\"torch.FloatTensor\")\n",
    "    a = _run()\n",
    "\n",
    "\n",
    "# _run()\n",
    "FLAGS = {}\n",
    "xmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method=\"fork\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.135774,
     "end_time": "2020-12-02T19:08:15.862334",
     "exception": false,
     "start_time": "2020-12-02T19:08:15.726560",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Thanks a lot for reading all the way\n",
    "\n",
    "# <font size=4 color='blue'>If you find this notebook useful, leave an upvote, that motivates me to write more such notebooks.</font>"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "papermill": {
   "duration": 2169.65242,
   "end_time": "2020-12-02T19:08:16.137317",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2020-12-02T18:32:06.484897",
   "version": "2.1.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}