code_llama_fim_fine_tuning.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/BohdanPetryshyn/code-llama-fim-fine-tuning/blob/main/code_llama_fim_fine_tuning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Code Llama FIM fine-tuning\n",
        "\n",
        "If you found a problem with this notebook, please report it in the original GitHub [repo](https://github.com/BohdanPetryshyn/code-llama-fim-fine-tuning?tab=readme-ov-file) as an issue."
      ],
      "metadata": {
        "id": "H5g_vy4GJxq2"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Install dependenices"
      ],
      "metadata": {
        "id": "RMn3ZqdbJ1nu"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "v8MG0g-eEW9X",
        "outputId": "ef268413-c5de-4056-bbd2-9784d7795629"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'repo'...\n",
            "remote: Enumerating objects: 28, done.\u001b[K\n",
            "remote: Counting objects: 100% (28/28), done.\u001b[K\n",
            "remote: Compressing objects: 100% (18/18), done.\u001b[K\n",
            "remote: Total 28 (delta 11), reused 27 (delta 10), pack-reused 0\u001b[K\n",
            "Receiving objects: 100% (28/28), 16.84 KiB | 16.84 MiB/s, done.\n",
            "Resolving deltas: 100% (11/11), done.\n"
          ]
        }
      ],
      "source": [
        "!git clone https://github.com/BohdanPetryshyn/code-llama-fim-fine-tuning.git repo"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UqWoeV0zJF8E",
        "outputId": "88d1d0af-2213-46b9-cb8c-c31ce492770f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting ninja\n",
            "  Downloading ninja-1.11.1.1-py2.py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl (307 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/307.2 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━\u001b[0m \u001b[32m297.0/307.2 kB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.2/307.2 kB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: ninja\n",
            "Successfully installed ninja-1.11.1.1\n",
            "1.11.1.git.kitware.jobserver-1\n"
          ]
        }
      ],
      "source": [
        "!pip install ninja\n",
        "!ninja --version"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xvjHmB_pIwWA",
        "outputId": "edb41b6c-5de8-42f9-be9e-c34f6e40dd38"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/repo\n",
            "Collecting unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git@27fa021a7bb959a53667dd4e7cdb9598c207aa0d (from -r requirements.txt (line 25))\n",
            "  Cloning https://github.com/unslothai/unsloth.git (to revision 27fa021a7bb959a53667dd4e7cdb9598c207aa0d) to /tmp/pip-install-wztqjbse/unsloth_9c722ab323924c2bb9712de818327d4b\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/unslothai/unsloth.git /tmp/pip-install-wztqjbse/unsloth_9c722ab323924c2bb9712de818327d4b\n",
            "  Running command git rev-parse -q --verify 'sha^27fa021a7bb959a53667dd4e7cdb9598c207aa0d'\n",
            "  Running command git fetch -q https://github.com/unslothai/unsloth.git 27fa021a7bb959a53667dd4e7cdb9598c207aa0d\n",
            "  Running command git checkout -q 27fa021a7bb959a53667dd4e7cdb9598c207aa0d\n",
            "  Resolved https://github.com/unslothai/unsloth.git to commit 27fa021a7bb959a53667dd4e7cdb9598c207aa0d\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting PyGithub==2.3.0 (from -r requirements.txt (line 1))\n",
            "  Downloading PyGithub-2.3.0-py3-none-any.whl (354 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m354.4/354.4 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting accelerate==0.30.1 (from -r requirements.txt (line 2))\n",
            "  Downloading accelerate-0.30.1-py3-none-any.whl (302 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.6/302.6 kB\u001b[0m \u001b[31m22.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting bitsandbytes==0.43.1 (from -r requirements.txt (line 3))\n",
            "  Downloading bitsandbytes-0.43.1-py3-none-manylinux_2_24_x86_64.whl (119.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.8/119.8 MB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datasets==2.19.1 (from -r requirements.txt (line 4))\n",
            "  Downloading datasets-2.19.1-py3-none-any.whl (542 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m542.0/542.0 kB\u001b[0m \u001b[31m51.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting datatrove==0.2.0 (from -r requirements.txt (line 5))\n",
            "  Downloading datatrove-0.2.0-py3-none-any.whl (16.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m16.6/16.6 MB\u001b[0m \u001b[31m70.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting deepspeed==0.14.2 (from -r requirements.txt (line 6))\n",
            "  Downloading deepspeed-0.14.2.tar.gz (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m39.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting einops==0.8.0 (from -r requirements.txt (line 7))\n",
            "  Downloading einops-0.8.0-py3-none-any.whl (43 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.2/43.2 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting evaluate==0.4.2 (from -r requirements.txt (line 8))\n",
            "  Downloading evaluate-0.4.2-py3-none-any.whl (84 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m10.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting flash-attn==2.5.9.post1 (from -r requirements.txt (line 9))\n",
            "  Downloading flash_attn-2.5.9.post1.tar.gz (2.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.6/2.6 MB\u001b[0m \u001b[31m95.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting huggingface-hub==0.23.1 (from -r requirements.txt (line 10))\n",
            "  Downloading huggingface_hub-0.23.1-py3-none-any.whl (401 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m401.3/401.3 kB\u001b[0m \u001b[31m43.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: matplotlib==3.7.1 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 11)) (3.7.1)\n",
            "Requirement already satisfied: nltk==3.8.1 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 12)) (3.8.1)\n",
            "Requirement already satisfied: numpy==1.25.2 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 13)) (1.25.2)\n",
            "Requirement already satisfied: pandas==2.0.3 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 14)) (2.0.3)\n",
            "Collecting peft==0.11.1 (from -r requirements.txt (line 15))\n",
            "  Downloading peft-0.11.1-py3-none-any.whl (251 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m251.6/251.6 kB\u001b[0m \u001b[31m31.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: safetensors==0.4.3 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 16)) (0.4.3)\n",
            "Requirement already satisfied: scipy==1.11.4 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 17)) (1.11.4)\n",
            "Requirement already satisfied: sentencepiece==0.1.99 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 18)) (0.1.99)\n",
            "Requirement already satisfied: tensorboard==2.15.2 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 19)) (2.15.2)\n",
            "Collecting tiktoken==0.7.0 (from -r requirements.txt (line 20))\n",
            "  Downloading tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m75.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: torch==2.3.0 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 21)) (2.3.0+cu121)\n",
            "Requirement already satisfied: tqdm==4.66.4 in /usr/local/lib/python3.10/dist-packages (from -r requirements.txt (line 22)) (4.66.4)\n",
            "Collecting transformers==4.41.1 (from -r requirements.txt (line 23))\n",
            "  Downloading transformers-4.41.1-py3-none-any.whl (9.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m9.1/9.1 MB\u001b[0m \u001b[31m112.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting trl==0.8.6 (from -r requirements.txt (line 24))\n",
            "  Downloading trl-0.8.6-py3-none-any.whl (245 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m245.2/245.2 kB\u001b[0m \u001b[31m33.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting wandb==0.17.0 (from -r requirements.txt (line 26))\n",
            "  Downloading wandb-0.17.0-py3-none-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.7/6.7 MB\u001b[0m \u001b[31m70.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting xformers==0.0.26.post1 (from -r requirements.txt (line 27))\n",
            "  Downloading xformers-0.0.26.post1-cp310-cp310-manylinux2014_x86_64.whl (222.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m222.7/222.7 MB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting pynacl>=1.4.0 (from PyGithub==2.3.0->-r requirements.txt (line 1))\n",
            "  Downloading PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (856 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m856.7/856.7 kB\u001b[0m \u001b[31m65.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests>=2.14.0 in /usr/local/lib/python3.10/dist-packages (from PyGithub==2.3.0->-r requirements.txt (line 1)) (2.31.0)\n",
            "Collecting pyjwt[crypto]>=2.4.0 (from PyGithub==2.3.0->-r requirements.txt (line 1))\n",
            "  Downloading PyJWT-2.8.0-py3-none-any.whl (22 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from PyGithub==2.3.0->-r requirements.txt (line 1)) (4.12.2)\n",
            "Requirement already satisfied: urllib3>=1.26.0 in /usr/local/lib/python3.10/dist-packages (from PyGithub==2.3.0->-r requirements.txt (line 1)) (2.0.7)\n",
            "Collecting Deprecated (from PyGithub==2.3.0->-r requirements.txt (line 1))\n",
            "  Downloading Deprecated-1.2.14-py2.py3-none-any.whl (9.6 kB)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate==0.30.1->-r requirements.txt (line 2)) (24.1)\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate==0.30.1->-r requirements.txt (line 2)) (5.9.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate==0.30.1->-r requirements.txt (line 2)) (6.0.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets==2.19.1->-r requirements.txt (line 4)) (3.14.0)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets==2.19.1->-r requirements.txt (line 4)) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets==2.19.1->-r requirements.txt (line 4)) (0.6)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets==2.19.1->-r requirements.txt (line 4))\n",
            "  Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m17.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting xxhash (from datasets==2.19.1->-r requirements.txt (line 4))\n",
            "  Downloading xxhash-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m27.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting multiprocess (from datasets==2.19.1->-r requirements.txt (line 4))\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m18.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]<=2024.3.1,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets==2.19.1->-r requirements.txt (line 4)) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets==2.19.1->-r requirements.txt (line 4)) (3.9.5)\n",
            "Collecting fsspec>=2023.12.2 (from datatrove==0.2.0->-r requirements.txt (line 5))\n",
            "  Downloading fsspec-2024.6.0-py3-none-any.whl (176 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m176.9/176.9 kB\u001b[0m \u001b[31m22.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: humanize in /usr/local/lib/python3.10/dist-packages (from datatrove==0.2.0->-r requirements.txt (line 5)) (4.7.0)\n",
            "Collecting loguru>=0.7.0 (from datatrove==0.2.0->-r requirements.txt (line 5))\n",
            "  Downloading loguru-0.7.2-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.5/62.5 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting hjson (from deepspeed==0.14.2->-r requirements.txt (line 6))\n",
            "  Downloading hjson-3.1.0-py3-none-any.whl (54 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.0/54.0 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: ninja in /usr/local/lib/python3.10/dist-packages (from deepspeed==0.14.2->-r requirements.txt (line 6)) (1.11.1.1)\n",
            "Requirement already satisfied: py-cpuinfo in /usr/local/lib/python3.10/dist-packages (from deepspeed==0.14.2->-r requirements.txt (line 6)) (9.0.0)\n",
            "Requirement already satisfied: pydantic in /usr/local/lib/python3.10/dist-packages (from deepspeed==0.14.2->-r requirements.txt (line 6)) (2.7.3)\n",
            "Collecting pynvml (from deepspeed==0.14.2->-r requirements.txt (line 6))\n",
            "  Downloading pynvml-11.5.0-py3-none-any.whl (53 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib==3.7.1->-r requirements.txt (line 11)) (1.2.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib==3.7.1->-r requirements.txt (line 11)) (0.12.1)\n",
            "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib==3.7.1->-r requirements.txt (line 11)) (4.53.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib==3.7.1->-r requirements.txt (line 11)) (1.4.5)\n",
            "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib==3.7.1->-r requirements.txt (line 11)) (9.4.0)\n",
            "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib==3.7.1->-r requirements.txt (line 11)) (3.1.2)\n",
            "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib==3.7.1->-r requirements.txt (line 11)) (2.8.2)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk==3.8.1->-r requirements.txt (line 12)) (8.1.7)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk==3.8.1->-r requirements.txt (line 12)) (1.4.2)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk==3.8.1->-r requirements.txt (line 12)) (2024.5.15)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas==2.0.3->-r requirements.txt (line 14)) (2023.4)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas==2.0.3->-r requirements.txt (line 14)) (2024.1)\n",
            "Requirement already satisfied: absl-py>=0.4 in /usr/local/lib/python3.10/dist-packages (from tensorboard==2.15.2->-r requirements.txt (line 19)) (1.4.0)\n",
            "Requirement already satisfied: grpcio>=1.48.2 in /usr/local/lib/python3.10/dist-packages (from tensorboard==2.15.2->-r requirements.txt (line 19)) (1.64.1)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.10/dist-packages (from tensorboard==2.15.2->-r requirements.txt (line 19)) (2.27.0)\n",
            "Requirement already satisfied: google-auth-oauthlib<2,>=0.5 in /usr/local/lib/python3.10/dist-packages (from tensorboard==2.15.2->-r requirements.txt (line 19)) (1.2.0)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.10/dist-packages (from tensorboard==2.15.2->-r requirements.txt (line 19)) (3.6)\n",
            "Requirement already satisfied: protobuf!=4.24.0,>=3.19.6 in /usr/local/lib/python3.10/dist-packages (from tensorboard==2.15.2->-r requirements.txt (line 19)) (3.20.3)\n",
            "Requirement already satisfied: setuptools>=41.0.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard==2.15.2->-r requirements.txt (line 19)) (67.7.2)\n",
            "Requirement already satisfied: six>1.9 in /usr/local/lib/python3.10/dist-packages (from tensorboard==2.15.2->-r requirements.txt (line 19)) (1.16.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.8.0,>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from tensorboard==2.15.2->-r requirements.txt (line 19)) (0.7.2)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from tensorboard==2.15.2->-r requirements.txt (line 19)) (3.0.3)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch==2.3.0->-r requirements.txt (line 21)) (1.12.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch==2.3.0->-r requirements.txt (line 21)) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch==2.3.0->-r requirements.txt (line 21)) (3.1.4)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch==2.3.0->-r requirements.txt (line 21))\n",
            "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch==2.3.0->-r requirements.txt (line 21))\n",
            "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch==2.3.0->-r requirements.txt (line 21))\n",
            "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch==2.3.0->-r requirements.txt (line 21))\n",
            "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch==2.3.0->-r requirements.txt (line 21))\n",
            "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch==2.3.0->-r requirements.txt (line 21))\n",
            "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch==2.3.0->-r requirements.txt (line 21))\n",
            "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch==2.3.0->-r requirements.txt (line 21))\n",
            "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch==2.3.0->-r requirements.txt (line 21))\n",
            "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "Collecting nvidia-nccl-cu12==2.20.5 (from torch==2.3.0->-r requirements.txt (line 21))\n",
            "  Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n",
            "Collecting nvidia-nvtx-cu12==12.1.105 (from torch==2.3.0->-r requirements.txt (line 21))\n",
            "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "Requirement already satisfied: triton==2.3.0 in /usr/local/lib/python3.10/dist-packages (from torch==2.3.0->-r requirements.txt (line 21)) (2.3.0)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers==4.41.1->-r requirements.txt (line 23)) (0.19.1)\n",
            "Collecting tyro>=0.5.11 (from trl==0.8.6->-r requirements.txt (line 24))\n",
            "  Downloading tyro-0.8.4-py3-none-any.whl (102 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m102.4/102.4 kB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting docker-pycreds>=0.4.0 (from wandb==0.17.0->-r requirements.txt (line 26))\n",
            "  Downloading docker_pycreds-0.4.0-py2.py3-none-any.whl (9.0 kB)\n",
            "Collecting gitpython!=3.1.29,>=1.0.0 (from wandb==0.17.0->-r requirements.txt (line 26))\n",
            "  Downloading GitPython-3.1.43-py3-none-any.whl (207 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.3/207.3 kB\u001b[0m \u001b[31m28.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: platformdirs in /usr/local/lib/python3.10/dist-packages (from wandb==0.17.0->-r requirements.txt (line 26)) (4.2.2)\n",
            "Collecting sentry-sdk>=1.0.0 (from wandb==0.17.0->-r requirements.txt (line 26))\n",
            "  Downloading sentry_sdk-2.5.1-py2.py3-none-any.whl (289 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m289.6/289.6 kB\u001b[0m \u001b[31m35.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting setproctitle (from wandb==0.17.0->-r requirements.txt (line 26))\n",
            "  Downloading setproctitle-1.3.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (30 kB)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch==2.3.0->-r requirements.txt (line 21))\n",
            "  Downloading nvidia_nvjitlink_cu12-12.5.40-py3-none-manylinux2014_x86_64.whl (21.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.3/21.3 MB\u001b[0m \u001b[31m75.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: wheel>=0.42.0 in /usr/local/lib/python3.10/dist-packages (from unsloth[colab-new]@ git+https://github.com/unslothai/unsloth.git@27fa021a7bb959a53667dd4e7cdb9598c207aa0d->-r requirements.txt (line 25)) (0.43.0)\n",
            "INFO: pip is looking at multiple versions of fsspec[http] to determine which version is compatible with other requirements. This could take a while.\n",
            "Collecting fsspec[http]<=2024.3.1,>=2023.1.0 (from datasets==2.19.1->-r requirements.txt (line 4))\n",
            "  Downloading fsspec-2024.3.1-py3-none-any.whl (171 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m172.0/172.0 kB\u001b[0m \u001b[31m22.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.19.1->-r requirements.txt (line 4)) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.19.1->-r requirements.txt (line 4)) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.19.1->-r requirements.txt (line 4)) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.19.1->-r requirements.txt (line 4)) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.19.1->-r requirements.txt (line 4)) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets==2.19.1->-r requirements.txt (line 4)) (4.0.3)\n",
            "Collecting gitdb<5,>=4.0.1 (from gitpython!=3.1.29,>=1.0.0->wandb==0.17.0->-r requirements.txt (line 26))\n",
            "  Downloading gitdb-4.0.11-py3-none-any.whl (62 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard==2.15.2->-r requirements.txt (line 19)) (5.3.3)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard==2.15.2->-r requirements.txt (line 19)) (0.4.0)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.10/dist-packages (from google-auth<3,>=1.6.3->tensorboard==2.15.2->-r requirements.txt (line 19)) (4.9)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from google-auth-oauthlib<2,>=0.5->tensorboard==2.15.2->-r requirements.txt (line 19)) (1.3.1)\n",
            "Requirement already satisfied: cryptography>=3.4.0 in /usr/local/lib/python3.10/dist-packages (from pyjwt[crypto]>=2.4.0->PyGithub==2.3.0->-r requirements.txt (line 1)) (42.0.8)\n",
            "Requirement already satisfied: cffi>=1.4.1 in /usr/local/lib/python3.10/dist-packages (from pynacl>=1.4.0->PyGithub==2.3.0->-r requirements.txt (line 1)) (1.16.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.14.0->PyGithub==2.3.0->-r requirements.txt (line 1)) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.14.0->PyGithub==2.3.0->-r requirements.txt (line 1)) (3.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.14.0->PyGithub==2.3.0->-r requirements.txt (line 1)) (2024.6.2)\n",
            "Requirement already satisfied: docstring-parser>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from tyro>=0.5.11->trl==0.8.6->-r requirements.txt (line 24)) (0.16)\n",
            "Requirement already satisfied: rich>=11.1.0 in /usr/local/lib/python3.10/dist-packages (from tyro>=0.5.11->trl==0.8.6->-r requirements.txt (line 24)) (13.7.1)\n",
            "Collecting shtab>=1.5.6 (from tyro>=0.5.11->trl==0.8.6->-r requirements.txt (line 24))\n",
            "  Downloading shtab-1.7.1-py3-none-any.whl (14 kB)\n",
            "Requirement already satisfied: MarkupSafe>=2.1.1 in /usr/local/lib/python3.10/dist-packages (from werkzeug>=1.0.1->tensorboard==2.15.2->-r requirements.txt (line 19)) (2.1.5)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from Deprecated->PyGithub==2.3.0->-r requirements.txt (line 1)) (1.14.1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic->deepspeed==0.14.2->-r requirements.txt (line 6)) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.18.4 in /usr/local/lib/python3.10/dist-packages (from pydantic->deepspeed==0.14.2->-r requirements.txt (line 6)) (2.18.4)\n",
            "Requirement already satisfied: mpmath<1.4.0,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch==2.3.0->-r requirements.txt (line 21)) (1.3.0)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.4.1->pynacl>=1.4.0->PyGithub==2.3.0->-r requirements.txt (line 1)) (2.22)\n",
            "Collecting smmap<6,>=3.0.1 (from gitdb<5,>=4.0.1->gitpython!=3.1.29,>=1.0.0->wandb==0.17.0->-r requirements.txt (line 26))\n",
            "  Downloading smmap-5.0.1-py3-none-any.whl (24 kB)\n",
            "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /usr/local/lib/python3.10/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard==2.15.2->-r requirements.txt (line 19)) (0.6.0)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<2,>=0.5->tensorboard==2.15.2->-r requirements.txt (line 19)) (3.2.2)\n",
            "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich>=11.1.0->tyro>=0.5.11->trl==0.8.6->-r requirements.txt (line 24)) (3.0.0)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich>=11.1.0->tyro>=0.5.11->trl==0.8.6->-r requirements.txt (line 24)) (2.16.1)\n",
            "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich>=11.1.0->tyro>=0.5.11->trl==0.8.6->-r requirements.txt (line 24)) (0.1.2)\n",
            "Building wheels for collected packages: deepspeed, flash-attn, unsloth\n",
            "  Building wheel for deepspeed (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for deepspeed: filename=deepspeed-0.14.2-py3-none-any.whl size=1432244 sha256=42c53f76b12f447613e80e3c87f602182789b71e533f07058b166045e6071ef8\n",
            "  Stored in directory: /root/.cache/pip/wheels/ea/7c/43/bed44d8414c099ff962b754f425f7ff77cc623cc8a98e0da70\n",
            "  Building wheel for flash-attn (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for flash-attn: filename=flash_attn-2.5.9.post1-cp310-cp310-linux_x86_64.whl size=120889689 sha256=5022ba11d48bf74926da9c16260f4ea2b9bb7f4e29bdb4bd6e1383ad1c55d16f\n",
            "  Stored in directory: /root/.cache/pip/wheels/cc/ad/f6/7ccf0238790d6346e9fe622923a76ec218e890d356b9a2754a\n",
            "  Building wheel for unsloth (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for unsloth: filename=unsloth-2024.5-py3-none-any.whl size=109360 sha256=901b2df84f096d49fb51e4fbc6331f11be2fe1b38018f41b477a4c56a443b041\n",
            "  Stored in directory: /root/.cache/pip/wheels/6d/11/2a/f1be1ce0882e88e425e56e10b1081175f03e7e37f42e7c84e1\n",
            "Successfully built deepspeed flash-attn unsloth\n",
            "Installing collected packages: hjson, xxhash, unsloth, smmap, shtab, setproctitle, sentry-sdk, pynvml, pyjwt, nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, loguru, fsspec, einops, docker-pycreds, dill, Deprecated, tiktoken, pynacl, nvidia-cusparse-cu12, nvidia-cudnn-cu12, multiprocess, huggingface-hub, gitdb, tyro, nvidia-cusolver-cu12, gitpython, datatrove, wandb, transformers, PyGithub, datasets, xformers, flash-attn, evaluate, deepspeed, bitsandbytes, accelerate, trl, peft\n",
            "  Attempting uninstall: pyjwt\n",
            "    Found existing installation: PyJWT 2.3.0\n",
            "    Uninstalling PyJWT-2.3.0:\n",
            "      Successfully uninstalled PyJWT-2.3.0\n",
            "  Attempting uninstall: fsspec\n",
            "    Found existing installation: fsspec 2023.6.0\n",
            "    Uninstalling fsspec-2023.6.0:\n",
            "      Successfully uninstalled fsspec-2023.6.0\n",
            "  Attempting uninstall: huggingface-hub\n",
            "    Found existing installation: huggingface-hub 0.23.3\n",
            "    Uninstalling huggingface-hub-0.23.3:\n",
            "      Successfully uninstalled huggingface-hub-0.23.3\n",
            "  Attempting uninstall: transformers\n",
            "    Found existing installation: transformers 4.41.2\n",
            "    Uninstalling transformers-4.41.2:\n",
            "      Successfully uninstalled transformers-4.41.2\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "gcsfs 2023.6.0 requires fsspec==2023.6.0, but you have fsspec 2024.3.1 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0mSuccessfully installed Deprecated-1.2.14 PyGithub-2.3.0 accelerate-0.30.1 bitsandbytes-0.43.1 datasets-2.19.1 datatrove-0.2.0 deepspeed-0.14.2 dill-0.3.8 docker-pycreds-0.4.0 einops-0.8.0 evaluate-0.4.2 flash-attn-2.5.9.post1 fsspec-2024.3.1 gitdb-4.0.11 gitpython-3.1.43 hjson-3.1.0 huggingface-hub-0.23.1 loguru-0.7.2 multiprocess-0.70.16 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.5.40 nvidia-nvtx-cu12-12.1.105 peft-0.11.1 pyjwt-2.8.0 pynacl-1.5.0 pynvml-11.5.0 sentry-sdk-2.5.1 setproctitle-1.3.3 shtab-1.7.1 smmap-5.0.1 tiktoken-0.7.0 transformers-4.41.1 trl-0.8.6 tyro-0.8.4 unsloth-2024.5 wandb-0.17.0 xformers-0.0.26.post1 xxhash-3.4.1\n"
          ]
        }
      ],
      "source": [
        "%cd repo\n",
        "!pip install -r requirements.txt"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Login to Weights and Biases and Hugging Face"
      ],
      "metadata": {
        "id": "buwM3hKZNzaV"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PUq-nUxnRtTC",
        "outputId": "8063792a-2a04-43a4-e192-0a6ec078b177"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Paste an API key from your profile and hit enter, or press ctrl+c to quit: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        }
      ],
      "source": [
        "!wandb login --relogin"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 145,
          "referenced_widgets": [
            "7adad8f01b7148dab6f14e9c06a7e121",
            "fa303d8c55b24a749323bcead74f2b72",
            "d05dbe871c684a4d9ec6af70365d2d22",
            "7df5a4fe65f1485495b2b07e6ac9e63b",
            "ff62f0037d3047b3880043dd54f8f83c",
            "402aa7fab1054cc39e96f3f6874b1da1",
            "70fc60afbb954619b71f638372816b55",
            "fe3cfefa2ee24778bcd814734c2159ea",
            "a2a279eb7c594c18a2a3f701da8a7e77",
            "b7af5f8b4acf4f64a3c835dca72a4ec0",
            "bacbeba271d9431f9057bcb21c4f6807",
            "9b878252c2d64c688d60428aa0c5d1a3",
            "e1d13ba5edae407ea96d87df297a3b97",
            "c361af9a7ae14786b93ff67dad41ebc4",
            "b4fb3e78617046e383e2b9b6772e60b4",
            "cf93cfbe66514dc183c4918140c4a708",
            "69aa14cac03149b398601a1a6e0e8ff1",
            "8f35f026c9e74e0c84a3fcf55d1c0f7e",
            "3e84f52b1e8141698466ea4bc6b76da8",
            "5993de4ad5c840d1924cf862cffc2a34",
            "a7c95591dba345b2b6721c751e06d256",
            "d3cbd4816c73493fafb9223183d47e5f",
            "337022a49d48474aad7b01563daf8836",
            "49d145c662b54f7a9c134684fe6341cc",
            "0ca4996f60a04d61b4b3c142ad34a44b",
            "421afb1fe86149c5b4bafeae55931802",
            "f5b326935a2741e1894f13e8c8ee3bd0",
            "0a45e68f1c234c8084e7d9935d113e06",
            "4ae883bfac18451cb205683db63046e7",
            "69fe511511e0416e813e38e7aab8544b",
            "795017bc9fbb4d2db79ff669cf1a8bfc",
            "5306a61587704526bb59601b16559224"
          ]
        },
        "id": "se069Y2AR9xn",
        "outputId": "0928c976-622d-4093-8230-1148c52a543c"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7adad8f01b7148dab6f14e9c06a7e121"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "from huggingface_hub import notebook_login\n",
        "\n",
        "notebook_login()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Train the model\n",
        "\n",
        "This script starts the training process. The fine-tuned model will be saved to your Hugging Face account. The progress can be tracked in real time in Weights and Biases."
      ],
      "metadata": {
        "id": "Wx_zEIhHN2mQ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kvgj7tSpHu-_",
        "outputId": "cfcbfe99-5f31-483a-c1e4-ab852f1577ab"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2024-06-15 09:00:43.075177: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
            "2024-06-15 09:00:43.127223: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-06-15 09:00:43.127275: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-06-15 09:00:43.128861: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-06-15 09:00:43.137416: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-06-15 09:00:44.282789: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "/usr/local/lib/python3.10/dist-packages/transformers/training_args.py:1474: FutureWarning: `evaluation_strategy` is deprecated and will be removed in version 4.46 of 🤗 Transformers. Use `eval_strategy` instead\n",
            "  warnings.warn(\n",
            "tokenizer_config.json: 100% 749/749 [00:00<00:00, 5.27MB/s]\n",
            "tokenizer.model: 100% 500k/500k [00:00<00:00, 70.6MB/s]\n",
            "tokenizer.json: 100% 1.84M/1.84M [00:00<00:00, 6.95MB/s]\n",
            "special_tokens_map.json: 100% 411/411 [00:00<00:00, 3.13MB/s]\n",
            "Downloading readme: 100% 332/332 [00:00<00:00, 2.14MB/s]\n",
            "Downloading data: 100% 43.2M/43.2M [00:02<00:00, 17.2MB/s]\n",
            "Generating train split: 100% 990/990 [00:00<00:00, 1103.19 examples/s]\n",
            "Size of the train set: 891. Size of the validation set: 99\n",
            "100% 400/400 [02:40<00:00,  2.50it/s]\n",
            "The character to token ratio of the dataset is: 3.79\n",
            "Chunked document into 6 chunks. Total chunks: 6.\n",
            "Chunked document into 2 chunks. Total chunks: 8.\n",
            "Chunked document into 14 chunks. Total chunks: 22.\n",
            "Chunked document into 2 chunks. Total chunks: 24.\n",
            "Chunked document into 11 chunks. Total chunks: 35.\n",
            "Chunked document into 7 chunks. Total chunks: 42.\n",
            "Chunked document into 1 chunks. Total chunks: 43.\n",
            "Chunked document into 1 chunks. Total chunks: 44.\n",
            "Chunked document into 1 chunks. Total chunks: 45.\n",
            "Chunked document into 26 chunks. Total chunks: 71.\n",
            "Chunked document into 87 chunks. Total chunks: 158.\n",
            "Chunked document into 16 chunks. Total chunks: 174.\n",
            "Chunked document into 1 chunks. Total chunks: 175.\n",
            "Chunked document into 10 chunks. Total chunks: 185.\n",
            "Chunked document into 845 chunks. Total chunks: 1030.\n",
            "Skipping last short sample\n",
            "A sample of valid dataset: {'input_ids': tensor([    1, 32007,  1722,  ..., 29908, 32010,     2]), 'labels': tensor([    1, 32007,  1722,  ..., 29908, 32010,     2])}\n",
            "config.json: 100% 637/637 [00:00<00:00, 5.02MB/s]\n",
            "model.safetensors.index.json: 100% 25.1k/25.1k [00:00<00:00, 85.5MB/s]\n",
            "Downloading shards:   0% 0/2 [00:00<?, ?it/s]\n",
            "model-00001-of-00002.safetensors:   0% 0.00/9.98G [00:00<?, ?B/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   0% 41.9M/9.98G [00:00<00:23, 415MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   1% 105M/9.98G [00:00<00:19, 508MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:   2% 168M/9.98G [00:00<00:18, 533MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   2% 231M/9.98G [00:00<00:17, 543MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   3% 294M/9.98G [00:00<00:17, 550MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   4% 357M/9.98G [00:00<00:17, 554MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   4% 419M/9.98G [00:00<00:17, 556MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   5% 482M/9.98G [00:00<00:18, 522MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   5% 545M/9.98G [00:01<00:18, 509MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   6% 598M/9.98G [00:01<00:18, 504MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   7% 650M/9.98G [00:01<00:18, 502MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   7% 703M/9.98G [00:01<00:18, 498MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   8% 755M/9.98G [00:01<00:18, 497MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   8% 807M/9.98G [00:01<00:18, 495MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   9% 860M/9.98G [00:01<00:18, 483MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:   9% 912M/9.98G [00:01<00:19, 472MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  10% 965M/9.98G [00:01<00:19, 467MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  10% 1.02G/9.98G [00:02<00:19, 465MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  11% 1.07G/9.98G [00:02<00:19, 464MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  11% 1.12G/9.98G [00:02<00:19, 463MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  12% 1.17G/9.98G [00:02<00:19, 459MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  12% 1.23G/9.98G [00:02<00:19, 460MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  13% 1.28G/9.98G [00:02<00:18, 463MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  13% 1.33G/9.98G [00:02<00:18, 461MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  14% 1.38G/9.98G [00:02<00:18, 463MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  14% 1.44G/9.98G [00:02<00:18, 457MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  15% 1.49G/9.98G [00:03<00:18, 470MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  15% 1.54G/9.98G [00:03<00:18, 465MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  16% 1.59G/9.98G [00:03<00:17, 476MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  17% 1.65G/9.98G [00:03<00:17, 479MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  17% 1.70G/9.98G [00:03<00:17, 481MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  18% 1.75G/9.98G [00:03<00:17, 477MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  18% 1.80G/9.98G [00:03<00:16, 481MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  19% 1.86G/9.98G [00:06<02:08, 63.1MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  19% 1.91G/9.98G [00:06<01:34, 85.4MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  20% 1.96G/9.98G [00:06<01:10, 113MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  20% 2.01G/9.98G [00:06<00:54, 146MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  21% 2.07G/9.98G [00:06<00:42, 185MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  21% 2.12G/9.98G [00:06<00:34, 227MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  22% 2.17G/9.98G [00:06<00:28, 269MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  22% 2.22G/9.98G [00:06<00:25, 309MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  23% 2.28G/9.98G [00:07<00:22, 344MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  23% 2.33G/9.98G [00:07<00:20, 375MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  24% 2.38G/9.98G [00:07<00:19, 398MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  24% 2.43G/9.98G [00:07<00:18, 417MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  25% 2.49G/9.98G [00:07<00:17, 431MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  25% 2.54G/9.98G [00:07<00:16, 442MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  26% 2.59G/9.98G [00:07<00:16, 453MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  26% 2.64G/9.98G [00:07<00:16, 457MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  27% 2.69G/9.98G [00:07<00:15, 461MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  28% 2.75G/9.98G [00:08<00:15, 466MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  28% 2.80G/9.98G [00:08<00:15, 469MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  29% 2.85G/9.98G [00:08<00:15, 474MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  29% 2.90G/9.98G [00:08<00:14, 475MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  30% 2.96G/9.98G [00:08<00:14, 476MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  30% 3.01G/9.98G [00:08<00:14, 470MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  31% 3.06G/9.98G [00:08<00:14, 467MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  31% 3.11G/9.98G [00:08<00:14, 462MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  32% 3.17G/9.98G [00:09<00:14, 462MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  32% 3.22G/9.98G [00:09<00:14, 458MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  33% 3.27G/9.98G [00:09<00:16, 417MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  33% 3.32G/9.98G [00:09<00:15, 418MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  34% 3.38G/9.98G [00:09<00:15, 430MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  34% 3.43G/9.98G [00:09<00:19, 328MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  35% 3.48G/9.98G [00:09<00:18, 354MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  35% 3.53G/9.98G [00:09<00:17, 379MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  36% 3.59G/9.98G [00:10<00:16, 398MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  36% 3.64G/9.98G [00:10<00:15, 418MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  37% 3.69G/9.98G [00:10<00:14, 424MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  38% 3.74G/9.98G [00:10<00:14, 439MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  38% 3.80G/9.98G [00:10<00:13, 458MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  39% 3.85G/9.98G [00:10<00:12, 476MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  39% 3.90G/9.98G [00:10<00:12, 489MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  40% 3.96G/9.98G [00:10<00:11, 502MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  40% 4.03G/9.98G [00:10<00:11, 510MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  41% 4.08G/9.98G [00:11<00:11, 511MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  41% 4.13G/9.98G [00:11<00:11, 499MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  42% 4.18G/9.98G [00:11<00:11, 488MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  42% 4.24G/9.98G [00:11<00:18, 314MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  43% 4.29G/9.98G [00:11<00:16, 348MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  44% 4.34G/9.98G [00:11<00:14, 376MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  44% 4.39G/9.98G [00:11<00:14, 396MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  45% 4.45G/9.98G [00:12<00:13, 413MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  45% 4.50G/9.98G [00:12<00:12, 426MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  46% 4.55G/9.98G [00:12<00:12, 436MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  46% 4.60G/9.98G [00:12<00:12, 443MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  47% 4.66G/9.98G [00:12<00:12, 435MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  47% 4.71G/9.98G [00:12<00:14, 357MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  48% 4.76G/9.98G [00:12<00:13, 394MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  48% 4.82G/9.98G [00:12<00:11, 432MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  49% 4.89G/9.98G [00:13<00:11, 457MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  50% 4.94G/9.98G [00:13<00:10, 470MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  50% 4.99G/9.98G [00:13<00:10, 477MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  51% 5.04G/9.98G [00:13<00:10, 483MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  51% 5.11G/9.98G [00:13<00:09, 499MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  52% 5.17G/9.98G [00:13<00:09, 510MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  52% 5.23G/9.98G [00:13<00:09, 524MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  53% 5.30G/9.98G [00:13<00:09, 518MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  54% 5.35G/9.98G [00:13<00:09, 505MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  54% 5.41G/9.98G [00:14<00:08, 514MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  55% 5.46G/9.98G [00:14<00:08, 515MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  55% 5.53G/9.98G [00:14<00:08, 521MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  56% 5.58G/9.98G [00:14<00:08, 522MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  56% 5.63G/9.98G [00:14<00:08, 522MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  57% 5.69G/9.98G [00:14<00:08, 525MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  58% 5.76G/9.98G [00:14<00:08, 517MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  58% 5.81G/9.98G [00:14<00:08, 513MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  59% 5.86G/9.98G [00:15<00:08, 494MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  59% 5.92G/9.98G [00:15<00:08, 506MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  60% 5.99G/9.98G [00:15<00:07, 514MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  61% 6.04G/9.98G [00:15<00:08, 478MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  61% 6.09G/9.98G [00:15<00:08, 456MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  62% 6.14G/9.98G [00:15<00:08, 453MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  62% 6.20G/9.98G [00:15<00:08, 422MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  63% 6.25G/9.98G [00:15<00:09, 389MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  63% 6.29G/9.98G [00:16<00:09, 396MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  63% 6.33G/9.98G [00:16<00:09, 389MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  64% 6.39G/9.98G [00:16<00:09, 387MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  64% 6.43G/9.98G [00:16<00:09, 394MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  65% 6.47G/9.98G [00:16<00:09, 377MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  65% 6.51G/9.98G [00:18<00:55, 62.5MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  66% 6.56G/9.98G [00:18<00:38, 88.7MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  66% 6.62G/9.98G [00:18<00:27, 121MB/s] \u001b[A\n",
            "model-00001-of-00002.safetensors:  67% 6.68G/9.98G [00:18<00:19, 167MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  67% 6.73G/9.98G [00:19<00:15, 207MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  68% 6.79G/9.98G [00:19<00:12, 261MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  69% 6.85G/9.98G [00:19<00:10, 304MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  69% 6.90G/9.98G [00:19<00:09, 336MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  70% 6.96G/9.98G [00:19<00:07, 387MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  70% 7.03G/9.98G [00:19<00:06, 429MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  71% 7.09G/9.98G [00:19<00:06, 460MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  72% 7.15G/9.98G [00:19<00:05, 483MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  72% 7.21G/9.98G [00:19<00:05, 501MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  73% 7.28G/9.98G [00:20<00:05, 516MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  74% 7.34G/9.98G [00:20<00:04, 528MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  74% 7.40G/9.98G [00:20<00:04, 537MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  75% 7.47G/9.98G [00:20<00:04, 542MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  75% 7.53G/9.98G [00:20<00:04, 545MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  76% 7.59G/9.98G [00:20<00:04, 508MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  77% 7.64G/9.98G [00:20<00:05, 433MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  77% 7.70G/9.98G [00:21<00:06, 367MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  78% 7.74G/9.98G [00:21<00:07, 297MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  78% 7.78G/9.98G [00:21<00:07, 292MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  79% 7.83G/9.98G [00:21<00:06, 319MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  79% 7.89G/9.98G [00:21<00:05, 351MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  80% 7.94G/9.98G [00:21<00:05, 377MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  80% 7.99G/9.98G [00:21<00:04, 399MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  81% 8.04G/9.98G [00:21<00:04, 413MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  81% 8.10G/9.98G [00:22<00:04, 425MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  82% 8.15G/9.98G [00:22<00:04, 434MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  82% 8.20G/9.98G [00:22<00:04, 444MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  83% 8.25G/9.98G [00:22<00:03, 445MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  83% 8.32G/9.98G [00:22<00:03, 472MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  84% 8.38G/9.98G [00:22<00:03, 493MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  85% 8.44G/9.98G [00:22<00:03, 505MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  85% 8.50G/9.98G [00:22<00:02, 504MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  86% 8.56G/9.98G [00:23<00:02, 489MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  86% 8.61G/9.98G [00:23<00:02, 485MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  87% 8.66G/9.98G [00:23<00:02, 482MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  87% 8.71G/9.98G [00:23<00:02, 478MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  88% 8.77G/9.98G [00:23<00:02, 475MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  88% 8.82G/9.98G [00:23<00:02, 473MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  89% 8.87G/9.98G [00:23<00:02, 476MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  89% 8.92G/9.98G [00:23<00:02, 467MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  90% 8.98G/9.98G [00:23<00:02, 470MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  90% 9.03G/9.98G [00:24<00:02, 470MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  91% 9.08G/9.98G [00:24<00:01, 470MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  92% 9.13G/9.98G [00:24<00:01, 470MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  92% 9.19G/9.98G [00:24<00:01, 468MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  93% 9.24G/9.98G [00:24<00:01, 464MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  93% 9.29G/9.98G [00:24<00:01, 468MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  94% 9.34G/9.98G [00:24<00:01, 463MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  94% 9.40G/9.98G [00:24<00:01, 466MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  95% 9.45G/9.98G [00:24<00:01, 462MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  95% 9.50G/9.98G [00:25<00:01, 465MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  96% 9.55G/9.98G [00:25<00:00, 465MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  96% 9.60G/9.98G [00:25<00:00, 465MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  97% 9.66G/9.98G [00:25<00:00, 464MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  97% 9.71G/9.98G [00:25<00:00, 466MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  98% 9.76G/9.98G [00:25<00:00, 465MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  98% 9.81G/9.98G [00:25<00:00, 469MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  99% 9.87G/9.98G [00:25<00:00, 470MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors:  99% 9.92G/9.98G [00:25<00:00, 460MB/s]\u001b[A\n",
            "model-00001-of-00002.safetensors: 100% 9.98G/9.98G [00:26<00:00, 382MB/s]\n",
            "Downloading shards:  50% 1/2 [00:26<00:26, 26.33s/it]\n",
            "model-00002-of-00002.safetensors:   0% 0.00/3.50G [00:00<?, ?B/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:   1% 52.4M/3.50G [00:00<00:07, 444MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:   3% 105M/3.50G [00:00<00:07, 454MB/s] \u001b[A\n",
            "model-00002-of-00002.safetensors:   4% 157M/3.50G [00:00<00:07, 459MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:   6% 210M/3.50G [00:00<00:07, 464MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:   7% 262M/3.50G [00:00<00:06, 466MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:   9% 315M/3.50G [00:00<00:06, 464MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  10% 367M/3.50G [00:00<00:06, 463MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  12% 419M/3.50G [00:00<00:06, 465MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  13% 472M/3.50G [00:01<00:06, 464MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  15% 524M/3.50G [00:01<00:06, 464MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  16% 577M/3.50G [00:01<00:06, 467MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  18% 629M/3.50G [00:01<00:06, 465MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  19% 682M/3.50G [00:01<00:05, 472MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  21% 734M/3.50G [00:01<00:06, 411MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  22% 786M/3.50G [00:01<00:06, 407MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  24% 839M/3.50G [00:01<00:06, 428MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  25% 891M/3.50G [00:02<00:06, 395MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  27% 933M/3.50G [00:02<00:06, 393MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  28% 975M/3.50G [00:02<00:06, 369MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  29% 1.03G/3.50G [00:02<00:06, 374MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  31% 1.08G/3.50G [00:02<00:05, 404MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  32% 1.12G/3.50G [00:04<00:36, 65.6MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  34% 1.17G/3.50G [00:04<00:25, 90.4MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  35% 1.23G/3.50G [00:04<00:18, 121MB/s] \u001b[A\n",
            "model-00002-of-00002.safetensors:  37% 1.28G/3.50G [00:04<00:14, 156MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  38% 1.33G/3.50G [00:05<00:10, 199MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  40% 1.38G/3.50G [00:05<00:08, 244MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  41% 1.45G/3.50G [00:05<00:06, 298MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  43% 1.50G/3.50G [00:05<00:06, 330MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  44% 1.55G/3.50G [00:05<00:05, 361MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  46% 1.60G/3.50G [00:05<00:04, 385MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  47% 1.66G/3.50G [00:05<00:04, 404MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  49% 1.71G/3.50G [00:05<00:04, 415MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  50% 1.76G/3.50G [00:05<00:04, 429MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  52% 1.81G/3.50G [00:06<00:03, 436MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  53% 1.87G/3.50G [00:06<00:03, 443MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  55% 1.92G/3.50G [00:06<00:03, 450MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  56% 1.97G/3.50G [00:06<00:03, 454MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  58% 2.02G/3.50G [00:06<00:03, 454MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  59% 2.08G/3.50G [00:06<00:03, 450MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  61% 2.13G/3.50G [00:06<00:03, 456MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  62% 2.18G/3.50G [00:06<00:02, 458MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  64% 2.23G/3.50G [00:07<00:02, 457MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  65% 2.29G/3.50G [00:07<00:02, 457MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  67% 2.34G/3.50G [00:07<00:02, 460MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  68% 2.39G/3.50G [00:07<00:02, 457MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  70% 2.44G/3.50G [00:07<00:02, 463MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  71% 2.50G/3.50G [00:07<00:02, 456MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  73% 2.55G/3.50G [00:07<00:02, 451MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  74% 2.60G/3.50G [00:07<00:02, 444MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  76% 2.65G/3.50G [00:08<00:02, 294MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  77% 2.71G/3.50G [00:08<00:02, 325MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  79% 2.76G/3.50G [00:08<00:02, 351MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  80% 2.81G/3.50G [00:08<00:01, 373MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  82% 2.86G/3.50G [00:08<00:01, 395MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  83% 2.92G/3.50G [00:08<00:01, 410MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  85% 2.97G/3.50G [00:08<00:01, 424MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  86% 3.02G/3.50G [00:09<00:01, 412MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  88% 3.07G/3.50G [00:09<00:01, 424MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  89% 3.12G/3.50G [00:09<00:00, 439MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  91% 3.18G/3.50G [00:09<00:00, 452MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  92% 3.23G/3.50G [00:09<00:00, 463MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  94% 3.28G/3.50G [00:09<00:00, 470MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  95% 3.33G/3.50G [00:09<00:00, 474MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  97% 3.39G/3.50G [00:09<00:00, 479MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors:  98% 3.44G/3.50G [00:09<00:00, 462MB/s]\u001b[A\n",
            "model-00002-of-00002.safetensors: 100% 3.50G/3.50G [00:10<00:00, 349MB/s]\n",
            "Downloading shards: 100% 2/2 [00:36<00:00, 18.33s/it]\n",
            "Loading checkpoint shards: 100% 2/2 [00:08<00:00,  4.05s/it]\n",
            "generation_config.json: 100% 116/116 [00:00<00:00, 870kB/s]\n",
            "max_steps is given, it will override any value given in num_train_epochs\n",
            "Using auto half precision backend\n",
            "PeftModelForCausalLM(\n",
            "  (base_model): LoraModel(\n",
            "    (model): LlamaForCausalLM(\n",
            "      (model): LlamaModel(\n",
            "        (embed_tokens): Embedding(32016, 4096)\n",
            "        (layers): ModuleList(\n",
            "          (0-31): 32 x LlamaDecoderLayer(\n",
            "            (self_attn): LlamaFlashAttention2(\n",
            "              (q_proj): lora.Linear4bit(\n",
            "                (base_layer): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
            "                (lora_dropout): ModuleDict(\n",
            "                  (default): Dropout(p=0.1, inplace=False)\n",
            "                )\n",
            "                (lora_A): ModuleDict(\n",
            "                  (default): Linear(in_features=4096, out_features=32, bias=False)\n",
            "                )\n",
            "                (lora_B): ModuleDict(\n",
            "                  (default): Linear(in_features=32, out_features=4096, bias=False)\n",
            "                )\n",
            "                (lora_embedding_A): ParameterDict()\n",
            "                (lora_embedding_B): ParameterDict()\n",
            "              )\n",
            "              (k_proj): lora.Linear4bit(\n",
            "                (base_layer): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
            "                (lora_dropout): ModuleDict(\n",
            "                  (default): Dropout(p=0.1, inplace=False)\n",
            "                )\n",
            "                (lora_A): ModuleDict(\n",
            "                  (default): Linear(in_features=4096, out_features=32, bias=False)\n",
            "                )\n",
            "                (lora_B): ModuleDict(\n",
            "                  (default): Linear(in_features=32, out_features=4096, bias=False)\n",
            "                )\n",
            "                (lora_embedding_A): ParameterDict()\n",
            "                (lora_embedding_B): ParameterDict()\n",
            "              )\n",
            "              (v_proj): lora.Linear4bit(\n",
            "                (base_layer): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
            "                (lora_dropout): ModuleDict(\n",
            "                  (default): Dropout(p=0.1, inplace=False)\n",
            "                )\n",
            "                (lora_A): ModuleDict(\n",
            "                  (default): Linear(in_features=4096, out_features=32, bias=False)\n",
            "                )\n",
            "                (lora_B): ModuleDict(\n",
            "                  (default): Linear(in_features=32, out_features=4096, bias=False)\n",
            "                )\n",
            "                (lora_embedding_A): ParameterDict()\n",
            "                (lora_embedding_B): ParameterDict()\n",
            "              )\n",
            "              (o_proj): lora.Linear4bit(\n",
            "                (base_layer): Linear4bit(in_features=4096, out_features=4096, bias=False)\n",
            "                (lora_dropout): ModuleDict(\n",
            "                  (default): Dropout(p=0.1, inplace=False)\n",
            "                )\n",
            "                (lora_A): ModuleDict(\n",
            "                  (default): Linear(in_features=4096, out_features=32, bias=False)\n",
            "                )\n",
            "                (lora_B): ModuleDict(\n",
            "                  (default): Linear(in_features=32, out_features=4096, bias=False)\n",
            "                )\n",
            "                (lora_embedding_A): ParameterDict()\n",
            "                (lora_embedding_B): ParameterDict()\n",
            "              )\n",
            "              (rotary_emb): LlamaRotaryEmbedding()\n",
            "            )\n",
            "            (mlp): LlamaMLP(\n",
            "              (gate_proj): lora.Linear4bit(\n",
            "                (base_layer): Linear4bit(in_features=4096, out_features=11008, bias=False)\n",
            "                (lora_dropout): ModuleDict(\n",
            "                  (default): Dropout(p=0.1, inplace=False)\n",
            "                )\n",
            "                (lora_A): ModuleDict(\n",
            "                  (default): Linear(in_features=4096, out_features=32, bias=False)\n",
            "                )\n",
            "                (lora_B): ModuleDict(\n",
            "                  (default): Linear(in_features=32, out_features=11008, bias=False)\n",
            "                )\n",
            "                (lora_embedding_A): ParameterDict()\n",
            "                (lora_embedding_B): ParameterDict()\n",
            "              )\n",
            "              (up_proj): lora.Linear4bit(\n",
            "                (base_layer): Linear4bit(in_features=4096, out_features=11008, bias=False)\n",
            "                (lora_dropout): ModuleDict(\n",
            "                  (default): Dropout(p=0.1, inplace=False)\n",
            "                )\n",
            "                (lora_A): ModuleDict(\n",
            "                  (default): Linear(in_features=4096, out_features=32, bias=False)\n",
            "                )\n",
            "                (lora_B): ModuleDict(\n",
            "                  (default): Linear(in_features=32, out_features=11008, bias=False)\n",
            "                )\n",
            "                (lora_embedding_A): ParameterDict()\n",
            "                (lora_embedding_B): ParameterDict()\n",
            "              )\n",
            "              (down_proj): lora.Linear4bit(\n",
            "                (base_layer): Linear4bit(in_features=11008, out_features=4096, bias=False)\n",
            "                (lora_dropout): ModuleDict(\n",
            "                  (default): Dropout(p=0.1, inplace=False)\n",
            "                )\n",
            "                (lora_A): ModuleDict(\n",
            "                  (default): Linear(in_features=11008, out_features=32, bias=False)\n",
            "                )\n",
            "                (lora_B): ModuleDict(\n",
            "                  (default): Linear(in_features=32, out_features=4096, bias=False)\n",
            "                )\n",
            "                (lora_embedding_A): ParameterDict()\n",
            "                (lora_embedding_B): ParameterDict()\n",
            "              )\n",
            "              (act_fn): SiLU()\n",
            "            )\n",
            "            (input_layernorm): LlamaRMSNorm()\n",
            "            (post_attention_layernorm): LlamaRMSNorm()\n",
            "          )\n",
            "        )\n",
            "        (norm): LlamaRMSNorm()\n",
            "      )\n",
            "      (lm_head): Linear(in_features=4096, out_features=32016, bias=False)\n",
            "    )\n",
            "  )\n",
            ")\n",
            "trainable params: 79,953,920 || all params: 6,818,500,608 || trainable%: 1.1726\n",
            "[2024-06-15 09:04:37,507] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n",
            "\u001b[93m [WARNING] \u001b[0m async_io requires the dev libaio .so object and headers but these were not found.\n",
            "\u001b[93m [WARNING] \u001b[0m async_io: please install the libaio-dev package with apt\n",
            "\u001b[93m [WARNING] \u001b[0m If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found.\n",
            "\u001b[93m [WARNING] \u001b[0m Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH\n",
            "\u001b[93m [WARNING] \u001b[0m sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.3\n",
            "\u001b[93m [WARNING] \u001b[0m using untested triton version (2.3.0), only 1.0.0 is known to be compatible\n",
            "***** Running training *****\n",
            "  Num examples = 3,200\n",
            "  Num Epochs = 9,223,372,036,854,775,807\n",
            "  Instantaneous batch size per device = 4\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 32\n",
            "  Gradient Accumulation steps = 8\n",
            "  Total optimization steps = 100\n",
            "  Number of trainable parameters = 79,953,920\n",
            "Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mbohdanpetryshyn\u001b[0m (\u001b[33mbohdan-petryshyn\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: wandb version 0.17.1 is available!  To upgrade, please run:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:  $ pip install wandb --upgrade\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Tracking run with wandb version 0.17.0\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run data is saved locally in \u001b[35m\u001b[1m/content/repo/wandb/run-20240615_090439-naaswf2d\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run \u001b[1m`wandb offline`\u001b[0m to turn off syncing.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Syncing run \u001b[33mtmp-codellama-7b-openapi-completion-ctx-lvl-fim-05-spm-2048\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at \u001b[34m\u001b[4mhttps://wandb.ai/bohdan-petryshyn/huggingface\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run at \u001b[34m\u001b[4mhttps://wandb.ai/bohdan-petryshyn/huggingface/runs/naaswf2d\u001b[0m\n",
            "  0% 0/100 [00:00<?, ?it/s]Chunked document into 9 chunks. Total chunks: 9.\n",
            "Chunked document into 146 chunks. Total chunks: 155.\n",
            "Chunked document into 10 chunks. Total chunks: 165.\n",
            "Chunked document into 6 chunks. Total chunks: 171.\n",
            "Chunked document into 14 chunks. Total chunks: 185.\n",
            "Chunked document into 38 chunks. Total chunks: 223.\n",
            "Chunked document into 6 chunks. Total chunks: 229.\n",
            "Chunked document into 1 chunks. Total chunks: 230.\n",
            "Chunked document into 8 chunks. Total chunks: 238.\n",
            "Chunked document into 284 chunks. Total chunks: 522.\n",
            "Chunked document into 2 chunks. Total chunks: 524.\n",
            "Chunked document into 1 chunks. Total chunks: 525.\n",
            "Chunked document into 42 chunks. Total chunks: 567.\n",
            "Chunked document into 2 chunks. Total chunks: 569.\n",
            "Chunked document into 2 chunks. Total chunks: 571.\n",
            "Chunked document into 4 chunks. Total chunks: 575.\n",
            "Chunked document into 1 chunks. Total chunks: 576.\n",
            "Chunked document into 106 chunks. Total chunks: 682.\n",
            "Chunked document into 5 chunks. Total chunks: 687.\n",
            "Chunked document into 4 chunks. Total chunks: 691.\n",
            "Chunked document into 1 chunks. Total chunks: 692.\n",
            "Chunked document into 23 chunks. Total chunks: 715.\n",
            "Chunked document into 51 chunks. Total chunks: 766.\n",
            "Chunked document into 1 chunks. Total chunks: 767.\n",
            "Chunked document into 10 chunks. Total chunks: 777.\n",
            "Chunked document into 5 chunks. Total chunks: 782.\n",
            "Chunked document into 19 chunks. Total chunks: 801.\n",
            "Chunked document into 43 chunks. Total chunks: 844.\n",
            "Chunked document into 62 chunks. Total chunks: 906.\n",
            "Chunked document into 12 chunks. Total chunks: 918.\n",
            "Chunked document into 5 chunks. Total chunks: 923.\n",
            "Chunked document into 3 chunks. Total chunks: 926.\n",
            "Chunked document into 7 chunks. Total chunks: 933.\n",
            "Chunked document into 6 chunks. Total chunks: 939.\n",
            "Chunked document into 23 chunks. Total chunks: 962.\n",
            "Chunked document into 10 chunks. Total chunks: 972.\n",
            "Chunked document into 14 chunks. Total chunks: 986.\n",
            "Chunked document into 3 chunks. Total chunks: 989.\n",
            "Chunked document into 1 chunks. Total chunks: 990.\n",
            "Chunked document into 9 chunks. Total chunks: 999.\n",
            "Chunked document into 4 chunks. Total chunks: 1003.\n",
            "Chunked document into 2 chunks. Total chunks: 1005.\n",
            "Chunked document into 42 chunks. Total chunks: 1047.\n",
            "suffix too short 7 7\n",
            "Skipping last short sample\n",
            "The input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in torch.bfloat16.\n",
            "{'loss': 0.827, 'grad_norm': 0.12219177186489105, 'learning_rate': 0.0002, 'epoch': 0.05}\n",
            "  5% 5/100 [02:34<47:22, 29.92s/it]Traceback (most recent call last):\n",
            "  File \"/content/repo/train.py\", line 392, in <module>\n",
            "    main(model_args, data_args, training_args)\n",
            "  File \"/content/repo/train.py\", line 372, in main\n",
            "    trainer.train(resume_from_checkpoint=checkpoint)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/trainer.py\", line 1876, in train\n",
            "    return inner_training_loop(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/trainer.py\", line 2216, in _inner_training_loop\n",
            "    tr_loss_step = self.training_step(model, inputs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/trainer.py\", line 3238, in training_step\n",
            "    loss = self.compute_loss(model, inputs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/trainer.py\", line 3264, in compute_loss\n",
            "    outputs = model(**inputs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1532, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1541, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py\", line 822, in forward\n",
            "    return model_forward(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/accelerate/utils/operations.py\", line 810, in __call__\n",
            "    return convert_to_fp32(self.model_forward(*args, **kwargs))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/amp/autocast_mode.py\", line 16, in decorate_autocast\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/peft/peft_model.py\", line 1430, in forward\n",
            "    return self.base_model(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1532, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1541, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/peft/tuners/tuners_utils.py\", line 179, in forward\n",
            "    return self.model.forward(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/accelerate/hooks.py\", line 166, in new_forward\n",
            "    output = module._old_forward(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/models/llama/modeling_llama.py\", line 1164, in forward\n",
            "    outputs = self.model(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1532, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1541, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/accelerate/hooks.py\", line 166, in new_forward\n",
            "    output = module._old_forward(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/models/llama/modeling_llama.py\", line 957, in forward\n",
            "    layer_outputs = self._gradient_checkpointing_func(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/_compile.py\", line 24, in inner\n",
            "    return torch._dynamo.disable(fn, recursive)(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/_dynamo/eval_frame.py\", line 451, in _fn\n",
            "    return fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/_dynamo/external_utils.py\", line 36, in inner\n",
            "    return fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py\", line 487, in checkpoint\n",
            "    return CheckpointFunction.apply(function, preserve, *args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/autograd/function.py\", line 598, in apply\n",
            "    return super().apply(*args, **kwargs)  # type: ignore[misc]\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/utils/checkpoint.py\", line 262, in forward\n",
            "    outputs = run_function(*args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1532, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1541, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/accelerate/hooks.py\", line 166, in new_forward\n",
            "    output = module._old_forward(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/models/llama/modeling_llama.py\", line 726, in forward\n",
            "    hidden_states = self.post_attention_layernorm(hidden_states)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1532, in _wrapped_call_impl\n",
            "    return self._call_impl(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\", line 1541, in _call_impl\n",
            "    return forward_call(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/accelerate/hooks.py\", line 166, in new_forward\n",
            "    output = module._old_forward(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/transformers/models/llama/modeling_llama.py\", line 88, in forward\n",
            "    hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)\n",
            "KeyboardInterrupt\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run history:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         train/epoch ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   train/global_step ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train/grad_norm ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          train/loss ▁\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Run summary:\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:         train/epoch 0.05\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:   train/global_step 5\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:     train/grad_norm 0.12219\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: train/learning_rate 0.0002\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m:          train/loss 0.827\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: 🚀 View run \u001b[33mtmp-codellama-7b-openapi-completion-ctx-lvl-fim-05-spm-2048\u001b[0m at: \u001b[34m\u001b[4mhttps://wandb.ai/bohdan-petryshyn/huggingface/runs/naaswf2d\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: ⭐️ View project at: \u001b[34m\u001b[4mhttps://wandb.ai/bohdan-petryshyn/huggingface\u001b[0m\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Synced 5 W&B file(s), 0 media file(s), 2 artifact file(s) and 0 other file(s)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Find logs at: \u001b[35m\u001b[1m./wandb/run-20240615_090439-naaswf2d/logs\u001b[0m\n",
            "^C\n"
          ]
        }
      ],
      "source": [
        "# Replace (at least) the following:\n",
        "#   - dataset_name - your own dataset\n",
        "#   - output_dir   - output model id in you Hugging Face account\n",
        "!python train.py \\\n",
        "  --model_name_or_path \"codellama/CodeLlama-7b-hf\" \\\n",
        "  --dataset_name \"BohdanPetryshyn/openapi-completion-refined\" \\\n",
        "  --splits \"train\" \\\n",
        "  --max_seq_len 2048 \\\n",
        "  --max_steps 100 \\\n",
        "  --save_steps 100 \\\n",
        "  --eval_steps 100 \\\n",
        "  --logging_steps 5 \\\n",
        "  --log_level \"info\" \\\n",
        "  --logging_strategy \"steps\" \\\n",
        "  --evaluation_strategy \"steps\" \\\n",
        "  --save_strategy \"steps\" \\\n",
        "  --push_to_hub \\\n",
        "  --hub_private_repo False \\\n",
        "  --hub_strategy \"every_save\" \\\n",
        "  --bf16 True \\\n",
        "  --learning_rate 2e-4 \\\n",
        "  --lr_scheduler_type \"cosine\" \\\n",
        "  --weight_decay 0.1 \\\n",
        "  --warmup_ratio 0.05 \\\n",
        "  --max_grad_norm 1.0 \\\n",
        "  --output_dir \"tmp-codellama-7b-openapi-completion-ctx-lvl-fim-05-spm-2048\" \\\n",
        "  --per_device_train_batch_size 4 \\\n",
        "  --per_device_eval_batch_size 4 \\\n",
        "  --gradient_accumulation_steps 8 \\\n",
        "  --gradient_checkpointing True \\\n",
        "  --use_reentrant True \\\n",
        "  --dataset_text_field \"content\" \\\n",
        "  --test_size 0.1 \\\n",
        "  --fim_rate 0.9 \\\n",
        "  --fim_spm_rate 0.5 \\\n",
        "  --use_peft_lora True \\\n",
        "  --lora_r 32 \\\n",
        "  --lora_alpha 64 \\\n",
        "  --lora_dropout 0.1 \\\n",
        "  --lora_target_modules \"all-linear\" \\\n",
        "  --use_4bit_quantization True \\\n",
        "  --use_nested_quant True \\\n",
        "  --bnb_4bit_compute_dtype \"bfloat16\" \\\n",
        "  --use_flash_attn True"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "A100",
      "machine_shape": "hm",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "7adad8f01b7148dab6f14e9c06a7e121": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "VBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a7c95591dba345b2b6721c751e06d256",
              "IPY_MODEL_d3cbd4816c73493fafb9223183d47e5f",
              "IPY_MODEL_337022a49d48474aad7b01563daf8836",
              "IPY_MODEL_49d145c662b54f7a9c134684fe6341cc"
            ],
            "layout": "IPY_MODEL_70fc60afbb954619b71f638372816b55"
          }
        },
        "fa303d8c55b24a749323bcead74f2b72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fe3cfefa2ee24778bcd814734c2159ea",
            "placeholder": "​",
            "style": "IPY_MODEL_a2a279eb7c594c18a2a3f701da8a7e77",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "d05dbe871c684a4d9ec6af70365d2d22": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "PasswordModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_b7af5f8b4acf4f64a3c835dca72a4ec0",
            "placeholder": "​",
            "style": "IPY_MODEL_bacbeba271d9431f9057bcb21c4f6807",
            "value": ""
          }
        },
        "7df5a4fe65f1485495b2b07e6ac9e63b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "CheckboxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_9b878252c2d64c688d60428aa0c5d1a3",
            "style": "IPY_MODEL_e1d13ba5edae407ea96d87df297a3b97",
            "value": true
          }
        },
        "ff62f0037d3047b3880043dd54f8f83c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_c361af9a7ae14786b93ff67dad41ebc4",
            "style": "IPY_MODEL_b4fb3e78617046e383e2b9b6772e60b4",
            "tooltip": ""
          }
        },
        "402aa7fab1054cc39e96f3f6874b1da1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cf93cfbe66514dc183c4918140c4a708",
            "placeholder": "​",
            "style": "IPY_MODEL_69aa14cac03149b398601a1a6e0e8ff1",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "70fc60afbb954619b71f638372816b55": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "fe3cfefa2ee24778bcd814734c2159ea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a2a279eb7c594c18a2a3f701da8a7e77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b7af5f8b4acf4f64a3c835dca72a4ec0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bacbeba271d9431f9057bcb21c4f6807": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9b878252c2d64c688d60428aa0c5d1a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e1d13ba5edae407ea96d87df297a3b97": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c361af9a7ae14786b93ff67dad41ebc4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b4fb3e78617046e383e2b9b6772e60b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ButtonStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "cf93cfbe66514dc183c4918140c4a708": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "69aa14cac03149b398601a1a6e0e8ff1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8f35f026c9e74e0c84a3fcf55d1c0f7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3e84f52b1e8141698466ea4bc6b76da8",
            "placeholder": "​",
            "style": "IPY_MODEL_5993de4ad5c840d1924cf862cffc2a34",
            "value": "Connecting..."
          }
        },
        "3e84f52b1e8141698466ea4bc6b76da8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5993de4ad5c840d1924cf862cffc2a34": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a7c95591dba345b2b6721c751e06d256": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0ca4996f60a04d61b4b3c142ad34a44b",
            "placeholder": "​",
            "style": "IPY_MODEL_421afb1fe86149c5b4bafeae55931802",
            "value": "Token is valid (permission: write)."
          }
        },
        "d3cbd4816c73493fafb9223183d47e5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f5b326935a2741e1894f13e8c8ee3bd0",
            "placeholder": "​",
            "style": "IPY_MODEL_0a45e68f1c234c8084e7d9935d113e06",
            "value": "Your token has been saved in your configured git credential helpers (store)."
          }
        },
        "337022a49d48474aad7b01563daf8836": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4ae883bfac18451cb205683db63046e7",
            "placeholder": "​",
            "style": "IPY_MODEL_69fe511511e0416e813e38e7aab8544b",
            "value": "Your token has been saved to /root/.cache/huggingface/token"
          }
        },
        "49d145c662b54f7a9c134684fe6341cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "LabelModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "LabelModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "LabelView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_795017bc9fbb4d2db79ff669cf1a8bfc",
            "placeholder": "​",
            "style": "IPY_MODEL_5306a61587704526bb59601b16559224",
            "value": "Login successful"
          }
        },
        "0ca4996f60a04d61b4b3c142ad34a44b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "421afb1fe86149c5b4bafeae55931802": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f5b326935a2741e1894f13e8c8ee3bd0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0a45e68f1c234c8084e7d9935d113e06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4ae883bfac18451cb205683db63046e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "69fe511511e0416e813e38e7aab8544b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "795017bc9fbb4d2db79ff669cf1a8bfc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5306a61587704526bb59601b16559224": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}