{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "460cw1_2020_my.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3.7.9 64-bit ('dl': conda)",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.9-final"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "918d066c9c564914868c2780db311b00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_33a7758e551e4666a0cff3546139c94b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_67b82551bc3945d99e066fd9592da766",
              "IPY_MODEL_60cfdfc6d30342ffaf43988d25a477ec"
            ]
          }
        },
        "33a7758e551e4666a0cff3546139c94b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "67b82551bc3945d99e066fd9592da766": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e11d183e733e4bccaf330a5f0454e5b6",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c7a2f4ea7dd24ecb8b0b40daea2cfc19"
          }
        },
        "60cfdfc6d30342ffaf43988d25a477ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_165e4959244d433caa6d104f8632b636",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 170500096/? [00:30&lt;00:00, 18010455.88it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_07119d1f416640429f315e6dc60e5de4"
          }
        },
        "e11d183e733e4bccaf330a5f0454e5b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c7a2f4ea7dd24ecb8b0b40daea2cfc19": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "165e4959244d433caa6d104f8632b636": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "07119d1f416640429f315e6dc60e5de4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HgGeQMItJMVo"
      },
      "source": [
        "# Coursework1: Convolutional Neural Networks "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Iz5hs-NtJMVq"
      },
      "source": [
        "## instructions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FF6miJ0pJMVr"
      },
      "source": [
        "Please submit a version of this notebook containing your answers **together with your trained model** on CATe as CW2.zip. Write your answers in the cells below each question."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OTd4PWOJJMVr"
      },
      "source": [
        "### Setting up working environment "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aawBRqNYJMVs"
      },
      "source": [
        "For this coursework you will need to train a large network, therefore we recommend you work with Google Colaboratory, which provides free GPU time. You will need a Google account to do so. \n",
        "\n",
        "Please log in to your account and go to the following page: https://colab.research.google.com. Then upload this notebook.\n",
        "\n",
        "For GPU support, go to \"Edit\" -> \"Notebook Settings\", and select \"Hardware accelerator\" as \"GPU\".\n",
        "\n",
        "You will need to install pytorch by running the following cell:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "99KzLz6ZJMVs",
        "outputId": "168b2724-f1a7-42d7-ce15-3ffa8c6c3082"
      },
      "source": [
        "!pip install torch torchvision"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: torch in c:\\users\\yunmao\\.conda\\envs\\ml\\lib\\site-packages (1.7.1)\n",
            "Requirement already satisfied: torchvision in c:\\users\\yunmao\\.conda\\envs\\ml\\lib\\site-packages (0.8.2)\n",
            "Requirement already satisfied: typing-extensions in c:\\users\\yunmao\\.conda\\envs\\ml\\lib\\site-packages (from torch) (3.7.4.3)\n",
            "Requirement already satisfied: numpy in c:\\users\\yunmao\\.conda\\envs\\ml\\lib\\site-packages (from torch) (1.19.5)\n",
            "Requirement already satisfied: pillow>=4.1.1 in c:\\users\\yunmao\\.conda\\envs\\ml\\lib\\site-packages (from torchvision) (8.1.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PzhJatU5JMVt"
      },
      "source": [
        "## Introduction"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m3XhBM-YJMVt"
      },
      "source": [
        "For this coursework you will implement one of the most commonly used model for image recognition tasks, the Residual Network. The architecture is introduced in 2015 by Kaiming He, et al. in the paper [\"Deep residual learning for image recognition\"](https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/He_Deep_Residual_Learning_CVPR_2016_paper.pdf). \n",
        "<br>\n",
        "\n",
        "In a residual network, each block contains some convolutional layers, plus \"skip\" connections, which allow the activations to by pass a layer, and then be summed up with the activations of the skipped layer. The image below illustrates a building block in residual networks."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mgFPNJz4JMVt"
      },
      "source": [
        "![resnet-block](utils/resnet-block.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nsER7qvIJMVt"
      },
      "source": [
        "Depending on the number of building blocks, resnets can have different architectures, for example ResNet-50, ResNet-101 and etc. Here you are required to build ResNet-18 to perform classification on the CIFAR-10 dataset, therefore your network will have the following architecture:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TjqXBo_KJMVu"
      },
      "source": [
        "![resnet](utils/resnet.png)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ellLCBc6JMVu"
      },
      "source": [
        "## Part 1 (40 points)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4BBu-gVCJMVu"
      },
      "source": [
        "In this part, you will use basic pytorch operations to define the 2D convolution, max pooling operation, linear layer as well as 2d batch normalization. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "60FUTQkaJMVu"
      },
      "source": [
        "### YOUR TASK"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "quBpjSufJMVu"
      },
      "source": [
        "- implement the forward pass for Conv2D, MaxPool2D, Linear and BatchNorm2d\n",
        "- You are **NOT** allowed to use the torch.nn modules"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "88OnDTEcJMVv"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "\n",
        "class Conv2d(nn.Module):\n",
        "    def __init__(self,\n",
        "                 in_channels,\n",
        "                 out_channels,\n",
        "                 kernel_size,\n",
        "                 stride=1,\n",
        "                 padding=0,\n",
        "                 bias=True):\n",
        "\n",
        "        super(Conv2d, self).__init__()\n",
        "        \"\"\"\n",
        "        An implementation of a convolutional layer.\n",
        "\n",
        "        The input consists of N data points, each with C channels, height H and\n",
        "        width W. We convolve each input with F different filters, where each filter\n",
        "        spans all C channels and has height HH and width WW.\n",
        "\n",
        "        Parameters:\n",
        "        - w: Filter weights of shape (F, C, HH, WW)\n",
        "        - b: Biases, of shape (F,)\n",
        "        - kernel_size: Size of the convolving kernel\n",
        "        - stride: The number of pixels between adjacent receptive fields in the\n",
        "            horizontal and vertical directions.\n",
        "        - padding: The number of pixels that will be used to zero-pad the input.\n",
        "        \"\"\"\n",
        "\n",
        "        ########################################################################\n",
        "        # TODO: Define the parameters used in the forward pass                 #\n",
        "        ########################################################################\n",
        "        # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "\n",
        "        self.in_channels = in_channels\n",
        "        self.out_channels = out_channels\n",
        "        self.stride = stride\n",
        "        self.padding = padding\n",
        "\n",
        "        if type(kernel_size) is tuple :\n",
        "            self.kernel_size = kernel_size\n",
        "        else:\n",
        "            self.kernel_size = (kernel_size, kernel_size)\n",
        "\n",
        "        # Good practice is to start your weights in the range of [-y, y] where y=1/sqrt(n)  (n is the number of inputs to a given neuron).\n",
        "        self.w = torch.randn((out_channels, in_channels, self.kernel_size[0], self.kernel_size[1]), requires_grad = True) * torch.sqrt(torch.tensor(1.0/in_channels))\n",
        "\n",
        "        if bias:\n",
        "            self.b = torch.randn(out_channels, requires_grad = True)  * torch.sqrt(torch.tensor(1.0/in_channels))\n",
        "        else:\n",
        "            self.b = None\n",
        "\n",
        "        self.bias = bias\n",
        "\n",
        "        # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "        ########################################################################\n",
        "        #                             END OF YOUR CODE                         #\n",
        "        ########################################################################\n",
        "\n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        Input:\n",
        "        - x: Input data of shape (N, C, H, W)\n",
        "        Output:\n",
        "        - out: Output data, of shape (N, F, H', W').\n",
        "        \"\"\"\n",
        "\n",
        "        ########################################################################\n",
        "        # TODO: Implement the forward pass                                     #\n",
        "        ########################################################################\n",
        "        # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "\n",
        "           #Calculate the output shape\n",
        "        out_H = (x.shape[2] + 2*self.padding - self.kernel_size[0]) // self.stride + 1\n",
        "        out_W = (x.shape[3] + 2*self.padding - self.kernel_size[1]) // self.stride + 1\n",
        "\n",
        "        x_unfold = F.unfold(x, kernel_size = self.kernel_size, padding = self.padding, stride = self.stride)\n",
        "\n",
        "        if self.bias:\n",
        "            out_unfold = x_unfold.permute(0, 2, 1).matmul(self.w.view(self.w.size(0), -1).t()).permute(0, 2, 1) + self.b.view(-1, 1)\n",
        "        else:\n",
        "            out_unfold = x_unfold.permute(0, 2, 1).matmul(self.w.view(self.w.size(0), -1).t()).permute(0, 2, 1)\n",
        "        out = out_unfold.view(x.shape[0], self.out_channels, out_H, out_W)\n",
        "\n",
        "        # print(nn.functional.conv2d(x, self.w, self.b, padding=1))\n",
        "\n",
        "        # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "        ########################################################################\n",
        "        #                             END OF YOUR CODE                         #\n",
        "        ########################################################################\n",
        "\n",
        "        return out\n",
        "# inputs = torch.rand(3, 3, 24, 24)\n",
        "# conv2 = Conv2d(in_channels=3, out_channels=3, kernel_size=(3, 3),stride=1, padding=1)\n",
        "# out2 = conv2.forward(inputs)\n",
        "# print(out2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cGf6Da9iJMVv"
      },
      "source": [
        "class MaxPool2d(nn.Module):\n",
        "    def __init__(self, kernel_size):\n",
        "        super(MaxPool2d, self).__init__()\n",
        "        \"\"\"\n",
        "        An implementation of a max-pooling layer.\n",
        "\n",
        "        Parameters:\n",
        "        - kernel_size: the size of the window to take a max over\n",
        "        \"\"\"\n",
        "        ########################################################################\n",
        "        # TODO: Define the parameters used in the forward pass                 #\n",
        "        ########################################################################\n",
        "        # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "\n",
        "        if type(kernel_size) is tuple :\n",
        "            self.kernel_size = kernel_size\n",
        "        else:\n",
        "            self.kernel_size = (kernel_size, kernel_size)\n",
        "\n",
        "        # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "        ########################################################################\n",
        "        #                             END OF YOUR CODE                         #\n",
        "        ########################################################################\n",
        "\n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        Input:\n",
        "        - x: Input data of shape (N, C, H, W)\n",
        "        Output:\n",
        "        - out: Output data, of shape (N, F, H', W').\n",
        "        \"\"\"\n",
        "        ########################################################################\n",
        "        # TODO: Implement the forward pass                                     #\n",
        "        ########################################################################\n",
        "        # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "\n",
        "        H_k, W_k = self.kernel_size\n",
        "\n",
        "        x_unfold = F.unfold(x, kernel_size = self.kernel_size, stride = self.kernel_size)\n",
        "\n",
        "        x_reshape = x_unfold.view(x.shape[0], x.shape[1], self.kernel_size[0] * self.kernel_size[1], -1)\n",
        "\n",
        "        x_max = x_reshape.max(axis = 2)[0]\n",
        "\n",
        "        H_out = x.shape[2] // H_k\n",
        "        W_out = x.shape[3] // W_k\n",
        "\n",
        "        out = x_max.view(x.shape[0], x.shape[1], H_out, W_out)\n",
        "        # print(out == nn.functional.max_pool2d(x, kernel_size = self.kernel_size))\n",
        "        # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "        ########################################################################\n",
        "        #                             END OF YOUR CODE                         #\n",
        "        ########################################################################\n",
        "\n",
        "        return out\n",
        "# inputs = torch.rand(2, 2, 3, 4)\n",
        "# maxpool = MaxPool2d(kernel_size=2)\n",
        "# print(maxpool.forward(inputs).shape)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aIR9IY5iJMVw"
      },
      "source": [
        "class Linear(nn.Module):\n",
        "    def __init__(self, in_channels, out_channels, bias=True):\n",
        "        super(Linear, self).__init__()\n",
        "        \"\"\"\n",
        "        An implementation of a Linear layer.\n",
        "\n",
        "        Parameters:\n",
        "        - weight: the learnable weights of the module of shape (in_channels, out_channels).\n",
        "        - bias: the learnable bias of the module of shape (out_channels).\n",
        "        \"\"\"\n",
        "        ########################################################################\n",
        "        # TODO: Define the parameters used in the forward pass                 #\n",
        "        ########################################################################\n",
        "        # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "\n",
        "        self.weight = torch.randn((in_channels, out_channels), requires_grad = True) * torch.sqrt(torch.tensor(1.0/in_channels))\n",
        "\n",
        "        if bias:\n",
        "            self.bias = torch.randn(out_channels, requires_grad = True)  * torch.sqrt(torch.tensor(1.0/in_channels))\n",
        "        else:\n",
        "            self.bias = None\n",
        "\n",
        "        # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "        ########################################################################\n",
        "        #                             END OF YOUR CODE                         #\n",
        "        ########################################################################\n",
        "\n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        Input:\n",
        "        - x: Input data of shape (N, *, H) where * means any number of additional\n",
        "        dimensions and H = in_channels\n",
        "        Output:\n",
        "        - out: Output data of shape (N, *, H') where * means any number of additional\n",
        "        dimensions and H' = out_channels\n",
        "        \"\"\"\n",
        "        ########################################################################\n",
        "        # TODO: Implement the forward pass                                     #\n",
        "        ########################################################################\n",
        "        # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "\n",
        "        if self.bias != None:\n",
        "            out = torch.matmul(x, self.weight) + self.bias\n",
        "        else:\n",
        "            out = torch.matmul(x, self.weight)\n",
        "        # nn.functional.linear(x,self.weight, self.bias)\n",
        "        # print(out == nn.functional.linear(x,self.weight.t(), self.bias))\n",
        "        # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "        ########################################################################\n",
        "        #                             END OF YOUR CODE                         #\n",
        "        ########################################################################\n",
        "\n",
        "        return out\n",
        "# inputs = torch.rand(3, 3, 23, 23)\n",
        "# linear = Linear(in_channels=23, out_channels = 4)\n",
        "# out2 = linear.forward(inputs)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qMpvG3jBJMVx"
      },
      "source": [
        "class BatchNorm2d(nn.Module):\n",
        "    def __init__(self, num_features, eps=1e-05, momentum=0.1):\n",
        "        super(BatchNorm2d, self).__init__()\n",
        "        \"\"\"\n",
        "        An implementation of a Batch Normalization over a mini-batch of 2D inputs.\n",
        "\n",
        "        The mean and standard-deviation are calculated per-dimension over the\n",
        "        mini-batches and gamma and beta are learnable parameter vectors of\n",
        "        size num_features.\n",
        "\n",
        "        Parameters:\n",
        "        - num_features: C from an expected input of size (N, C, H, W).\n",
        "        - eps: a value added to the denominator for numerical stability. Default: 1e-5\n",
        "        - momentum: momentum – the value used for the running_mean and running_var\n",
        "        computation. Default: 0.1\n",
        "        - gamma: the learnable weights of shape (num_features).\n",
        "        - beta: the learnable bias of the module of shape (num_features).\n",
        "        \"\"\"\n",
        "        ########################################################################\n",
        "        # TODO: Define the parameters used in the forward pass                 #\n",
        "        ########################################################################\n",
        "        # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "\n",
        "        self.num_features = num_features\n",
        "        self.eps = eps\n",
        "        self.momentum = momentum\n",
        "        self.gamma = torch.ones(num_features, requires_grad = True)\n",
        "        self.beta = torch.zeros(num_features, requires_grad = True)\n",
        "        self.running_mean = torch.zeros(num_features)\n",
        "        self.running_var = torch.ones(num_features)\n",
        "\n",
        "\n",
        "        # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "        ########################################################################\n",
        "        #                             END OF YOUR CODE                         #\n",
        "        ########################################################################\n",
        "\n",
        "    def forward(self, x):\n",
        "        \"\"\"\n",
        "        During training this layer keeps running estimates of its computed mean and\n",
        "        variance, which are then used for normalization during evaluation.\n",
        "        Input:\n",
        "        - x: Input data of shape (N, C, H, W)\n",
        "        Output:\n",
        "        - out: Output data of shape (N, C, H, W) (same shape as input)\n",
        "        \"\"\"\n",
        "        ########################################################################\n",
        "        # TODO: Implement the forward pass                                     #\n",
        "        #       (be aware of the difference for training and testing)          #\n",
        "        ########################################################################\n",
        "        # *****START OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "        mean = x.mean([0, 2, 3])\n",
        "        var = x.var([0, 2, 3])\n",
        "        self.running_mean = (1 - self.momentum) * self.running_mean + self.momentum * mean\n",
        "        running_var = (1 - self.momentum) * self.running_var + self.momentum * var\n",
        "        var = x.var([0,2,3], unbiased = False)\n",
        "        x = (x - mean.view(1, -1, 1, 1)) / torch.sqrt(var + self.eps).view(1, -1, 1, 1) * self.gamma.view(1, -1, 1, 1) + self.beta.view(1, -1, 1, 1)\n",
        "\n",
        "        # nn.functional.BatchNorm2d(x, self.num_features)\n",
        "        # print(x)\n",
        "        # *****END OF YOUR CODE (DO NOT DELETE/MODIFY THIS LINE)*****\n",
        "        ########################################################################\n",
        "        #                             END OF YOUR CODE                         #\n",
        "        ########################################################################\n",
        "\n",
        "        return x\n",
        "# inputs = torch.rand(3, 3, 23, 23)\n",
        "# batch = BatchNorm2d(3)\n",
        "# test = nn.BatchNorm2d(3)\n",
        "# out = test.forward(inputs)\n",
        "# out2 = batch.forward(inputs)\n",
        "# print(test.running_mean)\n",
        "# print(batch.running_mean)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nKR54x28JMVy"
      },
      "source": [
        "## Part 2"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DQmghXtJJMVy"
      },
      "source": [
        "In this part, you will train a ResNet-18 defined on the CIFAR-10 dataset. Code for training and evaluation are provided. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LSRwh-ASJMVz"
      },
      "source": [
        "### Your Task"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SVijmjCLJMVz"
      },
      "source": [
        "1. Train your network to achieve the best possible test set accuracy after a maximum of 10 epochs of training.\n",
        "\n",
        "2. You can use techniques such as optimal hyper-parameter searching, data pre-processing\n",
        "\n",
        "3. If necessary, you can also use another optimizer\n",
        "\n",
        "4. **Answer the following question:**\n",
        "Given such a network with a large number of trainable parameters, and a training set of a large number of data, what do you think is the best strategy for hyperparameter searching? "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aPWE0kvLJMVz"
      },
      "source": [
        "import torch\n",
        "from torch.nn import Conv2d, MaxPool2d\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BawVv_AcJMVz"
      },
      "source": [
        "Next, we define ResNet-18:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rn847LqAJMVz"
      },
      "source": [
        "# define resnet building blocks\n",
        "\n",
        "class ResidualBlock(nn.Module): \n",
        "    def __init__(self, inchannel, outchannel, stride=1): \n",
        "        \n",
        "        super(ResidualBlock, self).__init__() \n",
        "        \n",
        "        self.left = nn.Sequential(Conv2d(inchannel, outchannel, kernel_size=3, \n",
        "                                         stride=stride, padding=1, bias=False), \n",
        "                                  nn.BatchNorm2d(outchannel), \n",
        "                                  nn.ReLU(inplace=True), \n",
        "                                  Conv2d(outchannel, outchannel, kernel_size=3, \n",
        "                                         stride=1, padding=1, bias=False), \n",
        "                                  nn.BatchNorm2d(outchannel)) \n",
        "        \n",
        "        self.shortcut = nn.Sequential() \n",
        "        \n",
        "        if stride != 1 or inchannel != outchannel: \n",
        "            \n",
        "            self.shortcut = nn.Sequential(Conv2d(inchannel, outchannel, \n",
        "                                                 kernel_size=1, stride=stride, \n",
        "                                                 padding = 0, bias=False), \n",
        "                                          nn.BatchNorm2d(outchannel) ) \n",
        "            \n",
        "    def forward(self, x): \n",
        "        \n",
        "        out = self.left(x) \n",
        "        \n",
        "        out += self.shortcut(x) \n",
        "        \n",
        "        out = F.relu(out) \n",
        "        \n",
        "        return out\n",
        "\n",
        "\n",
        "    \n",
        "    # define resnet\n",
        "\n",
        "class ResNet(nn.Module):\n",
        "    \n",
        "    def __init__(self, ResidualBlock, num_classes = 10):\n",
        "        \n",
        "        super(ResNet, self).__init__()\n",
        "        \n",
        "        self.inchannel = 64\n",
        "        self.conv1 = nn.Sequential(Conv2d(3, 64, kernel_size = 3, stride = 1,\n",
        "                                            padding = 1, bias = False), \n",
        "                                  nn.BatchNorm2d(64), \n",
        "                                  nn.ReLU())\n",
        "        \n",
        "        self.layer1 = self.make_layer(ResidualBlock, 64, 2, stride = 1)\n",
        "        self.layer2 = self.make_layer(ResidualBlock, 128, 2, stride = 2)\n",
        "        self.layer3 = self.make_layer(ResidualBlock, 256, 2, stride = 2)\n",
        "        self.layer4 = self.make_layer(ResidualBlock, 512, 2, stride = 2)\n",
        "        self.maxpool = MaxPool2d(4)\n",
        "        self.fc = nn.Linear(512, num_classes)\n",
        "        \n",
        "    \n",
        "    def make_layer(self, block, channels, num_blocks, stride):\n",
        "        \n",
        "        strides = [stride] + [1] * (num_blocks - 1)\n",
        "        \n",
        "        layers = []\n",
        "        \n",
        "        for stride in strides:\n",
        "            \n",
        "            layers.append(block(self.inchannel, channels, stride))\n",
        "            \n",
        "            self.inchannel = channels\n",
        "            \n",
        "        return nn.Sequential(*layers)\n",
        "    \n",
        "    \n",
        "    def forward(self, x):\n",
        "        \n",
        "        x = self.conv1(x)\n",
        "        \n",
        "        x = self.layer1(x)\n",
        "        x = self.layer2(x)\n",
        "        x = self.layer3(x)\n",
        "        x = self.layer4(x)\n",
        "        \n",
        "        x = self.maxpool(x)\n",
        "        \n",
        "        x = x.view(x.size(0), -1)\n",
        "        \n",
        "        x = self.fc(x)\n",
        "        \n",
        "        return x\n",
        "    \n",
        "    \n",
        "def ResNet18():\n",
        "    return ResNet(ResidualBlock)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a5mpaI_FJMVz"
      },
      "source": [
        "### Loading dataset\n",
        "We will import images from the [torchvision.datasets](https://pytorch.org/docs/stable/torchvision/datasets.html) library <br>\n",
        "First, we need to define the alterations (transforms) we want to perform to our images - given that transformations are applied when importing the data. <br>\n",
        "Define the following transforms using the torchvision.datasets library -- you can read the transforms documentation [here](https://pytorch.org/docs/stable/torchvision/transforms.html): <br>\n",
        "1. Convert images to tensor\n",
        "2. Normalize mean and std of images with values:mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.2010]"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IVd9nxxcJMV0"
      },
      "source": [
        "import torch.optim as optim\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.utils.data import sampler\n",
        "\n",
        "import torchvision.datasets as dset\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "import torchvision.transforms as T\n",
        "\n",
        "##############################################################\n",
        "#                       YOUR CODE HERE                       #       \n",
        "##############################################################\n",
        "\n",
        "transform_aug = T.Compose([\n",
        "    T.RandomRotation(10),\n",
        "    T.RandomCrop(size=32, padding=4),\n",
        "    T.RandomHorizontalFlip(),\n",
        "    T.ToTensor(),\n",
        "    T.Normalize(mean = [0.4914, 0.4822, 0.4465], std = [0.2023, 0.1994, 0.2010])\n",
        "])\n",
        "\n",
        "transform = T.Compose([\n",
        "    T.ToTensor(),\n",
        "    T.Normalize(mean = [0.4914, 0.4822, 0.4465], std = [0.2023, 0.1994, 0.2010])\n",
        "])\n",
        "\n",
        "##############################################################\n",
        "#                       END OF YOUR CODE                     #\n",
        "##############################################################\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3ZT5uKAKJMV0"
      },
      "source": [
        "Now load the dataset using the transform you defined above, with batch_size = 64<br>\n",
        "You can check the documentation [here](https://pytorch.org/docs/stable/torchvision/datasets.html).\n",
        "Then create data loaders (using DataLoader from torch.utils.data) for the training and test set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 116,
          "referenced_widgets": [
            "918d066c9c564914868c2780db311b00",
            "33a7758e551e4666a0cff3546139c94b",
            "67b82551bc3945d99e066fd9592da766",
            "60cfdfc6d30342ffaf43988d25a477ec",
            "e11d183e733e4bccaf330a5f0454e5b6",
            "c7a2f4ea7dd24ecb8b0b40daea2cfc19",
            "165e4959244d433caa6d104f8632b636",
            "07119d1f416640429f315e6dc60e5de4"
          ]
        },
        "id": "LgUDdV7vJMV0",
        "outputId": "3df89f76-f02a-4ea6-aa38-067c13deec3a"
      },
      "source": [
        "\n",
        "##############################################################\n",
        "#                       YOUR CODE HERE                       #       \n",
        "##############################################################\n",
        "\n",
        "data_dir = './data'\n",
        "\n",
        "cifar10_train_data = dset.CIFAR10(data_dir, train = True, download = True, transform = transform_aug)\n",
        "loader_train = DataLoader(cifar10_train_data, batch_size=64, sampler=sampler.SubsetRandomSampler(range(49000)))\n",
        "\n",
        "cifar10_val_data = dset.CIFAR10(data_dir, train = True, download = True, transform = transform)\n",
        "loader_val = DataLoader(cifar10_val_data, batch_size=64, sampler=sampler.SubsetRandomSampler(range(49000, 50000)))\n",
        "\n",
        "cifar10_test_data = dset.CIFAR10(data_dir, train = False, download = True, transform = transform)\n",
        "loader_test = DataLoader(cifar10_test_data, batch_size=64)\n",
        "##############################################################\n",
        "#                       END OF YOUR CODE                     #       \n",
        "##############################################################\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./data/cifar-10-python.tar.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "918d066c9c564914868c2780db311b00",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting ./data/cifar-10-python.tar.gz to ./data\n",
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BkaYT4ahJMV0",
        "outputId": "3f5ecb71-9f6a-4aee-d92a-f2b16cdf8fee"
      },
      "source": [
        "USE_GPU = True\n",
        "dtype = torch.float32 \n",
        "print(torch.cuda.is_available())\n",
        "if USE_GPU and torch.cuda.is_available():\n",
        "    device = torch.device('cuda')\n",
        "else:\n",
        "    device = torch.device('cpu')\n",
        "    \n",
        "    \n",
        "\n",
        "print_every = 100\n",
        "def check_accuracy(loader, model):\n",
        "    # function for test accuracy on validation and test set\n",
        "    \n",
        "    if loader.dataset.train:\n",
        "        print('Checking accuracy on validation set')\n",
        "    else:\n",
        "        print('Checking accuracy on test set')   \n",
        "    num_correct = 0\n",
        "    num_samples = 0\n",
        "    model.eval()  # set model to evaluation mode\n",
        "    with torch.no_grad():\n",
        "        for x, y in loader:\n",
        "            x = x.to(device=device, dtype=dtype)  # move to device\n",
        "            y = y.to(device=device, dtype=torch.long)\n",
        "            scores = model(x)\n",
        "            _, preds = scores.max(1)\n",
        "            num_correct += (preds == y).sum()\n",
        "            num_samples += preds.size(0)\n",
        "        acc = float(num_correct) / num_samples\n",
        "        print('Got %d / %d correct (%.2f)' % (num_correct, num_samples, 100 * acc))\n",
        "    return acc\n",
        "        \n",
        "\n",
        "def train_part(model, optimizer, epochs=1):\n",
        "    \"\"\"\n",
        "    Train a model on CIFAR-10 using the PyTorch Module API.\n",
        "    \n",
        "    Inputs:\n",
        "    - model: A PyTorch Module giving the model to train.\n",
        "    - optimizer: An Optimizer object we will use to train the model\n",
        "    - epochs: (Optional) A Python integer giving the number of epochs to train for\n",
        "    \n",
        "    Returns: Nothing, but prints model accuracies during training.\n",
        "    \"\"\"\n",
        "    model = model.to(device=device)  # move the model parameters to CPU/GPU\n",
        "    for e in range(epochs):\n",
        "        print(len(loader_train))\n",
        "        for t, (x, y) in enumerate(loader_train):\n",
        "            model.train()  # put model to training mode\n",
        "            x = x.to(device=device, dtype=dtype)  # move to device, e.g. GPU\n",
        "            y = y.to(device=device, dtype=torch.long)\n",
        "\n",
        "            scores = model(x)\n",
        "            loss = F.cross_entropy(scores, y)\n",
        "\n",
        "            # Zero out all of the gradients for the variables which the optimizer\n",
        "            # will update.\n",
        "            optimizer.zero_grad()\n",
        "\n",
        "            loss.backward()\n",
        "\n",
        "            # Update the parameters of the model using the gradients\n",
        "            optimizer.step()\n",
        "\n",
        "            if t % print_every == 0:\n",
        "                print('Epoch: %d, Iteration %d, loss = %.4f' % (e, t, loss.item()))\n",
        "                # check_accuracy(loader_val, model)\n",
        "                print()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "True\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2RzkDweiJMV0",
        "outputId": "125ca379-ac2c-4adc-c9dd-df67153f73c4"
      },
      "source": [
        "# code for optimising your network performance\n",
        "\n",
        "##############################################################\n",
        "#                       YOUR CODE HERE                       #       \n",
        "##############################################################\n",
        "\n",
        "!pip install GPyopt"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: GPyopt in /usr/local/lib/python3.6/dist-packages (1.2.6)\n",
            "Requirement already satisfied: scipy>=0.16 in /usr/local/lib/python3.6/dist-packages (from GPyopt) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.7 in /usr/local/lib/python3.6/dist-packages (from GPyopt) (1.19.5)\n",
            "Requirement already satisfied: GPy>=1.8 in /usr/local/lib/python3.6/dist-packages (from GPyopt) (1.9.9)\n",
            "Requirement already satisfied: paramz>=0.9.0 in /usr/local/lib/python3.6/dist-packages (from GPy>=1.8->GPyopt) (0.9.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from GPy>=1.8->GPyopt) (1.15.0)\n",
            "Requirement already satisfied: decorator>=4.0.10 in /usr/local/lib/python3.6/dist-packages (from paramz>=0.9.0->GPy>=1.8->GPyopt) (4.4.2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4-FQLRSbJMV1",
        "outputId": "7d8af593-7a22-4b41-c472-f504fab4a566"
      },
      "source": [
        "import GPyOpt\n",
        "def Bayes_Opt(parameters):\n",
        "    lr = parameters[0, 0]\n",
        "    weight_decay = parameters[0, 1]\n",
        "    print(\"---- New lr = %.5f and weight_decay = %.5f ----\"% (lr, weight_decay))\n",
        "    model = ResNet18()\n",
        "    optimizer = optim.Adam(model.parameters(),lr = lr, weight_decay = weight_decay)\n",
        "    train_part(model, optimizer, epochs = 10)\n",
        "    return check_accuracy(loader_val, model)\n",
        "\n",
        "domain = [  {'name' : 'lr', 'type': 'continuous', 'domain': (0.0005, 0.005)},\n",
        "        {'name': 'weight_decay', 'type': 'continuous', 'domain': (0.0001, 0.001)},  ]\n",
        "opt = GPyOpt.methods.BayesianOptimization(f = Bayes_Opt,domain = domain,acquisition_type ='LCB',acquisition_weight = 0.1,maximize = True)\n",
        "\n",
        "\n",
        "opt.run_optimization( max_iter = 10)\n",
        "##############################################################\n",
        "#                       END OF YOUR CODE                     #\n",
        "##############################################################\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "---- New lr = 0.00212 and weight_decay = 0.00013 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 3.6991\n",
            "Checking accuracy on validation set\n",
            "Got 107 / 1000 correct (10.70)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 1.6612\n",
            "Checking accuracy on validation set\n",
            "Got 313 / 1000 correct (31.30)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.6770\n",
            "Checking accuracy on validation set\n",
            "Got 340 / 1000 correct (34.00)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.5844\n",
            "Checking accuracy on validation set\n",
            "Got 384 / 1000 correct (38.40)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.7507\n",
            "Checking accuracy on validation set\n",
            "Got 372 / 1000 correct (37.20)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.5577\n",
            "Checking accuracy on validation set\n",
            "Got 471 / 1000 correct (47.10)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.6247\n",
            "Checking accuracy on validation set\n",
            "Got 422 / 1000 correct (42.20)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 1.4275\n",
            "Checking accuracy on validation set\n",
            "Got 500 / 1000 correct (50.00)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 1.5315\n",
            "Checking accuracy on validation set\n",
            "Got 538 / 1000 correct (53.80)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 1.4147\n",
            "Checking accuracy on validation set\n",
            "Got 585 / 1000 correct (58.50)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 1.1238\n",
            "Checking accuracy on validation set\n",
            "Got 572 / 1000 correct (57.20)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 1.2450\n",
            "Checking accuracy on validation set\n",
            "Got 555 / 1000 correct (55.50)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 1.0476\n",
            "Checking accuracy on validation set\n",
            "Got 580 / 1000 correct (58.00)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 0.9857\n",
            "Checking accuracy on validation set\n",
            "Got 611 / 1000 correct (61.10)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 1.3515\n",
            "Checking accuracy on validation set\n",
            "Got 598 / 1000 correct (59.80)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 1.0223\n",
            "Checking accuracy on validation set\n",
            "Got 556 / 1000 correct (55.60)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 0.9896\n",
            "Checking accuracy on validation set\n",
            "Got 617 / 1000 correct (61.70)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 0.8825\n",
            "Checking accuracy on validation set\n",
            "Got 621 / 1000 correct (62.10)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 0.9283\n",
            "Checking accuracy on validation set\n",
            "Got 661 / 1000 correct (66.10)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 0.8696\n",
            "Checking accuracy on validation set\n",
            "Got 652 / 1000 correct (65.20)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 0.9148\n",
            "Checking accuracy on validation set\n",
            "Got 624 / 1000 correct (62.40)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 0.8600\n",
            "Checking accuracy on validation set\n",
            "Got 668 / 1000 correct (66.80)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 0.9862\n",
            "Checking accuracy on validation set\n",
            "Got 686 / 1000 correct (68.60)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 1.1166\n",
            "Checking accuracy on validation set\n",
            "Got 674 / 1000 correct (67.40)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 1.0123\n",
            "Checking accuracy on validation set\n",
            "Got 678 / 1000 correct (67.80)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 0.8012\n",
            "Checking accuracy on validation set\n",
            "Got 673 / 1000 correct (67.30)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 1.0722\n",
            "Checking accuracy on validation set\n",
            "Got 708 / 1000 correct (70.80)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 1.1618\n",
            "Checking accuracy on validation set\n",
            "Got 703 / 1000 correct (70.30)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.6213\n",
            "Checking accuracy on validation set\n",
            "Got 710 / 1000 correct (71.00)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.7957\n",
            "Checking accuracy on validation set\n",
            "Got 725 / 1000 correct (72.50)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.9643\n",
            "Checking accuracy on validation set\n",
            "Got 702 / 1000 correct (70.20)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 1.0311\n",
            "Checking accuracy on validation set\n",
            "Got 709 / 1000 correct (70.90)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.8642\n",
            "Checking accuracy on validation set\n",
            "Got 711 / 1000 correct (71.10)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 1.0074\n",
            "Checking accuracy on validation set\n",
            "Got 729 / 1000 correct (72.90)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.8587\n",
            "Checking accuracy on validation set\n",
            "Got 743 / 1000 correct (74.30)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.6553\n",
            "Checking accuracy on validation set\n",
            "Got 754 / 1000 correct (75.40)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 0.9331\n",
            "Checking accuracy on validation set\n",
            "Got 768 / 1000 correct (76.80)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 0.6007\n",
            "Checking accuracy on validation set\n",
            "Got 761 / 1000 correct (76.10)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.9240\n",
            "Checking accuracy on validation set\n",
            "Got 728 / 1000 correct (72.80)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.6366\n",
            "Checking accuracy on validation set\n",
            "Got 774 / 1000 correct (77.40)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.6410\n",
            "Checking accuracy on validation set\n",
            "Got 770 / 1000 correct (77.00)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.8375\n",
            "Checking accuracy on validation set\n",
            "Got 766 / 1000 correct (76.60)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.6668\n",
            "Checking accuracy on validation set\n",
            "Got 758 / 1000 correct (75.80)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.7622\n",
            "Checking accuracy on validation set\n",
            "Got 741 / 1000 correct (74.10)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 1.1154\n",
            "Checking accuracy on validation set\n",
            "Got 763 / 1000 correct (76.30)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.6868\n",
            "Checking accuracy on validation set\n",
            "Got 756 / 1000 correct (75.60)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.7558\n",
            "Checking accuracy on validation set\n",
            "Got 808 / 1000 correct (80.80)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.5453\n",
            "Checking accuracy on validation set\n",
            "Got 778 / 1000 correct (77.80)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.6800\n",
            "Checking accuracy on validation set\n",
            "Got 776 / 1000 correct (77.60)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.7670\n",
            "Checking accuracy on validation set\n",
            "Got 736 / 1000 correct (73.60)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.4405\n",
            "Checking accuracy on validation set\n",
            "Got 813 / 1000 correct (81.30)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.9417\n",
            "Checking accuracy on validation set\n",
            "Got 762 / 1000 correct (76.20)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.8832\n",
            "Checking accuracy on validation set\n",
            "Got 756 / 1000 correct (75.60)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 1.0137\n",
            "Checking accuracy on validation set\n",
            "Got 790 / 1000 correct (79.00)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.6195\n",
            "Checking accuracy on validation set\n",
            "Got 793 / 1000 correct (79.30)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.6126\n",
            "Checking accuracy on validation set\n",
            "Got 802 / 1000 correct (80.20)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.5029\n",
            "Checking accuracy on validation set\n",
            "Got 815 / 1000 correct (81.50)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.6902\n",
            "Checking accuracy on validation set\n",
            "Got 803 / 1000 correct (80.30)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.7581\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.6064\n",
            "Checking accuracy on validation set\n",
            "Got 787 / 1000 correct (78.70)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.6673\n",
            "Checking accuracy on validation set\n",
            "Got 738 / 1000 correct (73.80)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.5918\n",
            "Checking accuracy on validation set\n",
            "Got 811 / 1000 correct (81.10)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.6958\n",
            "Checking accuracy on validation set\n",
            "Got 791 / 1000 correct (79.10)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.5946\n",
            "Checking accuracy on validation set\n",
            "Got 784 / 1000 correct (78.40)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.8255\n",
            "Checking accuracy on validation set\n",
            "Got 799 / 1000 correct (79.90)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.6452\n",
            "Checking accuracy on validation set\n",
            "Got 796 / 1000 correct (79.60)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.6256\n",
            "Checking accuracy on validation set\n",
            "Got 804 / 1000 correct (80.40)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.7987\n",
            "Checking accuracy on validation set\n",
            "Got 817 / 1000 correct (81.70)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.4277\n",
            "Checking accuracy on validation set\n",
            "Got 804 / 1000 correct (80.40)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.5055\n",
            "Checking accuracy on validation set\n",
            "Got 800 / 1000 correct (80.00)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.5609\n",
            "Checking accuracy on validation set\n",
            "Got 797 / 1000 correct (79.70)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.7767\n",
            "Checking accuracy on validation set\n",
            "Got 824 / 1000 correct (82.40)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.5631\n",
            "Checking accuracy on validation set\n",
            "Got 785 / 1000 correct (78.50)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.6253\n",
            "Checking accuracy on validation set\n",
            "Got 818 / 1000 correct (81.80)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.4565\n",
            "Checking accuracy on validation set\n",
            "Got 802 / 1000 correct (80.20)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.6071\n",
            "Checking accuracy on validation set\n",
            "Got 812 / 1000 correct (81.20)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.7066\n",
            "Checking accuracy on validation set\n",
            "Got 792 / 1000 correct (79.20)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.4473\n",
            "Checking accuracy on validation set\n",
            "Got 821 / 1000 correct (82.10)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.6348\n",
            "Checking accuracy on validation set\n",
            "Got 820 / 1000 correct (82.00)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.5920\n",
            "Checking accuracy on validation set\n",
            "Got 814 / 1000 correct (81.40)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 834 / 1000 correct (83.40)\n",
            "---- New lr = 0.00436 and weight_decay = 0.00043 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 3.4620\n",
            "Checking accuracy on validation set\n",
            "Got 87 / 1000 correct (8.70)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 2.1996\n",
            "Checking accuracy on validation set\n",
            "Got 255 / 1000 correct (25.50)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 2.1659\n",
            "Checking accuracy on validation set\n",
            "Got 257 / 1000 correct (25.70)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.9238\n",
            "Checking accuracy on validation set\n",
            "Got 329 / 1000 correct (32.90)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.5040\n",
            "Checking accuracy on validation set\n",
            "Got 356 / 1000 correct (35.60)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.6353\n",
            "Checking accuracy on validation set\n",
            "Got 367 / 1000 correct (36.70)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.7766\n",
            "Checking accuracy on validation set\n",
            "Got 459 / 1000 correct (45.90)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 1.3713\n",
            "Checking accuracy on validation set\n",
            "Got 361 / 1000 correct (36.10)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 1.6148\n",
            "Checking accuracy on validation set\n",
            "Got 381 / 1000 correct (38.10)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 1.4392\n",
            "Checking accuracy on validation set\n",
            "Got 436 / 1000 correct (43.60)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 1.4399\n",
            "Checking accuracy on validation set\n",
            "Got 474 / 1000 correct (47.40)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 1.3040\n",
            "Checking accuracy on validation set\n",
            "Got 500 / 1000 correct (50.00)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 1.3870\n",
            "Checking accuracy on validation set\n",
            "Got 474 / 1000 correct (47.40)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 1.2443\n",
            "Checking accuracy on validation set\n",
            "Got 501 / 1000 correct (50.10)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 1.2861\n",
            "Checking accuracy on validation set\n",
            "Got 535 / 1000 correct (53.50)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 1.1665\n",
            "Checking accuracy on validation set\n",
            "Got 551 / 1000 correct (55.10)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 1.2621\n",
            "Checking accuracy on validation set\n",
            "Got 542 / 1000 correct (54.20)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 1.1407\n",
            "Checking accuracy on validation set\n",
            "Got 565 / 1000 correct (56.50)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 1.3779\n",
            "Checking accuracy on validation set\n",
            "Got 574 / 1000 correct (57.40)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 1.2618\n",
            "Checking accuracy on validation set\n",
            "Got 555 / 1000 correct (55.50)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 1.0655\n",
            "Checking accuracy on validation set\n",
            "Got 555 / 1000 correct (55.50)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 1.2842\n",
            "Checking accuracy on validation set\n",
            "Got 599 / 1000 correct (59.90)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 0.9316\n",
            "Checking accuracy on validation set\n",
            "Got 546 / 1000 correct (54.60)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 0.9524\n",
            "Checking accuracy on validation set\n",
            "Got 609 / 1000 correct (60.90)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 1.1476\n",
            "Checking accuracy on validation set\n",
            "Got 618 / 1000 correct (61.80)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 0.9909\n",
            "Checking accuracy on validation set\n",
            "Got 606 / 1000 correct (60.60)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 1.3002\n",
            "Checking accuracy on validation set\n",
            "Got 636 / 1000 correct (63.60)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 1.1747\n",
            "Checking accuracy on validation set\n",
            "Got 577 / 1000 correct (57.70)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.8237\n",
            "Checking accuracy on validation set\n",
            "Got 646 / 1000 correct (64.60)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.8466\n",
            "Checking accuracy on validation set\n",
            "Got 648 / 1000 correct (64.80)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 1.0334\n",
            "Checking accuracy on validation set\n",
            "Got 680 / 1000 correct (68.00)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 1.0465\n",
            "Checking accuracy on validation set\n",
            "Got 676 / 1000 correct (67.60)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.9823\n",
            "Checking accuracy on validation set\n",
            "Got 587 / 1000 correct (58.70)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 1.2976\n",
            "Checking accuracy on validation set\n",
            "Got 666 / 1000 correct (66.60)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 1.2033\n",
            "Checking accuracy on validation set\n",
            "Got 637 / 1000 correct (63.70)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 1.0674\n",
            "Checking accuracy on validation set\n",
            "Got 644 / 1000 correct (64.40)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 1.0377\n",
            "Checking accuracy on validation set\n",
            "Got 676 / 1000 correct (67.60)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 1.0306\n",
            "Checking accuracy on validation set\n",
            "Got 652 / 1000 correct (65.20)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.7871\n",
            "Checking accuracy on validation set\n",
            "Got 678 / 1000 correct (67.80)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.9151\n",
            "Checking accuracy on validation set\n",
            "Got 690 / 1000 correct (69.00)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 1.0005\n",
            "Checking accuracy on validation set\n",
            "Got 648 / 1000 correct (64.80)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.9758\n",
            "Checking accuracy on validation set\n",
            "Got 675 / 1000 correct (67.50)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.9337\n",
            "Checking accuracy on validation set\n",
            "Got 685 / 1000 correct (68.50)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 1.1043\n",
            "Checking accuracy on validation set\n",
            "Got 673 / 1000 correct (67.30)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 1.0914\n",
            "Checking accuracy on validation set\n",
            "Got 608 / 1000 correct (60.80)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.9031\n",
            "Checking accuracy on validation set\n",
            "Got 673 / 1000 correct (67.30)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 1.0843\n",
            "Checking accuracy on validation set\n",
            "Got 652 / 1000 correct (65.20)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.8896\n",
            "Checking accuracy on validation set\n",
            "Got 692 / 1000 correct (69.20)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 1.0099\n",
            "Checking accuracy on validation set\n",
            "Got 701 / 1000 correct (70.10)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.9278\n",
            "Checking accuracy on validation set\n",
            "Got 712 / 1000 correct (71.20)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 1.1078\n",
            "Checking accuracy on validation set\n",
            "Got 671 / 1000 correct (67.10)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.7642\n",
            "Checking accuracy on validation set\n",
            "Got 712 / 1000 correct (71.20)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.8321\n",
            "Checking accuracy on validation set\n",
            "Got 726 / 1000 correct (72.60)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.6917\n",
            "Checking accuracy on validation set\n",
            "Got 689 / 1000 correct (68.90)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.8216\n",
            "Checking accuracy on validation set\n",
            "Got 660 / 1000 correct (66.00)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 1.0368\n",
            "Checking accuracy on validation set\n",
            "Got 679 / 1000 correct (67.90)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.8373\n",
            "Checking accuracy on validation set\n",
            "Got 692 / 1000 correct (69.20)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 1.0273\n",
            "Checking accuracy on validation set\n",
            "Got 696 / 1000 correct (69.60)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.7657\n",
            "Checking accuracy on validation set\n",
            "Got 698 / 1000 correct (69.80)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.9009\n",
            "Checking accuracy on validation set\n",
            "Got 730 / 1000 correct (73.00)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.8699\n",
            "Checking accuracy on validation set\n",
            "Got 714 / 1000 correct (71.40)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 1.0327\n",
            "Checking accuracy on validation set\n",
            "Got 728 / 1000 correct (72.80)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.7949\n",
            "Checking accuracy on validation set\n",
            "Got 764 / 1000 correct (76.40)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.8411\n",
            "Checking accuracy on validation set\n",
            "Got 682 / 1000 correct (68.20)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.7828\n",
            "Checking accuracy on validation set\n",
            "Got 701 / 1000 correct (70.10)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.6952\n",
            "Checking accuracy on validation set\n",
            "Got 721 / 1000 correct (72.10)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.9176\n",
            "Checking accuracy on validation set\n",
            "Got 745 / 1000 correct (74.50)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.7200\n",
            "Checking accuracy on validation set\n",
            "Got 739 / 1000 correct (73.90)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.6784\n",
            "Checking accuracy on validation set\n",
            "Got 731 / 1000 correct (73.10)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.7591\n",
            "Checking accuracy on validation set\n",
            "Got 723 / 1000 correct (72.30)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.5839\n",
            "Checking accuracy on validation set\n",
            "Got 714 / 1000 correct (71.40)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.8639\n",
            "Checking accuracy on validation set\n",
            "Got 707 / 1000 correct (70.70)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.7626\n",
            "Checking accuracy on validation set\n",
            "Got 730 / 1000 correct (73.00)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.9066\n",
            "Checking accuracy on validation set\n",
            "Got 717 / 1000 correct (71.70)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.6524\n",
            "Checking accuracy on validation set\n",
            "Got 745 / 1000 correct (74.50)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.8014\n",
            "Checking accuracy on validation set\n",
            "Got 757 / 1000 correct (75.70)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.8328\n",
            "Checking accuracy on validation set\n",
            "Got 689 / 1000 correct (68.90)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.6587\n",
            "Checking accuracy on validation set\n",
            "Got 730 / 1000 correct (73.00)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.6628\n",
            "Checking accuracy on validation set\n",
            "Got 754 / 1000 correct (75.40)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.8611\n",
            "Checking accuracy on validation set\n",
            "Got 747 / 1000 correct (74.70)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 661 / 1000 correct (66.10)\n",
            "---- New lr = 0.00054 and weight_decay = 0.00075 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 3.4717\n",
            "Checking accuracy on validation set\n",
            "Got 136 / 1000 correct (13.60)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 1.6679\n",
            "Checking accuracy on validation set\n",
            "Got 339 / 1000 correct (33.90)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.3465\n",
            "Checking accuracy on validation set\n",
            "Got 441 / 1000 correct (44.10)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.2794\n",
            "Checking accuracy on validation set\n",
            "Got 468 / 1000 correct (46.80)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.0497\n",
            "Checking accuracy on validation set\n",
            "Got 472 / 1000 correct (47.20)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.5951\n",
            "Checking accuracy on validation set\n",
            "Got 562 / 1000 correct (56.20)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.1339\n",
            "Checking accuracy on validation set\n",
            "Got 524 / 1000 correct (52.40)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 1.1411\n",
            "Checking accuracy on validation set\n",
            "Got 504 / 1000 correct (50.40)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 0.9967\n",
            "Checking accuracy on validation set\n",
            "Got 634 / 1000 correct (63.40)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 1.2501\n",
            "Checking accuracy on validation set\n",
            "Got 617 / 1000 correct (61.70)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 1.3029\n",
            "Checking accuracy on validation set\n",
            "Got 559 / 1000 correct (55.90)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 1.3562\n",
            "Checking accuracy on validation set\n",
            "Got 672 / 1000 correct (67.20)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 0.9250\n",
            "Checking accuracy on validation set\n",
            "Got 675 / 1000 correct (67.50)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 1.0387\n",
            "Checking accuracy on validation set\n",
            "Got 611 / 1000 correct (61.10)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 0.8259\n",
            "Checking accuracy on validation set\n",
            "Got 674 / 1000 correct (67.40)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 0.8010\n",
            "Checking accuracy on validation set\n",
            "Got 712 / 1000 correct (71.20)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 0.7967\n",
            "Checking accuracy on validation set\n",
            "Got 595 / 1000 correct (59.50)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 0.6788\n",
            "Checking accuracy on validation set\n",
            "Got 699 / 1000 correct (69.90)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 0.9476\n",
            "Checking accuracy on validation set\n",
            "Got 712 / 1000 correct (71.20)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 0.7392\n",
            "Checking accuracy on validation set\n",
            "Got 739 / 1000 correct (73.90)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 0.9760\n",
            "Checking accuracy on validation set\n",
            "Got 686 / 1000 correct (68.60)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 0.9528\n",
            "Checking accuracy on validation set\n",
            "Got 701 / 1000 correct (70.10)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 0.6238\n",
            "Checking accuracy on validation set\n",
            "Got 760 / 1000 correct (76.00)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 0.9815\n",
            "Checking accuracy on validation set\n",
            "Got 743 / 1000 correct (74.30)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 0.5655\n",
            "Checking accuracy on validation set\n",
            "Got 770 / 1000 correct (77.00)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 0.7785\n",
            "Checking accuracy on validation set\n",
            "Got 769 / 1000 correct (76.90)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 0.7119\n",
            "Checking accuracy on validation set\n",
            "Got 755 / 1000 correct (75.50)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 0.7810\n",
            "Checking accuracy on validation set\n",
            "Got 764 / 1000 correct (76.40)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.6422\n",
            "Checking accuracy on validation set\n",
            "Got 747 / 1000 correct (74.70)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 1.1456\n",
            "Checking accuracy on validation set\n",
            "Got 779 / 1000 correct (77.90)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.9161\n",
            "Checking accuracy on validation set\n",
            "Got 773 / 1000 correct (77.30)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 0.7744\n",
            "Checking accuracy on validation set\n",
            "Got 772 / 1000 correct (77.20)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.8108\n",
            "Checking accuracy on validation set\n",
            "Got 783 / 1000 correct (78.30)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 0.7126\n",
            "Checking accuracy on validation set\n",
            "Got 795 / 1000 correct (79.50)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.4532\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.5126\n",
            "Checking accuracy on validation set\n",
            "Got 788 / 1000 correct (78.80)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 0.8368\n",
            "Checking accuracy on validation set\n",
            "Got 793 / 1000 correct (79.30)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 0.5903\n",
            "Checking accuracy on validation set\n",
            "Got 784 / 1000 correct (78.40)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.4587\n",
            "Checking accuracy on validation set\n",
            "Got 799 / 1000 correct (79.90)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.6397\n",
            "Checking accuracy on validation set\n",
            "Got 749 / 1000 correct (74.90)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.6492\n",
            "Checking accuracy on validation set\n",
            "Got 765 / 1000 correct (76.50)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.4773\n",
            "Checking accuracy on validation set\n",
            "Got 751 / 1000 correct (75.10)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.7213\n",
            "Checking accuracy on validation set\n",
            "Got 777 / 1000 correct (77.70)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.3517\n",
            "Checking accuracy on validation set\n",
            "Got 791 / 1000 correct (79.10)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 0.4851\n",
            "Checking accuracy on validation set\n",
            "Got 813 / 1000 correct (81.30)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.4831\n",
            "Checking accuracy on validation set\n",
            "Got 794 / 1000 correct (79.40)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.6262\n",
            "Checking accuracy on validation set\n",
            "Got 804 / 1000 correct (80.40)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.5309\n",
            "Checking accuracy on validation set\n",
            "Got 800 / 1000 correct (80.00)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.5612\n",
            "Checking accuracy on validation set\n",
            "Got 774 / 1000 correct (77.40)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.8659\n",
            "Checking accuracy on validation set\n",
            "Got 770 / 1000 correct (77.00)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.4710\n",
            "Checking accuracy on validation set\n",
            "Got 821 / 1000 correct (82.10)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.7906\n",
            "Checking accuracy on validation set\n",
            "Got 815 / 1000 correct (81.50)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.6560\n",
            "Checking accuracy on validation set\n",
            "Got 779 / 1000 correct (77.90)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.5261\n",
            "Checking accuracy on validation set\n",
            "Got 819 / 1000 correct (81.90)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.6829\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.6867\n",
            "Checking accuracy on validation set\n",
            "Got 827 / 1000 correct (82.70)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.4425\n",
            "Checking accuracy on validation set\n",
            "Got 795 / 1000 correct (79.50)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.3692\n",
            "Checking accuracy on validation set\n",
            "Got 805 / 1000 correct (80.50)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.7967\n",
            "Checking accuracy on validation set\n",
            "Got 823 / 1000 correct (82.30)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.4009\n",
            "Checking accuracy on validation set\n",
            "Got 815 / 1000 correct (81.50)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.5452\n",
            "Checking accuracy on validation set\n",
            "Got 835 / 1000 correct (83.50)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.6414\n",
            "Checking accuracy on validation set\n",
            "Got 808 / 1000 correct (80.80)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.4052\n",
            "Checking accuracy on validation set\n",
            "Got 776 / 1000 correct (77.60)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.4581\n",
            "Checking accuracy on validation set\n",
            "Got 820 / 1000 correct (82.00)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.3370\n",
            "Checking accuracy on validation set\n",
            "Got 816 / 1000 correct (81.60)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.5719\n",
            "Checking accuracy on validation set\n",
            "Got 829 / 1000 correct (82.90)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.6107\n",
            "Checking accuracy on validation set\n",
            "Got 813 / 1000 correct (81.30)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.3562\n",
            "Checking accuracy on validation set\n",
            "Got 842 / 1000 correct (84.20)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.5714\n",
            "Checking accuracy on validation set\n",
            "Got 813 / 1000 correct (81.30)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.7531\n",
            "Checking accuracy on validation set\n",
            "Got 822 / 1000 correct (82.20)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.7050\n",
            "Checking accuracy on validation set\n",
            "Got 834 / 1000 correct (83.40)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.5880\n",
            "Checking accuracy on validation set\n",
            "Got 824 / 1000 correct (82.40)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.4624\n",
            "Checking accuracy on validation set\n",
            "Got 836 / 1000 correct (83.60)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.2769\n",
            "Checking accuracy on validation set\n",
            "Got 846 / 1000 correct (84.60)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.4989\n",
            "Checking accuracy on validation set\n",
            "Got 840 / 1000 correct (84.00)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.3623\n",
            "Checking accuracy on validation set\n",
            "Got 857 / 1000 correct (85.70)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.5116\n",
            "Checking accuracy on validation set\n",
            "Got 845 / 1000 correct (84.50)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.6583\n",
            "Checking accuracy on validation set\n",
            "Got 839 / 1000 correct (83.90)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.5442\n",
            "Checking accuracy on validation set\n",
            "Got 829 / 1000 correct (82.90)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.4197\n",
            "Checking accuracy on validation set\n",
            "Got 846 / 1000 correct (84.60)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 815 / 1000 correct (81.50)\n",
            "---- New lr = 0.00237 and weight_decay = 0.00025 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 4.1793\n",
            "Checking accuracy on validation set\n",
            "Got 118 / 1000 correct (11.80)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 2.5484\n",
            "Checking accuracy on validation set\n",
            "Got 303 / 1000 correct (30.30)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.9622\n",
            "Checking accuracy on validation set\n",
            "Got 315 / 1000 correct (31.50)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.7700\n",
            "Checking accuracy on validation set\n",
            "Got 370 / 1000 correct (37.00)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.8527\n",
            "Checking accuracy on validation set\n",
            "Got 388 / 1000 correct (38.80)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.6491\n",
            "Checking accuracy on validation set\n",
            "Got 462 / 1000 correct (46.20)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.7105\n",
            "Checking accuracy on validation set\n",
            "Got 434 / 1000 correct (43.40)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 1.5549\n",
            "Checking accuracy on validation set\n",
            "Got 459 / 1000 correct (45.90)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 1.3154\n",
            "Checking accuracy on validation set\n",
            "Got 447 / 1000 correct (44.70)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 1.2250\n",
            "Checking accuracy on validation set\n",
            "Got 482 / 1000 correct (48.20)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 1.5368\n",
            "Checking accuracy on validation set\n",
            "Got 423 / 1000 correct (42.30)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 1.4170\n",
            "Checking accuracy on validation set\n",
            "Got 541 / 1000 correct (54.10)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 1.1212\n",
            "Checking accuracy on validation set\n",
            "Got 514 / 1000 correct (51.40)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 1.0606\n",
            "Checking accuracy on validation set\n",
            "Got 573 / 1000 correct (57.30)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 1.2285\n",
            "Checking accuracy on validation set\n",
            "Got 465 / 1000 correct (46.50)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 1.0321\n",
            "Checking accuracy on validation set\n",
            "Got 585 / 1000 correct (58.50)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 1.2234\n",
            "Checking accuracy on validation set\n",
            "Got 589 / 1000 correct (58.90)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 1.1061\n",
            "Checking accuracy on validation set\n",
            "Got 590 / 1000 correct (59.00)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 0.9904\n",
            "Checking accuracy on validation set\n",
            "Got 539 / 1000 correct (53.90)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 0.9322\n",
            "Checking accuracy on validation set\n",
            "Got 550 / 1000 correct (55.00)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 1.4275\n",
            "Checking accuracy on validation set\n",
            "Got 647 / 1000 correct (64.70)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 1.0921\n",
            "Checking accuracy on validation set\n",
            "Got 616 / 1000 correct (61.60)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 0.8230\n",
            "Checking accuracy on validation set\n",
            "Got 673 / 1000 correct (67.30)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 1.3125\n",
            "Checking accuracy on validation set\n",
            "Got 678 / 1000 correct (67.80)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 0.8059\n",
            "Checking accuracy on validation set\n",
            "Got 649 / 1000 correct (64.90)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 1.2662\n",
            "Checking accuracy on validation set\n",
            "Got 652 / 1000 correct (65.20)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 1.4342\n",
            "Checking accuracy on validation set\n",
            "Got 671 / 1000 correct (67.10)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 1.0389\n",
            "Checking accuracy on validation set\n",
            "Got 610 / 1000 correct (61.00)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.8884\n",
            "Checking accuracy on validation set\n",
            "Got 653 / 1000 correct (65.30)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.9431\n",
            "Checking accuracy on validation set\n",
            "Got 584 / 1000 correct (58.40)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.9420\n",
            "Checking accuracy on validation set\n",
            "Got 694 / 1000 correct (69.40)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 1.0104\n",
            "Checking accuracy on validation set\n",
            "Got 658 / 1000 correct (65.80)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.6480\n",
            "Checking accuracy on validation set\n",
            "Got 725 / 1000 correct (72.50)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 0.9509\n",
            "Checking accuracy on validation set\n",
            "Got 724 / 1000 correct (72.40)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.8382\n",
            "Checking accuracy on validation set\n",
            "Got 680 / 1000 correct (68.00)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.8911\n",
            "Checking accuracy on validation set\n",
            "Got 729 / 1000 correct (72.90)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 0.8223\n",
            "Checking accuracy on validation set\n",
            "Got 706 / 1000 correct (70.60)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 0.8344\n",
            "Checking accuracy on validation set\n",
            "Got 748 / 1000 correct (74.80)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.9187\n",
            "Checking accuracy on validation set\n",
            "Got 726 / 1000 correct (72.60)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.7117\n",
            "Checking accuracy on validation set\n",
            "Got 710 / 1000 correct (71.00)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.9971\n",
            "Checking accuracy on validation set\n",
            "Got 723 / 1000 correct (72.30)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.5312\n",
            "Checking accuracy on validation set\n",
            "Got 755 / 1000 correct (75.50)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.6531\n",
            "Checking accuracy on validation set\n",
            "Got 763 / 1000 correct (76.30)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.9101\n",
            "Checking accuracy on validation set\n",
            "Got 738 / 1000 correct (73.80)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 0.9794\n",
            "Checking accuracy on validation set\n",
            "Got 738 / 1000 correct (73.80)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.8032\n",
            "Checking accuracy on validation set\n",
            "Got 785 / 1000 correct (78.50)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.6739\n",
            "Checking accuracy on validation set\n",
            "Got 761 / 1000 correct (76.10)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.5911\n",
            "Checking accuracy on validation set\n",
            "Got 767 / 1000 correct (76.70)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.8268\n",
            "Checking accuracy on validation set\n",
            "Got 749 / 1000 correct (74.90)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.8238\n",
            "Checking accuracy on validation set\n",
            "Got 779 / 1000 correct (77.90)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.6990\n",
            "Checking accuracy on validation set\n",
            "Got 742 / 1000 correct (74.20)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.7243\n",
            "Checking accuracy on validation set\n",
            "Got 766 / 1000 correct (76.60)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.8468\n",
            "Checking accuracy on validation set\n",
            "Got 780 / 1000 correct (78.00)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.5584\n",
            "Checking accuracy on validation set\n",
            "Got 781 / 1000 correct (78.10)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.7031\n",
            "Checking accuracy on validation set\n",
            "Got 753 / 1000 correct (75.30)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.8523\n",
            "Checking accuracy on validation set\n",
            "Got 796 / 1000 correct (79.60)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.6473\n",
            "Checking accuracy on validation set\n",
            "Got 790 / 1000 correct (79.00)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.7900\n",
            "Checking accuracy on validation set\n",
            "Got 783 / 1000 correct (78.30)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.7323\n",
            "Checking accuracy on validation set\n",
            "Got 794 / 1000 correct (79.40)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.7542\n",
            "Checking accuracy on validation set\n",
            "Got 811 / 1000 correct (81.10)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.3939\n",
            "Checking accuracy on validation set\n",
            "Got 797 / 1000 correct (79.70)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.9694\n",
            "Checking accuracy on validation set\n",
            "Got 798 / 1000 correct (79.80)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.7124\n",
            "Checking accuracy on validation set\n",
            "Got 784 / 1000 correct (78.40)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.6660\n",
            "Checking accuracy on validation set\n",
            "Got 788 / 1000 correct (78.80)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.6534\n",
            "Checking accuracy on validation set\n",
            "Got 808 / 1000 correct (80.80)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.4023\n",
            "Checking accuracy on validation set\n",
            "Got 805 / 1000 correct (80.50)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.6430\n",
            "Checking accuracy on validation set\n",
            "Got 806 / 1000 correct (80.60)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.7326\n",
            "Checking accuracy on validation set\n",
            "Got 783 / 1000 correct (78.30)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.4063\n",
            "Checking accuracy on validation set\n",
            "Got 767 / 1000 correct (76.70)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.6452\n",
            "Checking accuracy on validation set\n",
            "Got 801 / 1000 correct (80.10)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.7223\n",
            "Checking accuracy on validation set\n",
            "Got 801 / 1000 correct (80.10)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.6381\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.5030\n",
            "Checking accuracy on validation set\n",
            "Got 807 / 1000 correct (80.70)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.6153\n",
            "Checking accuracy on validation set\n",
            "Got 796 / 1000 correct (79.60)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.5535\n",
            "Checking accuracy on validation set\n",
            "Got 779 / 1000 correct (77.90)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.7405\n",
            "Checking accuracy on validation set\n",
            "Got 780 / 1000 correct (78.00)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.3113\n",
            "Checking accuracy on validation set\n",
            "Got 791 / 1000 correct (79.10)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.4472\n",
            "Checking accuracy on validation set\n",
            "Got 812 / 1000 correct (81.20)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.4203\n",
            "Checking accuracy on validation set\n",
            "Got 813 / 1000 correct (81.30)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.6771\n",
            "Checking accuracy on validation set\n",
            "Got 776 / 1000 correct (77.60)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 812 / 1000 correct (81.20)\n",
            "---- New lr = 0.00120 and weight_decay = 0.00080 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 3.2614\n",
            "Checking accuracy on validation set\n",
            "Got 117 / 1000 correct (11.70)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 1.9088\n",
            "Checking accuracy on validation set\n",
            "Got 337 / 1000 correct (33.70)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.6255\n",
            "Checking accuracy on validation set\n",
            "Got 395 / 1000 correct (39.50)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.4188\n",
            "Checking accuracy on validation set\n",
            "Got 426 / 1000 correct (42.60)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.4744\n",
            "Checking accuracy on validation set\n",
            "Got 478 / 1000 correct (47.80)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.5330\n",
            "Checking accuracy on validation set\n",
            "Got 333 / 1000 correct (33.30)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.6294\n",
            "Checking accuracy on validation set\n",
            "Got 453 / 1000 correct (45.30)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 1.5761\n",
            "Checking accuracy on validation set\n",
            "Got 471 / 1000 correct (47.10)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 1.3879\n",
            "Checking accuracy on validation set\n",
            "Got 526 / 1000 correct (52.60)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 1.3661\n",
            "Checking accuracy on validation set\n",
            "Got 540 / 1000 correct (54.00)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 1.2735\n",
            "Checking accuracy on validation set\n",
            "Got 524 / 1000 correct (52.40)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 1.2293\n",
            "Checking accuracy on validation set\n",
            "Got 577 / 1000 correct (57.70)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 1.2795\n",
            "Checking accuracy on validation set\n",
            "Got 581 / 1000 correct (58.10)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 1.2490\n",
            "Checking accuracy on validation set\n",
            "Got 595 / 1000 correct (59.50)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 0.9781\n",
            "Checking accuracy on validation set\n",
            "Got 589 / 1000 correct (58.90)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 1.0093\n",
            "Checking accuracy on validation set\n",
            "Got 568 / 1000 correct (56.80)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 1.0968\n",
            "Checking accuracy on validation set\n",
            "Got 573 / 1000 correct (57.30)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 1.1275\n",
            "Checking accuracy on validation set\n",
            "Got 546 / 1000 correct (54.60)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 0.8807\n",
            "Checking accuracy on validation set\n",
            "Got 592 / 1000 correct (59.20)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 1.2455\n",
            "Checking accuracy on validation set\n",
            "Got 613 / 1000 correct (61.30)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 0.9981\n",
            "Checking accuracy on validation set\n",
            "Got 643 / 1000 correct (64.30)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 1.1209\n",
            "Checking accuracy on validation set\n",
            "Got 666 / 1000 correct (66.60)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 1.0208\n",
            "Checking accuracy on validation set\n",
            "Got 686 / 1000 correct (68.60)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 1.0614\n",
            "Checking accuracy on validation set\n",
            "Got 578 / 1000 correct (57.80)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 0.8487\n",
            "Checking accuracy on validation set\n",
            "Got 581 / 1000 correct (58.10)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 1.0686\n",
            "Checking accuracy on validation set\n",
            "Got 693 / 1000 correct (69.30)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 0.9531\n",
            "Checking accuracy on validation set\n",
            "Got 641 / 1000 correct (64.10)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 0.7356\n",
            "Checking accuracy on validation set\n",
            "Got 695 / 1000 correct (69.50)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 1.0272\n",
            "Checking accuracy on validation set\n",
            "Got 581 / 1000 correct (58.10)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.9614\n",
            "Checking accuracy on validation set\n",
            "Got 674 / 1000 correct (67.40)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.8424\n",
            "Checking accuracy on validation set\n",
            "Got 705 / 1000 correct (70.50)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 0.8609\n",
            "Checking accuracy on validation set\n",
            "Got 701 / 1000 correct (70.10)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.8225\n",
            "Checking accuracy on validation set\n",
            "Got 716 / 1000 correct (71.60)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 0.7704\n",
            "Checking accuracy on validation set\n",
            "Got 677 / 1000 correct (67.70)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.9049\n",
            "Checking accuracy on validation set\n",
            "Got 705 / 1000 correct (70.50)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.9314\n",
            "Checking accuracy on validation set\n",
            "Got 707 / 1000 correct (70.70)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 0.9410\n",
            "Checking accuracy on validation set\n",
            "Got 698 / 1000 correct (69.80)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 1.0104\n",
            "Checking accuracy on validation set\n",
            "Got 727 / 1000 correct (72.70)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.8121\n",
            "Checking accuracy on validation set\n",
            "Got 744 / 1000 correct (74.40)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.8204\n",
            "Checking accuracy on validation set\n",
            "Got 731 / 1000 correct (73.10)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.6797\n",
            "Checking accuracy on validation set\n",
            "Got 748 / 1000 correct (74.80)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.6976\n",
            "Checking accuracy on validation set\n",
            "Got 762 / 1000 correct (76.20)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.6350\n",
            "Checking accuracy on validation set\n",
            "Got 733 / 1000 correct (73.30)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.8498\n",
            "Checking accuracy on validation set\n",
            "Got 732 / 1000 correct (73.20)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 0.6859\n",
            "Checking accuracy on validation set\n",
            "Got 746 / 1000 correct (74.60)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.5634\n",
            "Checking accuracy on validation set\n",
            "Got 772 / 1000 correct (77.20)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.6001\n",
            "Checking accuracy on validation set\n",
            "Got 738 / 1000 correct (73.80)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.5672\n",
            "Checking accuracy on validation set\n",
            "Got 731 / 1000 correct (73.10)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.6694\n",
            "Checking accuracy on validation set\n",
            "Got 763 / 1000 correct (76.30)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.7682\n",
            "Checking accuracy on validation set\n",
            "Got 727 / 1000 correct (72.70)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.5734\n",
            "Checking accuracy on validation set\n",
            "Got 773 / 1000 correct (77.30)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.6905\n",
            "Checking accuracy on validation set\n",
            "Got 778 / 1000 correct (77.80)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.6748\n",
            "Checking accuracy on validation set\n",
            "Got 780 / 1000 correct (78.00)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.5193\n",
            "Checking accuracy on validation set\n",
            "Got 774 / 1000 correct (77.40)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.6373\n",
            "Checking accuracy on validation set\n",
            "Got 794 / 1000 correct (79.40)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.7484\n",
            "Checking accuracy on validation set\n",
            "Got 785 / 1000 correct (78.50)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.7486\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.7751\n",
            "Checking accuracy on validation set\n",
            "Got 794 / 1000 correct (79.40)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.4654\n",
            "Checking accuracy on validation set\n",
            "Got 786 / 1000 correct (78.60)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 1.0860\n",
            "Checking accuracy on validation set\n",
            "Got 772 / 1000 correct (77.20)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.7464\n",
            "Checking accuracy on validation set\n",
            "Got 785 / 1000 correct (78.50)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.5870\n",
            "Checking accuracy on validation set\n",
            "Got 792 / 1000 correct (79.20)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.7671\n",
            "Checking accuracy on validation set\n",
            "Got 790 / 1000 correct (79.00)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.7933\n",
            "Checking accuracy on validation set\n",
            "Got 781 / 1000 correct (78.10)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.4610\n",
            "Checking accuracy on validation set\n",
            "Got 797 / 1000 correct (79.70)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.6934\n",
            "Checking accuracy on validation set\n",
            "Got 795 / 1000 correct (79.50)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.6043\n",
            "Checking accuracy on validation set\n",
            "Got 764 / 1000 correct (76.40)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.5587\n",
            "Checking accuracy on validation set\n",
            "Got 779 / 1000 correct (77.90)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.6358\n",
            "Checking accuracy on validation set\n",
            "Got 790 / 1000 correct (79.00)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.7137\n",
            "Checking accuracy on validation set\n",
            "Got 783 / 1000 correct (78.30)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.4559\n",
            "Checking accuracy on validation set\n",
            "Got 798 / 1000 correct (79.80)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.7269\n",
            "Checking accuracy on validation set\n",
            "Got 806 / 1000 correct (80.60)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.7253\n",
            "Checking accuracy on validation set\n",
            "Got 792 / 1000 correct (79.20)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.7316\n",
            "Checking accuracy on validation set\n",
            "Got 827 / 1000 correct (82.70)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.5183\n",
            "Checking accuracy on validation set\n",
            "Got 804 / 1000 correct (80.40)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.5698\n",
            "Checking accuracy on validation set\n",
            "Got 810 / 1000 correct (81.00)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.6955\n",
            "Checking accuracy on validation set\n",
            "Got 822 / 1000 correct (82.20)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.7105\n",
            "Checking accuracy on validation set\n",
            "Got 829 / 1000 correct (82.90)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.6265\n",
            "Checking accuracy on validation set\n",
            "Got 810 / 1000 correct (81.00)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.6090\n",
            "Checking accuracy on validation set\n",
            "Got 780 / 1000 correct (78.00)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "---- New lr = 0.00050 and weight_decay = 0.00082 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 3.3882\n",
            "Checking accuracy on validation set\n",
            "Got 96 / 1000 correct (9.60)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 1.9832\n",
            "Checking accuracy on validation set\n",
            "Got 383 / 1000 correct (38.30)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.7399\n",
            "Checking accuracy on validation set\n",
            "Got 374 / 1000 correct (37.40)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.4217\n",
            "Checking accuracy on validation set\n",
            "Got 405 / 1000 correct (40.50)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 0.9706\n",
            "Checking accuracy on validation set\n",
            "Got 516 / 1000 correct (51.60)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.5235\n",
            "Checking accuracy on validation set\n",
            "Got 559 / 1000 correct (55.90)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.4245\n",
            "Checking accuracy on validation set\n",
            "Got 504 / 1000 correct (50.40)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 1.1195\n",
            "Checking accuracy on validation set\n",
            "Got 628 / 1000 correct (62.80)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 1.0691\n",
            "Checking accuracy on validation set\n",
            "Got 630 / 1000 correct (63.00)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 1.3584\n",
            "Checking accuracy on validation set\n",
            "Got 626 / 1000 correct (62.60)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 0.8176\n",
            "Checking accuracy on validation set\n",
            "Got 637 / 1000 correct (63.70)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 1.1158\n",
            "Checking accuracy on validation set\n",
            "Got 689 / 1000 correct (68.90)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 0.8554\n",
            "Checking accuracy on validation set\n",
            "Got 653 / 1000 correct (65.30)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 0.8557\n",
            "Checking accuracy on validation set\n",
            "Got 742 / 1000 correct (74.20)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 0.9247\n",
            "Checking accuracy on validation set\n",
            "Got 678 / 1000 correct (67.80)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 1.1115\n",
            "Checking accuracy on validation set\n",
            "Got 687 / 1000 correct (68.70)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 0.9358\n",
            "Checking accuracy on validation set\n",
            "Got 690 / 1000 correct (69.00)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 0.7606\n",
            "Checking accuracy on validation set\n",
            "Got 735 / 1000 correct (73.50)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 0.7134\n",
            "Checking accuracy on validation set\n",
            "Got 687 / 1000 correct (68.70)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 0.8810\n",
            "Checking accuracy on validation set\n",
            "Got 736 / 1000 correct (73.60)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 0.8457\n",
            "Checking accuracy on validation set\n",
            "Got 757 / 1000 correct (75.70)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 0.8176\n",
            "Checking accuracy on validation set\n",
            "Got 765 / 1000 correct (76.50)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 0.8160\n",
            "Checking accuracy on validation set\n",
            "Got 736 / 1000 correct (73.60)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 0.6686\n",
            "Checking accuracy on validation set\n",
            "Got 738 / 1000 correct (73.80)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 0.7775\n",
            "Checking accuracy on validation set\n",
            "Got 745 / 1000 correct (74.50)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 0.8526\n",
            "Checking accuracy on validation set\n",
            "Got 746 / 1000 correct (74.60)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 0.7031\n",
            "Checking accuracy on validation set\n",
            "Got 747 / 1000 correct (74.70)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 0.5623\n",
            "Checking accuracy on validation set\n",
            "Got 754 / 1000 correct (75.40)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.8627\n",
            "Checking accuracy on validation set\n",
            "Got 787 / 1000 correct (78.70)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.6818\n",
            "Checking accuracy on validation set\n",
            "Got 682 / 1000 correct (68.20)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.6608\n",
            "Checking accuracy on validation set\n",
            "Got 729 / 1000 correct (72.90)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 0.8219\n",
            "Checking accuracy on validation set\n",
            "Got 760 / 1000 correct (76.00)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.6142\n",
            "Checking accuracy on validation set\n",
            "Got 761 / 1000 correct (76.10)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 0.5636\n",
            "Checking accuracy on validation set\n",
            "Got 761 / 1000 correct (76.10)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.8941\n",
            "Checking accuracy on validation set\n",
            "Got 765 / 1000 correct (76.50)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.6417\n",
            "Checking accuracy on validation set\n",
            "Got 764 / 1000 correct (76.40)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 0.4629\n",
            "Checking accuracy on validation set\n",
            "Got 778 / 1000 correct (77.80)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 0.3687\n",
            "Checking accuracy on validation set\n",
            "Got 793 / 1000 correct (79.30)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.7354\n",
            "Checking accuracy on validation set\n",
            "Got 764 / 1000 correct (76.40)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.5455\n",
            "Checking accuracy on validation set\n",
            "Got 789 / 1000 correct (78.90)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.6358\n",
            "Checking accuracy on validation set\n",
            "Got 805 / 1000 correct (80.50)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.8770\n",
            "Checking accuracy on validation set\n",
            "Got 778 / 1000 correct (77.80)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.7338\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.5864\n",
            "Checking accuracy on validation set\n",
            "Got 767 / 1000 correct (76.70)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 0.6946\n",
            "Checking accuracy on validation set\n",
            "Got 822 / 1000 correct (82.20)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.5171\n",
            "Checking accuracy on validation set\n",
            "Got 783 / 1000 correct (78.30)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.6001\n",
            "Checking accuracy on validation set\n",
            "Got 791 / 1000 correct (79.10)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.7053\n",
            "Checking accuracy on validation set\n",
            "Got 819 / 1000 correct (81.90)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.5662\n",
            "Checking accuracy on validation set\n",
            "Got 784 / 1000 correct (78.40)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.7583\n",
            "Checking accuracy on validation set\n",
            "Got 803 / 1000 correct (80.30)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.8706\n",
            "Checking accuracy on validation set\n",
            "Got 812 / 1000 correct (81.20)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.4611\n",
            "Checking accuracy on validation set\n",
            "Got 808 / 1000 correct (80.80)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.5987\n",
            "Checking accuracy on validation set\n",
            "Got 794 / 1000 correct (79.40)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.4876\n",
            "Checking accuracy on validation set\n",
            "Got 806 / 1000 correct (80.60)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.5553\n",
            "Checking accuracy on validation set\n",
            "Got 769 / 1000 correct (76.90)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.3599\n",
            "Checking accuracy on validation set\n",
            "Got 848 / 1000 correct (84.80)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.8698\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.4808\n",
            "Checking accuracy on validation set\n",
            "Got 825 / 1000 correct (82.50)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.3942\n",
            "Checking accuracy on validation set\n",
            "Got 802 / 1000 correct (80.20)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.5623\n",
            "Checking accuracy on validation set\n",
            "Got 813 / 1000 correct (81.30)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.4901\n",
            "Checking accuracy on validation set\n",
            "Got 817 / 1000 correct (81.70)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.4920\n",
            "Checking accuracy on validation set\n",
            "Got 818 / 1000 correct (81.80)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.5605\n",
            "Checking accuracy on validation set\n",
            "Got 823 / 1000 correct (82.30)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.4478\n",
            "Checking accuracy on validation set\n",
            "Got 823 / 1000 correct (82.30)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.4168\n",
            "Checking accuracy on validation set\n",
            "Got 791 / 1000 correct (79.10)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.6236\n",
            "Checking accuracy on validation set\n",
            "Got 834 / 1000 correct (83.40)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.3991\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.6085\n",
            "Checking accuracy on validation set\n",
            "Got 837 / 1000 correct (83.70)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.6161\n",
            "Checking accuracy on validation set\n",
            "Got 826 / 1000 correct (82.60)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.7398\n",
            "Checking accuracy on validation set\n",
            "Got 817 / 1000 correct (81.70)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.4308\n",
            "Checking accuracy on validation set\n",
            "Got 811 / 1000 correct (81.10)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.5133\n",
            "Checking accuracy on validation set\n",
            "Got 822 / 1000 correct (82.20)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.3946\n",
            "Checking accuracy on validation set\n",
            "Got 839 / 1000 correct (83.90)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.5042\n",
            "Checking accuracy on validation set\n",
            "Got 855 / 1000 correct (85.50)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.4650\n",
            "Checking accuracy on validation set\n",
            "Got 831 / 1000 correct (83.10)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.4590\n",
            "Checking accuracy on validation set\n",
            "Got 794 / 1000 correct (79.40)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.4735\n",
            "Checking accuracy on validation set\n",
            "Got 832 / 1000 correct (83.20)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.3961\n",
            "Checking accuracy on validation set\n",
            "Got 804 / 1000 correct (80.40)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.3924\n",
            "Checking accuracy on validation set\n",
            "Got 817 / 1000 correct (81.70)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.4914\n",
            "Checking accuracy on validation set\n",
            "Got 829 / 1000 correct (82.90)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 847 / 1000 correct (84.70)\n",
            "---- New lr = 0.00053 and weight_decay = 0.00074 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 2.8960\n",
            "Checking accuracy on validation set\n",
            "Got 89 / 1000 correct (8.90)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 1.7372\n",
            "Checking accuracy on validation set\n",
            "Got 349 / 1000 correct (34.90)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.5385\n",
            "Checking accuracy on validation set\n",
            "Got 467 / 1000 correct (46.70)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.7382\n",
            "Checking accuracy on validation set\n",
            "Got 514 / 1000 correct (51.40)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.3829\n",
            "Checking accuracy on validation set\n",
            "Got 524 / 1000 correct (52.40)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.3320\n",
            "Checking accuracy on validation set\n",
            "Got 553 / 1000 correct (55.30)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.4322\n",
            "Checking accuracy on validation set\n",
            "Got 505 / 1000 correct (50.50)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 0.9005\n",
            "Checking accuracy on validation set\n",
            "Got 597 / 1000 correct (59.70)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 1.1869\n",
            "Checking accuracy on validation set\n",
            "Got 603 / 1000 correct (60.30)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 1.0709\n",
            "Checking accuracy on validation set\n",
            "Got 604 / 1000 correct (60.40)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 0.8830\n",
            "Checking accuracy on validation set\n",
            "Got 604 / 1000 correct (60.40)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 1.0941\n",
            "Checking accuracy on validation set\n",
            "Got 668 / 1000 correct (66.80)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 0.9058\n",
            "Checking accuracy on validation set\n",
            "Got 661 / 1000 correct (66.10)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 0.7797\n",
            "Checking accuracy on validation set\n",
            "Got 712 / 1000 correct (71.20)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 0.8978\n",
            "Checking accuracy on validation set\n",
            "Got 697 / 1000 correct (69.70)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 0.9699\n",
            "Checking accuracy on validation set\n",
            "Got 709 / 1000 correct (70.90)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 0.7268\n",
            "Checking accuracy on validation set\n",
            "Got 736 / 1000 correct (73.60)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 0.6918\n",
            "Checking accuracy on validation set\n",
            "Got 673 / 1000 correct (67.30)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 0.9680\n",
            "Checking accuracy on validation set\n",
            "Got 686 / 1000 correct (68.60)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 0.6771\n",
            "Checking accuracy on validation set\n",
            "Got 735 / 1000 correct (73.50)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 0.8356\n",
            "Checking accuracy on validation set\n",
            "Got 745 / 1000 correct (74.50)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 0.7316\n",
            "Checking accuracy on validation set\n",
            "Got 749 / 1000 correct (74.90)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 0.7657\n",
            "Checking accuracy on validation set\n",
            "Got 725 / 1000 correct (72.50)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 0.6672\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 0.7681\n",
            "Checking accuracy on validation set\n",
            "Got 733 / 1000 correct (73.30)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 0.7143\n",
            "Checking accuracy on validation set\n",
            "Got 775 / 1000 correct (77.50)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 0.6585\n",
            "Checking accuracy on validation set\n",
            "Got 745 / 1000 correct (74.50)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 0.6900\n",
            "Checking accuracy on validation set\n",
            "Got 791 / 1000 correct (79.10)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.5632\n",
            "Checking accuracy on validation set\n",
            "Got 795 / 1000 correct (79.50)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.6221\n",
            "Checking accuracy on validation set\n",
            "Got 757 / 1000 correct (75.70)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.6540\n",
            "Checking accuracy on validation set\n",
            "Got 788 / 1000 correct (78.80)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 0.9284\n",
            "Checking accuracy on validation set\n",
            "Got 766 / 1000 correct (76.60)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.6393\n",
            "Checking accuracy on validation set\n",
            "Got 769 / 1000 correct (76.90)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 0.5543\n",
            "Checking accuracy on validation set\n",
            "Got 807 / 1000 correct (80.70)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.5578\n",
            "Checking accuracy on validation set\n",
            "Got 813 / 1000 correct (81.30)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.6493\n",
            "Checking accuracy on validation set\n",
            "Got 776 / 1000 correct (77.60)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 0.6937\n",
            "Checking accuracy on validation set\n",
            "Got 783 / 1000 correct (78.30)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 0.6973\n",
            "Checking accuracy on validation set\n",
            "Got 739 / 1000 correct (73.90)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.6218\n",
            "Checking accuracy on validation set\n",
            "Got 796 / 1000 correct (79.60)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.6870\n",
            "Checking accuracy on validation set\n",
            "Got 787 / 1000 correct (78.70)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.7695\n",
            "Checking accuracy on validation set\n",
            "Got 812 / 1000 correct (81.20)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.9082\n",
            "Checking accuracy on validation set\n",
            "Got 750 / 1000 correct (75.00)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.5385\n",
            "Checking accuracy on validation set\n",
            "Got 798 / 1000 correct (79.80)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.4258\n",
            "Checking accuracy on validation set\n",
            "Got 797 / 1000 correct (79.70)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 0.5241\n",
            "Checking accuracy on validation set\n",
            "Got 798 / 1000 correct (79.80)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.4779\n",
            "Checking accuracy on validation set\n",
            "Got 784 / 1000 correct (78.40)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.7324\n",
            "Checking accuracy on validation set\n",
            "Got 815 / 1000 correct (81.50)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.7170\n",
            "Checking accuracy on validation set\n",
            "Got 797 / 1000 correct (79.70)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.5928\n",
            "Checking accuracy on validation set\n",
            "Got 793 / 1000 correct (79.30)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.7935\n",
            "Checking accuracy on validation set\n",
            "Got 807 / 1000 correct (80.70)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.5279\n",
            "Checking accuracy on validation set\n",
            "Got 813 / 1000 correct (81.30)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.5025\n",
            "Checking accuracy on validation set\n",
            "Got 791 / 1000 correct (79.10)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.7199\n",
            "Checking accuracy on validation set\n",
            "Got 814 / 1000 correct (81.40)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.6331\n",
            "Checking accuracy on validation set\n",
            "Got 801 / 1000 correct (80.10)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.6008\n",
            "Checking accuracy on validation set\n",
            "Got 802 / 1000 correct (80.20)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.5692\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.6853\n",
            "Checking accuracy on validation set\n",
            "Got 812 / 1000 correct (81.20)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.4740\n",
            "Checking accuracy on validation set\n",
            "Got 806 / 1000 correct (80.60)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.8133\n",
            "Checking accuracy on validation set\n",
            "Got 824 / 1000 correct (82.40)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.5928\n",
            "Checking accuracy on validation set\n",
            "Got 829 / 1000 correct (82.90)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.6576\n",
            "Checking accuracy on validation set\n",
            "Got 824 / 1000 correct (82.40)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.5687\n",
            "Checking accuracy on validation set\n",
            "Got 802 / 1000 correct (80.20)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.3314\n",
            "Checking accuracy on validation set\n",
            "Got 813 / 1000 correct (81.30)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.6263\n",
            "Checking accuracy on validation set\n",
            "Got 797 / 1000 correct (79.70)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.6491\n",
            "Checking accuracy on validation set\n",
            "Got 815 / 1000 correct (81.50)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.5072\n",
            "Checking accuracy on validation set\n",
            "Got 827 / 1000 correct (82.70)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.3904\n",
            "Checking accuracy on validation set\n",
            "Got 850 / 1000 correct (85.00)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.4085\n",
            "Checking accuracy on validation set\n",
            "Got 838 / 1000 correct (83.80)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.5357\n",
            "Checking accuracy on validation set\n",
            "Got 823 / 1000 correct (82.30)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.4488\n",
            "Checking accuracy on validation set\n",
            "Got 841 / 1000 correct (84.10)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.7515\n",
            "Checking accuracy on validation set\n",
            "Got 834 / 1000 correct (83.40)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.6092\n",
            "Checking accuracy on validation set\n",
            "Got 837 / 1000 correct (83.70)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.5596\n",
            "Checking accuracy on validation set\n",
            "Got 776 / 1000 correct (77.60)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.5594\n",
            "Checking accuracy on validation set\n",
            "Got 822 / 1000 correct (82.20)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.5287\n",
            "Checking accuracy on validation set\n",
            "Got 855 / 1000 correct (85.50)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.7262\n",
            "Checking accuracy on validation set\n",
            "Got 846 / 1000 correct (84.60)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.5378\n",
            "Checking accuracy on validation set\n",
            "Got 838 / 1000 correct (83.80)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.5358\n",
            "Checking accuracy on validation set\n",
            "Got 853 / 1000 correct (85.30)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.4880\n",
            "Checking accuracy on validation set\n",
            "Got 834 / 1000 correct (83.40)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.5292\n",
            "Checking accuracy on validation set\n",
            "Got 838 / 1000 correct (83.80)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 851 / 1000 correct (85.10)\n",
            "---- New lr = 0.00051 and weight_decay = 0.00085 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 4.1503\n",
            "Checking accuracy on validation set\n",
            "Got 102 / 1000 correct (10.20)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 1.9966\n",
            "Checking accuracy on validation set\n",
            "Got 375 / 1000 correct (37.50)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.6229\n",
            "Checking accuracy on validation set\n",
            "Got 425 / 1000 correct (42.50)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.2188\n",
            "Checking accuracy on validation set\n",
            "Got 469 / 1000 correct (46.90)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.5325\n",
            "Checking accuracy on validation set\n",
            "Got 507 / 1000 correct (50.70)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.4456\n",
            "Checking accuracy on validation set\n",
            "Got 533 / 1000 correct (53.30)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.1287\n",
            "Checking accuracy on validation set\n",
            "Got 533 / 1000 correct (53.30)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 1.0200\n",
            "Checking accuracy on validation set\n",
            "Got 580 / 1000 correct (58.00)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 1.1188\n",
            "Checking accuracy on validation set\n",
            "Got 637 / 1000 correct (63.70)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 0.9769\n",
            "Checking accuracy on validation set\n",
            "Got 629 / 1000 correct (62.90)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 0.9254\n",
            "Checking accuracy on validation set\n",
            "Got 635 / 1000 correct (63.50)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 0.9253\n",
            "Checking accuracy on validation set\n",
            "Got 644 / 1000 correct (64.40)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 0.7224\n",
            "Checking accuracy on validation set\n",
            "Got 707 / 1000 correct (70.70)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 0.8657\n",
            "Checking accuracy on validation set\n",
            "Got 699 / 1000 correct (69.90)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 0.8441\n",
            "Checking accuracy on validation set\n",
            "Got 686 / 1000 correct (68.60)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 1.0471\n",
            "Checking accuracy on validation set\n",
            "Got 709 / 1000 correct (70.90)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 0.8703\n",
            "Checking accuracy on validation set\n",
            "Got 727 / 1000 correct (72.70)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 0.7444\n",
            "Checking accuracy on validation set\n",
            "Got 730 / 1000 correct (73.00)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 0.8265\n",
            "Checking accuracy on validation set\n",
            "Got 753 / 1000 correct (75.30)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 0.6313\n",
            "Checking accuracy on validation set\n",
            "Got 731 / 1000 correct (73.10)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 0.7653\n",
            "Checking accuracy on validation set\n",
            "Got 748 / 1000 correct (74.80)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 0.5917\n",
            "Checking accuracy on validation set\n",
            "Got 763 / 1000 correct (76.30)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 0.5945\n",
            "Checking accuracy on validation set\n",
            "Got 751 / 1000 correct (75.10)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 0.5390\n",
            "Checking accuracy on validation set\n",
            "Got 770 / 1000 correct (77.00)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 0.6110\n",
            "Checking accuracy on validation set\n",
            "Got 773 / 1000 correct (77.30)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 0.5770\n",
            "Checking accuracy on validation set\n",
            "Got 770 / 1000 correct (77.00)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 0.7545\n",
            "Checking accuracy on validation set\n",
            "Got 715 / 1000 correct (71.50)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 0.7257\n",
            "Checking accuracy on validation set\n",
            "Got 758 / 1000 correct (75.80)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.6690\n",
            "Checking accuracy on validation set\n",
            "Got 746 / 1000 correct (74.60)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.6368\n",
            "Checking accuracy on validation set\n",
            "Got 707 / 1000 correct (70.70)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.8973\n",
            "Checking accuracy on validation set\n",
            "Got 779 / 1000 correct (77.90)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 0.6669\n",
            "Checking accuracy on validation set\n",
            "Got 788 / 1000 correct (78.80)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.7403\n",
            "Checking accuracy on validation set\n",
            "Got 758 / 1000 correct (75.80)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 0.7201\n",
            "Checking accuracy on validation set\n",
            "Got 731 / 1000 correct (73.10)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.5322\n",
            "Checking accuracy on validation set\n",
            "Got 774 / 1000 correct (77.40)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.8529\n",
            "Checking accuracy on validation set\n",
            "Got 774 / 1000 correct (77.40)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 0.9235\n",
            "Checking accuracy on validation set\n",
            "Got 791 / 1000 correct (79.10)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 0.6760\n",
            "Checking accuracy on validation set\n",
            "Got 791 / 1000 correct (79.10)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.5609\n",
            "Checking accuracy on validation set\n",
            "Got 747 / 1000 correct (74.70)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.6591\n",
            "Checking accuracy on validation set\n",
            "Got 785 / 1000 correct (78.50)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.5439\n",
            "Checking accuracy on validation set\n",
            "Got 805 / 1000 correct (80.50)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.7734\n",
            "Checking accuracy on validation set\n",
            "Got 776 / 1000 correct (77.60)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.4753\n",
            "Checking accuracy on validation set\n",
            "Got 770 / 1000 correct (77.00)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.6923\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 0.5035\n",
            "Checking accuracy on validation set\n",
            "Got 793 / 1000 correct (79.30)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.5697\n",
            "Checking accuracy on validation set\n",
            "Got 804 / 1000 correct (80.40)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.5093\n",
            "Checking accuracy on validation set\n",
            "Got 796 / 1000 correct (79.60)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.3828\n",
            "Checking accuracy on validation set\n",
            "Got 772 / 1000 correct (77.20)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.4392\n",
            "Checking accuracy on validation set\n",
            "Got 812 / 1000 correct (81.20)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.4809\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.5452\n",
            "Checking accuracy on validation set\n",
            "Got 805 / 1000 correct (80.50)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.6074\n",
            "Checking accuracy on validation set\n",
            "Got 807 / 1000 correct (80.70)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.5006\n",
            "Checking accuracy on validation set\n",
            "Got 797 / 1000 correct (79.70)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.4080\n",
            "Checking accuracy on validation set\n",
            "Got 828 / 1000 correct (82.80)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.4281\n",
            "Checking accuracy on validation set\n",
            "Got 797 / 1000 correct (79.70)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.7381\n",
            "Checking accuracy on validation set\n",
            "Got 821 / 1000 correct (82.10)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.4305\n",
            "Checking accuracy on validation set\n",
            "Got 832 / 1000 correct (83.20)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.5672\n",
            "Checking accuracy on validation set\n",
            "Got 814 / 1000 correct (81.40)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.5379\n",
            "Checking accuracy on validation set\n",
            "Got 786 / 1000 correct (78.60)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.6583\n",
            "Checking accuracy on validation set\n",
            "Got 811 / 1000 correct (81.10)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.6010\n",
            "Checking accuracy on validation set\n",
            "Got 800 / 1000 correct (80.00)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.5061\n",
            "Checking accuracy on validation set\n",
            "Got 805 / 1000 correct (80.50)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.5041\n",
            "Checking accuracy on validation set\n",
            "Got 831 / 1000 correct (83.10)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.6369\n",
            "Checking accuracy on validation set\n",
            "Got 816 / 1000 correct (81.60)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.4875\n",
            "Checking accuracy on validation set\n",
            "Got 838 / 1000 correct (83.80)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.4042\n",
            "Checking accuracy on validation set\n",
            "Got 825 / 1000 correct (82.50)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.3034\n",
            "Checking accuracy on validation set\n",
            "Got 835 / 1000 correct (83.50)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.6222\n",
            "Checking accuracy on validation set\n",
            "Got 818 / 1000 correct (81.80)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.2876\n",
            "Checking accuracy on validation set\n",
            "Got 818 / 1000 correct (81.80)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.4746\n",
            "Checking accuracy on validation set\n",
            "Got 809 / 1000 correct (80.90)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.4563\n",
            "Checking accuracy on validation set\n",
            "Got 837 / 1000 correct (83.70)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.6177\n",
            "Checking accuracy on validation set\n",
            "Got 824 / 1000 correct (82.40)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.3782\n",
            "Checking accuracy on validation set\n",
            "Got 801 / 1000 correct (80.10)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.4504\n",
            "Checking accuracy on validation set\n",
            "Got 822 / 1000 correct (82.20)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.4213\n",
            "Checking accuracy on validation set\n",
            "Got 837 / 1000 correct (83.70)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.6403\n",
            "Checking accuracy on validation set\n",
            "Got 808 / 1000 correct (80.80)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.4950\n",
            "Checking accuracy on validation set\n",
            "Got 801 / 1000 correct (80.10)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.5395\n",
            "Checking accuracy on validation set\n",
            "Got 815 / 1000 correct (81.50)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.5900\n",
            "Checking accuracy on validation set\n",
            "Got 816 / 1000 correct (81.60)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.7208\n",
            "Checking accuracy on validation set\n",
            "Got 812 / 1000 correct (81.20)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 784 / 1000 correct (78.40)\n",
            "---- New lr = 0.00050 and weight_decay = 0.00074 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 2.9900\n",
            "Checking accuracy on validation set\n",
            "Got 93 / 1000 correct (9.30)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 2.2026\n",
            "Checking accuracy on validation set\n",
            "Got 385 / 1000 correct (38.50)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.5419\n",
            "Checking accuracy on validation set\n",
            "Got 441 / 1000 correct (44.10)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.3920\n",
            "Checking accuracy on validation set\n",
            "Got 434 / 1000 correct (43.40)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.5053\n",
            "Checking accuracy on validation set\n",
            "Got 525 / 1000 correct (52.50)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.2482\n",
            "Checking accuracy on validation set\n",
            "Got 492 / 1000 correct (49.20)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.1716\n",
            "Checking accuracy on validation set\n",
            "Got 580 / 1000 correct (58.00)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 1.1843\n",
            "Checking accuracy on validation set\n",
            "Got 585 / 1000 correct (58.50)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 1.2650\n",
            "Checking accuracy on validation set\n",
            "Got 631 / 1000 correct (63.10)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 1.3987\n",
            "Checking accuracy on validation set\n",
            "Got 623 / 1000 correct (62.30)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 1.1273\n",
            "Checking accuracy on validation set\n",
            "Got 661 / 1000 correct (66.10)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 1.1902\n",
            "Checking accuracy on validation set\n",
            "Got 651 / 1000 correct (65.10)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 0.9114\n",
            "Checking accuracy on validation set\n",
            "Got 655 / 1000 correct (65.50)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 1.0766\n",
            "Checking accuracy on validation set\n",
            "Got 678 / 1000 correct (67.80)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 0.8231\n",
            "Checking accuracy on validation set\n",
            "Got 667 / 1000 correct (66.70)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 1.1207\n",
            "Checking accuracy on validation set\n",
            "Got 698 / 1000 correct (69.80)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 1.0396\n",
            "Checking accuracy on validation set\n",
            "Got 716 / 1000 correct (71.60)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 0.8301\n",
            "Checking accuracy on validation set\n",
            "Got 744 / 1000 correct (74.40)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 0.5622\n",
            "Checking accuracy on validation set\n",
            "Got 697 / 1000 correct (69.70)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 0.9594\n",
            "Checking accuracy on validation set\n",
            "Got 725 / 1000 correct (72.50)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 0.7337\n",
            "Checking accuracy on validation set\n",
            "Got 697 / 1000 correct (69.70)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 0.6436\n",
            "Checking accuracy on validation set\n",
            "Got 760 / 1000 correct (76.00)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 0.8499\n",
            "Checking accuracy on validation set\n",
            "Got 723 / 1000 correct (72.30)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 0.6876\n",
            "Checking accuracy on validation set\n",
            "Got 731 / 1000 correct (73.10)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 0.6217\n",
            "Checking accuracy on validation set\n",
            "Got 756 / 1000 correct (75.60)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 0.7299\n",
            "Checking accuracy on validation set\n",
            "Got 779 / 1000 correct (77.90)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 0.5290\n",
            "Checking accuracy on validation set\n",
            "Got 764 / 1000 correct (76.40)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 0.6067\n",
            "Checking accuracy on validation set\n",
            "Got 775 / 1000 correct (77.50)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.6803\n",
            "Checking accuracy on validation set\n",
            "Got 818 / 1000 correct (81.80)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.4996\n",
            "Checking accuracy on validation set\n",
            "Got 748 / 1000 correct (74.80)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.5941\n",
            "Checking accuracy on validation set\n",
            "Got 770 / 1000 correct (77.00)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 0.7592\n",
            "Checking accuracy on validation set\n",
            "Got 789 / 1000 correct (78.90)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.5923\n",
            "Checking accuracy on validation set\n",
            "Got 783 / 1000 correct (78.30)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 0.6799\n",
            "Checking accuracy on validation set\n",
            "Got 792 / 1000 correct (79.20)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.5093\n",
            "Checking accuracy on validation set\n",
            "Got 699 / 1000 correct (69.90)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.6285\n",
            "Checking accuracy on validation set\n",
            "Got 796 / 1000 correct (79.60)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 0.5999\n",
            "Checking accuracy on validation set\n",
            "Got 725 / 1000 correct (72.50)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 0.8450\n",
            "Checking accuracy on validation set\n",
            "Got 806 / 1000 correct (80.60)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.6552\n",
            "Checking accuracy on validation set\n",
            "Got 800 / 1000 correct (80.00)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.6284\n",
            "Checking accuracy on validation set\n",
            "Got 783 / 1000 correct (78.30)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.6110\n",
            "Checking accuracy on validation set\n",
            "Got 784 / 1000 correct (78.40)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.4533\n",
            "Checking accuracy on validation set\n",
            "Got 750 / 1000 correct (75.00)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.6417\n",
            "Checking accuracy on validation set\n",
            "Got 780 / 1000 correct (78.00)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.6912\n",
            "Checking accuracy on validation set\n",
            "Got 803 / 1000 correct (80.30)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 0.5965\n",
            "Checking accuracy on validation set\n",
            "Got 819 / 1000 correct (81.90)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.5194\n",
            "Checking accuracy on validation set\n",
            "Got 807 / 1000 correct (80.70)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.5114\n",
            "Checking accuracy on validation set\n",
            "Got 821 / 1000 correct (82.10)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.7473\n",
            "Checking accuracy on validation set\n",
            "Got 806 / 1000 correct (80.60)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.5990\n",
            "Checking accuracy on validation set\n",
            "Got 806 / 1000 correct (80.60)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.7416\n",
            "Checking accuracy on validation set\n",
            "Got 812 / 1000 correct (81.20)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.5837\n",
            "Checking accuracy on validation set\n",
            "Got 801 / 1000 correct (80.10)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.5577\n",
            "Checking accuracy on validation set\n",
            "Got 802 / 1000 correct (80.20)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.5457\n",
            "Checking accuracy on validation set\n",
            "Got 840 / 1000 correct (84.00)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.6175\n",
            "Checking accuracy on validation set\n",
            "Got 801 / 1000 correct (80.10)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.6260\n",
            "Checking accuracy on validation set\n",
            "Got 794 / 1000 correct (79.40)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.3134\n",
            "Checking accuracy on validation set\n",
            "Got 817 / 1000 correct (81.70)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.3607\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.5121\n",
            "Checking accuracy on validation set\n",
            "Got 803 / 1000 correct (80.30)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.9030\n",
            "Checking accuracy on validation set\n",
            "Got 799 / 1000 correct (79.90)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.6064\n",
            "Checking accuracy on validation set\n",
            "Got 808 / 1000 correct (80.80)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.5213\n",
            "Checking accuracy on validation set\n",
            "Got 808 / 1000 correct (80.80)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.7041\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.3822\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.5971\n",
            "Checking accuracy on validation set\n",
            "Got 834 / 1000 correct (83.40)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.3462\n",
            "Checking accuracy on validation set\n",
            "Got 834 / 1000 correct (83.40)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.2971\n",
            "Checking accuracy on validation set\n",
            "Got 818 / 1000 correct (81.80)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.5118\n",
            "Checking accuracy on validation set\n",
            "Got 823 / 1000 correct (82.30)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.4683\n",
            "Checking accuracy on validation set\n",
            "Got 825 / 1000 correct (82.50)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.4492\n",
            "Checking accuracy on validation set\n",
            "Got 799 / 1000 correct (79.90)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.5109\n",
            "Checking accuracy on validation set\n",
            "Got 831 / 1000 correct (83.10)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.5005\n",
            "Checking accuracy on validation set\n",
            "Got 818 / 1000 correct (81.80)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.3625\n",
            "Checking accuracy on validation set\n",
            "Got 812 / 1000 correct (81.20)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.6624\n",
            "Checking accuracy on validation set\n",
            "Got 815 / 1000 correct (81.50)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.5690\n",
            "Checking accuracy on validation set\n",
            "Got 831 / 1000 correct (83.10)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.7475\n",
            "Checking accuracy on validation set\n",
            "Got 784 / 1000 correct (78.40)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.7934\n",
            "Checking accuracy on validation set\n",
            "Got 852 / 1000 correct (85.20)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.4027\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.5645\n",
            "Checking accuracy on validation set\n",
            "Got 849 / 1000 correct (84.90)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.5980\n",
            "Checking accuracy on validation set\n",
            "Got 855 / 1000 correct (85.50)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.5367\n",
            "Checking accuracy on validation set\n",
            "Got 845 / 1000 correct (84.50)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 835 / 1000 correct (83.50)\n",
            "---- New lr = 0.00051 and weight_decay = 0.00075 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 3.0983\n",
            "Checking accuracy on validation set\n",
            "Got 108 / 1000 correct (10.80)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 1.7051\n",
            "Checking accuracy on validation set\n",
            "Got 420 / 1000 correct (42.00)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.5716\n",
            "Checking accuracy on validation set\n",
            "Got 464 / 1000 correct (46.40)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.3148\n",
            "Checking accuracy on validation set\n",
            "Got 456 / 1000 correct (45.60)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.0571\n",
            "Checking accuracy on validation set\n",
            "Got 536 / 1000 correct (53.60)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.2714\n",
            "Checking accuracy on validation set\n",
            "Got 611 / 1000 correct (61.10)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.0131\n",
            "Checking accuracy on validation set\n",
            "Got 604 / 1000 correct (60.40)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 1.4530\n",
            "Checking accuracy on validation set\n",
            "Got 561 / 1000 correct (56.10)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 1.2481\n",
            "Checking accuracy on validation set\n",
            "Got 556 / 1000 correct (55.60)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 1.1219\n",
            "Checking accuracy on validation set\n",
            "Got 654 / 1000 correct (65.40)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 1.1604\n",
            "Checking accuracy on validation set\n",
            "Got 606 / 1000 correct (60.60)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 0.9005\n",
            "Checking accuracy on validation set\n",
            "Got 694 / 1000 correct (69.40)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 0.8375\n",
            "Checking accuracy on validation set\n",
            "Got 718 / 1000 correct (71.80)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 1.0693\n",
            "Checking accuracy on validation set\n",
            "Got 669 / 1000 correct (66.90)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 1.0741\n",
            "Checking accuracy on validation set\n",
            "Got 676 / 1000 correct (67.60)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 0.8250\n",
            "Checking accuracy on validation set\n",
            "Got 721 / 1000 correct (72.10)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 0.8854\n",
            "Checking accuracy on validation set\n",
            "Got 719 / 1000 correct (71.90)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 0.9415\n",
            "Checking accuracy on validation set\n",
            "Got 708 / 1000 correct (70.80)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 0.7790\n",
            "Checking accuracy on validation set\n",
            "Got 673 / 1000 correct (67.30)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 0.7506\n",
            "Checking accuracy on validation set\n",
            "Got 704 / 1000 correct (70.40)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 0.5354\n",
            "Checking accuracy on validation set\n",
            "Got 773 / 1000 correct (77.30)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 0.6427\n",
            "Checking accuracy on validation set\n",
            "Got 707 / 1000 correct (70.70)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 0.9203\n",
            "Checking accuracy on validation set\n",
            "Got 706 / 1000 correct (70.60)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 0.6959\n",
            "Checking accuracy on validation set\n",
            "Got 760 / 1000 correct (76.00)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 0.8380\n",
            "Checking accuracy on validation set\n",
            "Got 784 / 1000 correct (78.40)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 0.6226\n",
            "Checking accuracy on validation set\n",
            "Got 791 / 1000 correct (79.10)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 0.5761\n",
            "Checking accuracy on validation set\n",
            "Got 788 / 1000 correct (78.80)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 0.6658\n",
            "Checking accuracy on validation set\n",
            "Got 742 / 1000 correct (74.20)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.5191\n",
            "Checking accuracy on validation set\n",
            "Got 770 / 1000 correct (77.00)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.5908\n",
            "Checking accuracy on validation set\n",
            "Got 761 / 1000 correct (76.10)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.6046\n",
            "Checking accuracy on validation set\n",
            "Got 767 / 1000 correct (76.70)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 0.6939\n",
            "Checking accuracy on validation set\n",
            "Got 797 / 1000 correct (79.70)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.5852\n",
            "Checking accuracy on validation set\n",
            "Got 787 / 1000 correct (78.70)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 0.6786\n",
            "Checking accuracy on validation set\n",
            "Got 771 / 1000 correct (77.10)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.5774\n",
            "Checking accuracy on validation set\n",
            "Got 790 / 1000 correct (79.00)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.5768\n",
            "Checking accuracy on validation set\n",
            "Got 785 / 1000 correct (78.50)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 0.7820\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 0.6224\n",
            "Checking accuracy on validation set\n",
            "Got 799 / 1000 correct (79.90)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.5407\n",
            "Checking accuracy on validation set\n",
            "Got 787 / 1000 correct (78.70)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.5748\n",
            "Checking accuracy on validation set\n",
            "Got 762 / 1000 correct (76.20)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.6400\n",
            "Checking accuracy on validation set\n",
            "Got 792 / 1000 correct (79.20)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.4682\n",
            "Checking accuracy on validation set\n",
            "Got 777 / 1000 correct (77.70)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.5524\n",
            "Checking accuracy on validation set\n",
            "Got 806 / 1000 correct (80.60)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.6756\n",
            "Checking accuracy on validation set\n",
            "Got 800 / 1000 correct (80.00)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 0.5301\n",
            "Checking accuracy on validation set\n",
            "Got 788 / 1000 correct (78.80)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.5190\n",
            "Checking accuracy on validation set\n",
            "Got 805 / 1000 correct (80.50)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.5214\n",
            "Checking accuracy on validation set\n",
            "Got 801 / 1000 correct (80.10)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.4764\n",
            "Checking accuracy on validation set\n",
            "Got 828 / 1000 correct (82.80)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.5956\n",
            "Checking accuracy on validation set\n",
            "Got 803 / 1000 correct (80.30)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.7601\n",
            "Checking accuracy on validation set\n",
            "Got 824 / 1000 correct (82.40)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.5999\n",
            "Checking accuracy on validation set\n",
            "Got 800 / 1000 correct (80.00)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.6388\n",
            "Checking accuracy on validation set\n",
            "Got 813 / 1000 correct (81.30)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.6397\n",
            "Checking accuracy on validation set\n",
            "Got 829 / 1000 correct (82.90)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.6234\n",
            "Checking accuracy on validation set\n",
            "Got 780 / 1000 correct (78.00)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.4905\n",
            "Checking accuracy on validation set\n",
            "Got 826 / 1000 correct (82.60)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.4038\n",
            "Checking accuracy on validation set\n",
            "Got 789 / 1000 correct (78.90)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.5944\n",
            "Checking accuracy on validation set\n",
            "Got 805 / 1000 correct (80.50)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.5494\n",
            "Checking accuracy on validation set\n",
            "Got 840 / 1000 correct (84.00)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.5485\n",
            "Checking accuracy on validation set\n",
            "Got 824 / 1000 correct (82.40)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.6186\n",
            "Checking accuracy on validation set\n",
            "Got 811 / 1000 correct (81.10)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.5281\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.5067\n",
            "Checking accuracy on validation set\n",
            "Got 819 / 1000 correct (81.90)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.4757\n",
            "Checking accuracy on validation set\n",
            "Got 832 / 1000 correct (83.20)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.5740\n",
            "Checking accuracy on validation set\n",
            "Got 805 / 1000 correct (80.50)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.4340\n",
            "Checking accuracy on validation set\n",
            "Got 808 / 1000 correct (80.80)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.7337\n",
            "Checking accuracy on validation set\n",
            "Got 822 / 1000 correct (82.20)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.4004\n",
            "Checking accuracy on validation set\n",
            "Got 814 / 1000 correct (81.40)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.5963\n",
            "Checking accuracy on validation set\n",
            "Got 829 / 1000 correct (82.90)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.3859\n",
            "Checking accuracy on validation set\n",
            "Got 836 / 1000 correct (83.60)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.4326\n",
            "Checking accuracy on validation set\n",
            "Got 817 / 1000 correct (81.70)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.8595\n",
            "Checking accuracy on validation set\n",
            "Got 837 / 1000 correct (83.70)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.5769\n",
            "Checking accuracy on validation set\n",
            "Got 824 / 1000 correct (82.40)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.6562\n",
            "Checking accuracy on validation set\n",
            "Got 752 / 1000 correct (75.20)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.6340\n",
            "Checking accuracy on validation set\n",
            "Got 844 / 1000 correct (84.40)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.5386\n",
            "Checking accuracy on validation set\n",
            "Got 828 / 1000 correct (82.80)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.4806\n",
            "Checking accuracy on validation set\n",
            "Got 817 / 1000 correct (81.70)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.4095\n",
            "Checking accuracy on validation set\n",
            "Got 824 / 1000 correct (82.40)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.5529\n",
            "Checking accuracy on validation set\n",
            "Got 846 / 1000 correct (84.60)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.5816\n",
            "Checking accuracy on validation set\n",
            "Got 845 / 1000 correct (84.50)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.4187\n",
            "Checking accuracy on validation set\n",
            "Got 837 / 1000 correct (83.70)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 814 / 1000 correct (81.40)\n",
            "---- New lr = 0.00051 and weight_decay = 0.00097 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 3.1594\n",
            "Checking accuracy on validation set\n",
            "Got 119 / 1000 correct (11.90)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 1.5933\n",
            "Checking accuracy on validation set\n",
            "Got 339 / 1000 correct (33.90)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.6613\n",
            "Checking accuracy on validation set\n",
            "Got 453 / 1000 correct (45.30)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.4392\n",
            "Checking accuracy on validation set\n",
            "Got 489 / 1000 correct (48.90)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.3757\n",
            "Checking accuracy on validation set\n",
            "Got 533 / 1000 correct (53.30)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.2997\n",
            "Checking accuracy on validation set\n",
            "Got 555 / 1000 correct (55.50)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.3592\n",
            "Checking accuracy on validation set\n",
            "Got 592 / 1000 correct (59.20)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 1.1877\n",
            "Checking accuracy on validation set\n",
            "Got 568 / 1000 correct (56.80)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 1.0762\n",
            "Checking accuracy on validation set\n",
            "Got 598 / 1000 correct (59.80)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 1.1156\n",
            "Checking accuracy on validation set\n",
            "Got 605 / 1000 correct (60.50)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 1.0227\n",
            "Checking accuracy on validation set\n",
            "Got 605 / 1000 correct (60.50)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 1.0193\n",
            "Checking accuracy on validation set\n",
            "Got 594 / 1000 correct (59.40)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 0.9157\n",
            "Checking accuracy on validation set\n",
            "Got 644 / 1000 correct (64.40)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 1.0235\n",
            "Checking accuracy on validation set\n",
            "Got 619 / 1000 correct (61.90)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 0.9720\n",
            "Checking accuracy on validation set\n",
            "Got 697 / 1000 correct (69.70)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 1.1894\n",
            "Checking accuracy on validation set\n",
            "Got 665 / 1000 correct (66.50)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 0.7986\n",
            "Checking accuracy on validation set\n",
            "Got 704 / 1000 correct (70.40)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 0.6674\n",
            "Checking accuracy on validation set\n",
            "Got 729 / 1000 correct (72.90)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 0.9065\n",
            "Checking accuracy on validation set\n",
            "Got 697 / 1000 correct (69.70)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 0.8165\n",
            "Checking accuracy on validation set\n",
            "Got 691 / 1000 correct (69.10)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 0.6364\n",
            "Checking accuracy on validation set\n",
            "Got 730 / 1000 correct (73.00)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 0.7128\n",
            "Checking accuracy on validation set\n",
            "Got 724 / 1000 correct (72.40)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 0.7546\n",
            "Checking accuracy on validation set\n",
            "Got 770 / 1000 correct (77.00)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 0.7555\n",
            "Checking accuracy on validation set\n",
            "Got 755 / 1000 correct (75.50)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 0.9789\n",
            "Checking accuracy on validation set\n",
            "Got 717 / 1000 correct (71.70)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 0.6425\n",
            "Checking accuracy on validation set\n",
            "Got 682 / 1000 correct (68.20)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 0.4944\n",
            "Checking accuracy on validation set\n",
            "Got 761 / 1000 correct (76.10)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 0.8772\n",
            "Checking accuracy on validation set\n",
            "Got 800 / 1000 correct (80.00)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.6425\n",
            "Checking accuracy on validation set\n",
            "Got 735 / 1000 correct (73.50)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.8637\n",
            "Checking accuracy on validation set\n",
            "Got 754 / 1000 correct (75.40)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.5974\n",
            "Checking accuracy on validation set\n",
            "Got 746 / 1000 correct (74.60)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 0.5531\n",
            "Checking accuracy on validation set\n",
            "Got 769 / 1000 correct (76.90)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.5518\n",
            "Checking accuracy on validation set\n",
            "Got 686 / 1000 correct (68.60)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 0.6001\n",
            "Checking accuracy on validation set\n",
            "Got 788 / 1000 correct (78.80)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.5977\n",
            "Checking accuracy on validation set\n",
            "Got 793 / 1000 correct (79.30)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.8598\n",
            "Checking accuracy on validation set\n",
            "Got 741 / 1000 correct (74.10)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 0.6167\n",
            "Checking accuracy on validation set\n",
            "Got 784 / 1000 correct (78.40)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 0.5918\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.5196\n",
            "Checking accuracy on validation set\n",
            "Got 818 / 1000 correct (81.80)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.5925\n",
            "Checking accuracy on validation set\n",
            "Got 787 / 1000 correct (78.70)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.5271\n",
            "Checking accuracy on validation set\n",
            "Got 805 / 1000 correct (80.50)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.6188\n",
            "Checking accuracy on validation set\n",
            "Got 749 / 1000 correct (74.90)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.7416\n",
            "Checking accuracy on validation set\n",
            "Got 785 / 1000 correct (78.50)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.6546\n",
            "Checking accuracy on validation set\n",
            "Got 722 / 1000 correct (72.20)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 0.5793\n",
            "Checking accuracy on validation set\n",
            "Got 773 / 1000 correct (77.30)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.7126\n",
            "Checking accuracy on validation set\n",
            "Got 798 / 1000 correct (79.80)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.7926\n",
            "Checking accuracy on validation set\n",
            "Got 808 / 1000 correct (80.80)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.5117\n",
            "Checking accuracy on validation set\n",
            "Got 815 / 1000 correct (81.50)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.5511\n",
            "Checking accuracy on validation set\n",
            "Got 773 / 1000 correct (77.30)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.5697\n",
            "Checking accuracy on validation set\n",
            "Got 810 / 1000 correct (81.00)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.6638\n",
            "Checking accuracy on validation set\n",
            "Got 793 / 1000 correct (79.30)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.5790\n",
            "Checking accuracy on validation set\n",
            "Got 801 / 1000 correct (80.10)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.5849\n",
            "Checking accuracy on validation set\n",
            "Got 800 / 1000 correct (80.00)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.8372\n",
            "Checking accuracy on validation set\n",
            "Got 810 / 1000 correct (81.00)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.4263\n",
            "Checking accuracy on validation set\n",
            "Got 818 / 1000 correct (81.80)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.4212\n",
            "Checking accuracy on validation set\n",
            "Got 828 / 1000 correct (82.80)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.5369\n",
            "Checking accuracy on validation set\n",
            "Got 832 / 1000 correct (83.20)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.3156\n",
            "Checking accuracy on validation set\n",
            "Got 827 / 1000 correct (82.70)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.7106\n",
            "Checking accuracy on validation set\n",
            "Got 841 / 1000 correct (84.10)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.6470\n",
            "Checking accuracy on validation set\n",
            "Got 819 / 1000 correct (81.90)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.4659\n",
            "Checking accuracy on validation set\n",
            "Got 829 / 1000 correct (82.90)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.4866\n",
            "Checking accuracy on validation set\n",
            "Got 798 / 1000 correct (79.80)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.5374\n",
            "Checking accuracy on validation set\n",
            "Got 824 / 1000 correct (82.40)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.5527\n",
            "Checking accuracy on validation set\n",
            "Got 773 / 1000 correct (77.30)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.5973\n",
            "Checking accuracy on validation set\n",
            "Got 767 / 1000 correct (76.70)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.7957\n",
            "Checking accuracy on validation set\n",
            "Got 842 / 1000 correct (84.20)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.3198\n",
            "Checking accuracy on validation set\n",
            "Got 794 / 1000 correct (79.40)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.6724\n",
            "Checking accuracy on validation set\n",
            "Got 813 / 1000 correct (81.30)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.5941\n",
            "Checking accuracy on validation set\n",
            "Got 790 / 1000 correct (79.00)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.7354\n",
            "Checking accuracy on validation set\n",
            "Got 838 / 1000 correct (83.80)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.6024\n",
            "Checking accuracy on validation set\n",
            "Got 840 / 1000 correct (84.00)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.5895\n",
            "Checking accuracy on validation set\n",
            "Got 860 / 1000 correct (86.00)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.6464\n",
            "Checking accuracy on validation set\n",
            "Got 835 / 1000 correct (83.50)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.4896\n",
            "Checking accuracy on validation set\n",
            "Got 809 / 1000 correct (80.90)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.3090\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.2924\n",
            "Checking accuracy on validation set\n",
            "Got 811 / 1000 correct (81.10)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.4450\n",
            "Checking accuracy on validation set\n",
            "Got 812 / 1000 correct (81.20)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.4467\n",
            "Checking accuracy on validation set\n",
            "Got 826 / 1000 correct (82.60)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.4059\n",
            "Checking accuracy on validation set\n",
            "Got 818 / 1000 correct (81.80)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.5869\n",
            "Checking accuracy on validation set\n",
            "Got 831 / 1000 correct (83.10)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 853 / 1000 correct (85.30)\n",
            "---- New lr = 0.00050 and weight_decay = 0.00091 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 3.5292\n",
            "Checking accuracy on validation set\n",
            "Got 99 / 1000 correct (9.90)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 1.7849\n",
            "Checking accuracy on validation set\n",
            "Got 401 / 1000 correct (40.10)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.5000\n",
            "Checking accuracy on validation set\n",
            "Got 441 / 1000 correct (44.10)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.1252\n",
            "Checking accuracy on validation set\n",
            "Got 483 / 1000 correct (48.30)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.2558\n",
            "Checking accuracy on validation set\n",
            "Got 527 / 1000 correct (52.70)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.2677\n",
            "Checking accuracy on validation set\n",
            "Got 512 / 1000 correct (51.20)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.1492\n",
            "Checking accuracy on validation set\n",
            "Got 555 / 1000 correct (55.50)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 0.9287\n",
            "Checking accuracy on validation set\n",
            "Got 597 / 1000 correct (59.70)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 0.9018\n",
            "Checking accuracy on validation set\n",
            "Got 601 / 1000 correct (60.10)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 1.0772\n",
            "Checking accuracy on validation set\n",
            "Got 642 / 1000 correct (64.20)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 1.2542\n",
            "Checking accuracy on validation set\n",
            "Got 645 / 1000 correct (64.50)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 1.0525\n",
            "Checking accuracy on validation set\n",
            "Got 654 / 1000 correct (65.40)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 1.0601\n",
            "Checking accuracy on validation set\n",
            "Got 659 / 1000 correct (65.90)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 0.8084\n",
            "Checking accuracy on validation set\n",
            "Got 685 / 1000 correct (68.50)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 0.9537\n",
            "Checking accuracy on validation set\n",
            "Got 724 / 1000 correct (72.40)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 0.7780\n",
            "Checking accuracy on validation set\n",
            "Got 700 / 1000 correct (70.00)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 0.9292\n",
            "Checking accuracy on validation set\n",
            "Got 702 / 1000 correct (70.20)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 0.7438\n",
            "Checking accuracy on validation set\n",
            "Got 754 / 1000 correct (75.40)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 1.0062\n",
            "Checking accuracy on validation set\n",
            "Got 709 / 1000 correct (70.90)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 0.6028\n",
            "Checking accuracy on validation set\n",
            "Got 759 / 1000 correct (75.90)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 0.9864\n",
            "Checking accuracy on validation set\n",
            "Got 698 / 1000 correct (69.80)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 0.7675\n",
            "Checking accuracy on validation set\n",
            "Got 703 / 1000 correct (70.30)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 0.8466\n",
            "Checking accuracy on validation set\n",
            "Got 754 / 1000 correct (75.40)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 0.8401\n",
            "Checking accuracy on validation set\n",
            "Got 756 / 1000 correct (75.60)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 0.9811\n",
            "Checking accuracy on validation set\n",
            "Got 750 / 1000 correct (75.00)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 0.4526\n",
            "Checking accuracy on validation set\n",
            "Got 784 / 1000 correct (78.40)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 0.6802\n",
            "Checking accuracy on validation set\n",
            "Got 738 / 1000 correct (73.80)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 0.5794\n",
            "Checking accuracy on validation set\n",
            "Got 768 / 1000 correct (76.80)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.8341\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.8173\n",
            "Checking accuracy on validation set\n",
            "Got 780 / 1000 correct (78.00)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.8851\n",
            "Checking accuracy on validation set\n",
            "Got 780 / 1000 correct (78.00)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 0.8478\n",
            "Checking accuracy on validation set\n",
            "Got 762 / 1000 correct (76.20)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.6617\n",
            "Checking accuracy on validation set\n",
            "Got 800 / 1000 correct (80.00)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 0.7000\n",
            "Checking accuracy on validation set\n",
            "Got 778 / 1000 correct (77.80)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.6212\n",
            "Checking accuracy on validation set\n",
            "Got 789 / 1000 correct (78.90)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.7232\n",
            "Checking accuracy on validation set\n",
            "Got 754 / 1000 correct (75.40)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 0.8025\n",
            "Checking accuracy on validation set\n",
            "Got 805 / 1000 correct (80.50)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 0.6613\n",
            "Checking accuracy on validation set\n",
            "Got 791 / 1000 correct (79.10)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.6481\n",
            "Checking accuracy on validation set\n",
            "Got 758 / 1000 correct (75.80)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.9401\n",
            "Checking accuracy on validation set\n",
            "Got 753 / 1000 correct (75.30)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.5713\n",
            "Checking accuracy on validation set\n",
            "Got 800 / 1000 correct (80.00)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.6121\n",
            "Checking accuracy on validation set\n",
            "Got 789 / 1000 correct (78.90)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.6714\n",
            "Checking accuracy on validation set\n",
            "Got 802 / 1000 correct (80.20)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.5078\n",
            "Checking accuracy on validation set\n",
            "Got 801 / 1000 correct (80.10)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 0.7692\n",
            "Checking accuracy on validation set\n",
            "Got 802 / 1000 correct (80.20)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.7845\n",
            "Checking accuracy on validation set\n",
            "Got 815 / 1000 correct (81.50)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.5401\n",
            "Checking accuracy on validation set\n",
            "Got 818 / 1000 correct (81.80)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.3791\n",
            "Checking accuracy on validation set\n",
            "Got 798 / 1000 correct (79.80)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.5391\n",
            "Checking accuracy on validation set\n",
            "Got 791 / 1000 correct (79.10)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.5360\n",
            "Checking accuracy on validation set\n",
            "Got 814 / 1000 correct (81.40)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.3402\n",
            "Checking accuracy on validation set\n",
            "Got 808 / 1000 correct (80.80)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.8290\n",
            "Checking accuracy on validation set\n",
            "Got 815 / 1000 correct (81.50)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.6662\n",
            "Checking accuracy on validation set\n",
            "Got 801 / 1000 correct (80.10)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.5482\n",
            "Checking accuracy on validation set\n",
            "Got 810 / 1000 correct (81.00)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.7195\n",
            "Checking accuracy on validation set\n",
            "Got 801 / 1000 correct (80.10)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.5386\n",
            "Checking accuracy on validation set\n",
            "Got 834 / 1000 correct (83.40)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.4818\n",
            "Checking accuracy on validation set\n",
            "Got 797 / 1000 correct (79.70)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.4594\n",
            "Checking accuracy on validation set\n",
            "Got 843 / 1000 correct (84.30)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.6278\n",
            "Checking accuracy on validation set\n",
            "Got 831 / 1000 correct (83.10)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.4539\n",
            "Checking accuracy on validation set\n",
            "Got 785 / 1000 correct (78.50)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.7451\n",
            "Checking accuracy on validation set\n",
            "Got 771 / 1000 correct (77.10)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.5179\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.5149\n",
            "Checking accuracy on validation set\n",
            "Got 818 / 1000 correct (81.80)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.5291\n",
            "Checking accuracy on validation set\n",
            "Got 813 / 1000 correct (81.30)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.8229\n",
            "Checking accuracy on validation set\n",
            "Got 834 / 1000 correct (83.40)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.4574\n",
            "Checking accuracy on validation set\n",
            "Got 841 / 1000 correct (84.10)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.4412\n",
            "Checking accuracy on validation set\n",
            "Got 821 / 1000 correct (82.10)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.6149\n",
            "Checking accuracy on validation set\n",
            "Got 827 / 1000 correct (82.70)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.5153\n",
            "Checking accuracy on validation set\n",
            "Got 836 / 1000 correct (83.60)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.5199\n",
            "Checking accuracy on validation set\n",
            "Got 824 / 1000 correct (82.40)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.5540\n",
            "Checking accuracy on validation set\n",
            "Got 829 / 1000 correct (82.90)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.3660\n",
            "Checking accuracy on validation set\n",
            "Got 817 / 1000 correct (81.70)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.5438\n",
            "Checking accuracy on validation set\n",
            "Got 730 / 1000 correct (73.00)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.3687\n",
            "Checking accuracy on validation set\n",
            "Got 828 / 1000 correct (82.80)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.3749\n",
            "Checking accuracy on validation set\n",
            "Got 824 / 1000 correct (82.40)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.4254\n",
            "Checking accuracy on validation set\n",
            "Got 832 / 1000 correct (83.20)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.3552\n",
            "Checking accuracy on validation set\n",
            "Got 829 / 1000 correct (82.90)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.3540\n",
            "Checking accuracy on validation set\n",
            "Got 829 / 1000 correct (82.90)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.4388\n",
            "Checking accuracy on validation set\n",
            "Got 804 / 1000 correct (80.40)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.4890\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 847 / 1000 correct (84.70)\n",
            "---- New lr = 0.00050 and weight_decay = 0.00082 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 3.9432\n",
            "Checking accuracy on validation set\n",
            "Got 93 / 1000 correct (9.30)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 1.7274\n",
            "Checking accuracy on validation set\n",
            "Got 361 / 1000 correct (36.10)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.6559\n",
            "Checking accuracy on validation set\n",
            "Got 465 / 1000 correct (46.50)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.4980\n",
            "Checking accuracy on validation set\n",
            "Got 463 / 1000 correct (46.30)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.3796\n",
            "Checking accuracy on validation set\n",
            "Got 529 / 1000 correct (52.90)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.4555\n",
            "Checking accuracy on validation set\n",
            "Got 507 / 1000 correct (50.70)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.2790\n",
            "Checking accuracy on validation set\n",
            "Got 588 / 1000 correct (58.80)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 1.1431\n",
            "Checking accuracy on validation set\n",
            "Got 564 / 1000 correct (56.40)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 1.1642\n",
            "Checking accuracy on validation set\n",
            "Got 588 / 1000 correct (58.80)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 1.1614\n",
            "Checking accuracy on validation set\n",
            "Got 596 / 1000 correct (59.60)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 0.9741\n",
            "Checking accuracy on validation set\n",
            "Got 625 / 1000 correct (62.50)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 1.0658\n",
            "Checking accuracy on validation set\n",
            "Got 656 / 1000 correct (65.60)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 0.9641\n",
            "Checking accuracy on validation set\n",
            "Got 656 / 1000 correct (65.60)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 1.2149\n",
            "Checking accuracy on validation set\n",
            "Got 678 / 1000 correct (67.80)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 0.9369\n",
            "Checking accuracy on validation set\n",
            "Got 689 / 1000 correct (68.90)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 0.9825\n",
            "Checking accuracy on validation set\n",
            "Got 669 / 1000 correct (66.90)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 1.0409\n",
            "Checking accuracy on validation set\n",
            "Got 691 / 1000 correct (69.10)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 1.0830\n",
            "Checking accuracy on validation set\n",
            "Got 707 / 1000 correct (70.70)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 0.8835\n",
            "Checking accuracy on validation set\n",
            "Got 719 / 1000 correct (71.90)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 1.1544\n",
            "Checking accuracy on validation set\n",
            "Got 672 / 1000 correct (67.20)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 0.4849\n",
            "Checking accuracy on validation set\n",
            "Got 734 / 1000 correct (73.40)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 0.6828\n",
            "Checking accuracy on validation set\n",
            "Got 783 / 1000 correct (78.30)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 0.9992\n",
            "Checking accuracy on validation set\n",
            "Got 748 / 1000 correct (74.80)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 1.0991\n",
            "Checking accuracy on validation set\n",
            "Got 716 / 1000 correct (71.60)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 0.7112\n",
            "Checking accuracy on validation set\n",
            "Got 781 / 1000 correct (78.10)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 0.6186\n",
            "Checking accuracy on validation set\n",
            "Got 697 / 1000 correct (69.70)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 1.0166\n",
            "Checking accuracy on validation set\n",
            "Got 784 / 1000 correct (78.40)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 0.4893\n",
            "Checking accuracy on validation set\n",
            "Got 766 / 1000 correct (76.60)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.6277\n",
            "Checking accuracy on validation set\n",
            "Got 758 / 1000 correct (75.80)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.5451\n",
            "Checking accuracy on validation set\n",
            "Got 779 / 1000 correct (77.90)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.7836\n",
            "Checking accuracy on validation set\n",
            "Got 772 / 1000 correct (77.20)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 0.8119\n",
            "Checking accuracy on validation set\n",
            "Got 753 / 1000 correct (75.30)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.6852\n",
            "Checking accuracy on validation set\n",
            "Got 743 / 1000 correct (74.30)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 0.6550\n",
            "Checking accuracy on validation set\n",
            "Got 801 / 1000 correct (80.10)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.5477\n",
            "Checking accuracy on validation set\n",
            "Got 784 / 1000 correct (78.40)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.5926\n",
            "Checking accuracy on validation set\n",
            "Got 771 / 1000 correct (77.10)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 0.8370\n",
            "Checking accuracy on validation set\n",
            "Got 758 / 1000 correct (75.80)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 0.5467\n",
            "Checking accuracy on validation set\n",
            "Got 780 / 1000 correct (78.00)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.4604\n",
            "Checking accuracy on validation set\n",
            "Got 797 / 1000 correct (79.70)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.7892\n",
            "Checking accuracy on validation set\n",
            "Got 785 / 1000 correct (78.50)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.7822\n",
            "Checking accuracy on validation set\n",
            "Got 819 / 1000 correct (81.90)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.8198\n",
            "Checking accuracy on validation set\n",
            "Got 746 / 1000 correct (74.60)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.5756\n",
            "Checking accuracy on validation set\n",
            "Got 773 / 1000 correct (77.30)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.6554\n",
            "Checking accuracy on validation set\n",
            "Got 798 / 1000 correct (79.80)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 0.7153\n",
            "Checking accuracy on validation set\n",
            "Got 784 / 1000 correct (78.40)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.6718\n",
            "Checking accuracy on validation set\n",
            "Got 771 / 1000 correct (77.10)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.5048\n",
            "Checking accuracy on validation set\n",
            "Got 804 / 1000 correct (80.40)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.6823\n",
            "Checking accuracy on validation set\n",
            "Got 790 / 1000 correct (79.00)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.6810\n",
            "Checking accuracy on validation set\n",
            "Got 817 / 1000 correct (81.70)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.4934\n",
            "Checking accuracy on validation set\n",
            "Got 819 / 1000 correct (81.90)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.7071\n",
            "Checking accuracy on validation set\n",
            "Got 741 / 1000 correct (74.10)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.5266\n",
            "Checking accuracy on validation set\n",
            "Got 793 / 1000 correct (79.30)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.6158\n",
            "Checking accuracy on validation set\n",
            "Got 822 / 1000 correct (82.20)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.5155\n",
            "Checking accuracy on validation set\n",
            "Got 807 / 1000 correct (80.70)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.4234\n",
            "Checking accuracy on validation set\n",
            "Got 768 / 1000 correct (76.80)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.3946\n",
            "Checking accuracy on validation set\n",
            "Got 826 / 1000 correct (82.60)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.4830\n",
            "Checking accuracy on validation set\n",
            "Got 777 / 1000 correct (77.70)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.6948\n",
            "Checking accuracy on validation set\n",
            "Got 812 / 1000 correct (81.20)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.5233\n",
            "Checking accuracy on validation set\n",
            "Got 829 / 1000 correct (82.90)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.5337\n",
            "Checking accuracy on validation set\n",
            "Got 806 / 1000 correct (80.60)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.5247\n",
            "Checking accuracy on validation set\n",
            "Got 791 / 1000 correct (79.10)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.5005\n",
            "Checking accuracy on validation set\n",
            "Got 834 / 1000 correct (83.40)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.4929\n",
            "Checking accuracy on validation set\n",
            "Got 841 / 1000 correct (84.10)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.6678\n",
            "Checking accuracy on validation set\n",
            "Got 789 / 1000 correct (78.90)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.7033\n",
            "Checking accuracy on validation set\n",
            "Got 828 / 1000 correct (82.80)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.4644\n",
            "Checking accuracy on validation set\n",
            "Got 827 / 1000 correct (82.70)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.3878\n",
            "Checking accuracy on validation set\n",
            "Got 825 / 1000 correct (82.50)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.5264\n",
            "Checking accuracy on validation set\n",
            "Got 811 / 1000 correct (81.10)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.5193\n",
            "Checking accuracy on validation set\n",
            "Got 815 / 1000 correct (81.50)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.6293\n",
            "Checking accuracy on validation set\n",
            "Got 834 / 1000 correct (83.40)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.5938\n",
            "Checking accuracy on validation set\n",
            "Got 829 / 1000 correct (82.90)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.3853\n",
            "Checking accuracy on validation set\n",
            "Got 858 / 1000 correct (85.80)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.2763\n",
            "Checking accuracy on validation set\n",
            "Got 819 / 1000 correct (81.90)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.3056\n",
            "Checking accuracy on validation set\n",
            "Got 804 / 1000 correct (80.40)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.4894\n",
            "Checking accuracy on validation set\n",
            "Got 828 / 1000 correct (82.80)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.5017\n",
            "Checking accuracy on validation set\n",
            "Got 844 / 1000 correct (84.40)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.4624\n",
            "Checking accuracy on validation set\n",
            "Got 824 / 1000 correct (82.40)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.7533\n",
            "Checking accuracy on validation set\n",
            "Got 850 / 1000 correct (85.00)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.3735\n",
            "Checking accuracy on validation set\n",
            "Got 849 / 1000 correct (84.90)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.4570\n",
            "Checking accuracy on validation set\n",
            "Got 832 / 1000 correct (83.20)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 814 / 1000 correct (81.40)\n",
            "---- New lr = 0.00055 and weight_decay = 0.00098 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 3.3854\n",
            "Checking accuracy on validation set\n",
            "Got 116 / 1000 correct (11.60)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 1.6567\n",
            "Checking accuracy on validation set\n",
            "Got 363 / 1000 correct (36.30)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.4229\n",
            "Checking accuracy on validation set\n",
            "Got 389 / 1000 correct (38.90)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.5334\n",
            "Checking accuracy on validation set\n",
            "Got 471 / 1000 correct (47.10)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.4760\n",
            "Checking accuracy on validation set\n",
            "Got 506 / 1000 correct (50.60)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.1962\n",
            "Checking accuracy on validation set\n",
            "Got 533 / 1000 correct (53.30)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.2452\n",
            "Checking accuracy on validation set\n",
            "Got 558 / 1000 correct (55.80)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 1.2508\n",
            "Checking accuracy on validation set\n",
            "Got 583 / 1000 correct (58.30)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 1.3188\n",
            "Checking accuracy on validation set\n",
            "Got 548 / 1000 correct (54.80)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 0.9639\n",
            "Checking accuracy on validation set\n",
            "Got 615 / 1000 correct (61.50)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 1.0628\n",
            "Checking accuracy on validation set\n",
            "Got 647 / 1000 correct (64.70)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 1.0704\n",
            "Checking accuracy on validation set\n",
            "Got 665 / 1000 correct (66.50)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 0.7760\n",
            "Checking accuracy on validation set\n",
            "Got 661 / 1000 correct (66.10)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 0.9646\n",
            "Checking accuracy on validation set\n",
            "Got 671 / 1000 correct (67.10)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 1.0256\n",
            "Checking accuracy on validation set\n",
            "Got 611 / 1000 correct (61.10)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 0.8378\n",
            "Checking accuracy on validation set\n",
            "Got 649 / 1000 correct (64.90)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 0.9279\n",
            "Checking accuracy on validation set\n",
            "Got 674 / 1000 correct (67.40)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 0.7689\n",
            "Checking accuracy on validation set\n",
            "Got 701 / 1000 correct (70.10)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 1.0087\n",
            "Checking accuracy on validation set\n",
            "Got 713 / 1000 correct (71.30)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 0.8847\n",
            "Checking accuracy on validation set\n",
            "Got 728 / 1000 correct (72.80)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 0.9505\n",
            "Checking accuracy on validation set\n",
            "Got 703 / 1000 correct (70.30)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 0.9368\n",
            "Checking accuracy on validation set\n",
            "Got 729 / 1000 correct (72.90)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 0.9765\n",
            "Checking accuracy on validation set\n",
            "Got 677 / 1000 correct (67.70)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 0.7841\n",
            "Checking accuracy on validation set\n",
            "Got 736 / 1000 correct (73.60)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 0.7116\n",
            "Checking accuracy on validation set\n",
            "Got 742 / 1000 correct (74.20)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 0.7271\n",
            "Checking accuracy on validation set\n",
            "Got 737 / 1000 correct (73.70)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 0.6250\n",
            "Checking accuracy on validation set\n",
            "Got 686 / 1000 correct (68.60)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 0.7341\n",
            "Checking accuracy on validation set\n",
            "Got 762 / 1000 correct (76.20)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.6676\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.7619\n",
            "Checking accuracy on validation set\n",
            "Got 716 / 1000 correct (71.60)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.5638\n",
            "Checking accuracy on validation set\n",
            "Got 781 / 1000 correct (78.10)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 0.6864\n",
            "Checking accuracy on validation set\n",
            "Got 747 / 1000 correct (74.70)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.5565\n",
            "Checking accuracy on validation set\n",
            "Got 778 / 1000 correct (77.80)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 0.7806\n",
            "Checking accuracy on validation set\n",
            "Got 731 / 1000 correct (73.10)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.6519\n",
            "Checking accuracy on validation set\n",
            "Got 759 / 1000 correct (75.90)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.8548\n",
            "Checking accuracy on validation set\n",
            "Got 689 / 1000 correct (68.90)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 1.0225\n",
            "Checking accuracy on validation set\n",
            "Got 773 / 1000 correct (77.30)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 0.5555\n",
            "Checking accuracy on validation set\n",
            "Got 771 / 1000 correct (77.10)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.6534\n",
            "Checking accuracy on validation set\n",
            "Got 752 / 1000 correct (75.20)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.6509\n",
            "Checking accuracy on validation set\n",
            "Got 781 / 1000 correct (78.10)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.4907\n",
            "Checking accuracy on validation set\n",
            "Got 796 / 1000 correct (79.60)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.8136\n",
            "Checking accuracy on validation set\n",
            "Got 783 / 1000 correct (78.30)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.4548\n",
            "Checking accuracy on validation set\n",
            "Got 807 / 1000 correct (80.70)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.6562\n",
            "Checking accuracy on validation set\n",
            "Got 778 / 1000 correct (77.80)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 0.7857\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.7260\n",
            "Checking accuracy on validation set\n",
            "Got 809 / 1000 correct (80.90)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.5587\n",
            "Checking accuracy on validation set\n",
            "Got 746 / 1000 correct (74.60)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.5603\n",
            "Checking accuracy on validation set\n",
            "Got 767 / 1000 correct (76.70)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.3230\n",
            "Checking accuracy on validation set\n",
            "Got 763 / 1000 correct (76.30)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.6212\n",
            "Checking accuracy on validation set\n",
            "Got 790 / 1000 correct (79.00)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.5422\n",
            "Checking accuracy on validation set\n",
            "Got 809 / 1000 correct (80.90)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.5094\n",
            "Checking accuracy on validation set\n",
            "Got 789 / 1000 correct (78.90)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.5657\n",
            "Checking accuracy on validation set\n",
            "Got 810 / 1000 correct (81.00)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.5565\n",
            "Checking accuracy on validation set\n",
            "Got 797 / 1000 correct (79.70)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.5747\n",
            "Checking accuracy on validation set\n",
            "Got 800 / 1000 correct (80.00)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.4881\n",
            "Checking accuracy on validation set\n",
            "Got 821 / 1000 correct (82.10)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.4859\n",
            "Checking accuracy on validation set\n",
            "Got 821 / 1000 correct (82.10)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.4174\n",
            "Checking accuracy on validation set\n",
            "Got 831 / 1000 correct (83.10)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.3539\n",
            "Checking accuracy on validation set\n",
            "Got 798 / 1000 correct (79.80)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.5127\n",
            "Checking accuracy on validation set\n",
            "Got 810 / 1000 correct (81.00)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.5854\n",
            "Checking accuracy on validation set\n",
            "Got 810 / 1000 correct (81.00)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.5352\n",
            "Checking accuracy on validation set\n",
            "Got 820 / 1000 correct (82.00)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.3682\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.5187\n",
            "Checking accuracy on validation set\n",
            "Got 804 / 1000 correct (80.40)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.8194\n",
            "Checking accuracy on validation set\n",
            "Got 789 / 1000 correct (78.90)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.6189\n",
            "Checking accuracy on validation set\n",
            "Got 807 / 1000 correct (80.70)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.5894\n",
            "Checking accuracy on validation set\n",
            "Got 833 / 1000 correct (83.30)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.5716\n",
            "Checking accuracy on validation set\n",
            "Got 815 / 1000 correct (81.50)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.3859\n",
            "Checking accuracy on validation set\n",
            "Got 786 / 1000 correct (78.60)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.4780\n",
            "Checking accuracy on validation set\n",
            "Got 823 / 1000 correct (82.30)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.5495\n",
            "Checking accuracy on validation set\n",
            "Got 819 / 1000 correct (81.90)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.5138\n",
            "Checking accuracy on validation set\n",
            "Got 813 / 1000 correct (81.30)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.3479\n",
            "Checking accuracy on validation set\n",
            "Got 815 / 1000 correct (81.50)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.3710\n",
            "Checking accuracy on validation set\n",
            "Got 819 / 1000 correct (81.90)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.3846\n",
            "Checking accuracy on validation set\n",
            "Got 837 / 1000 correct (83.70)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.3355\n",
            "Checking accuracy on validation set\n",
            "Got 810 / 1000 correct (81.00)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.5888\n",
            "Checking accuracy on validation set\n",
            "Got 806 / 1000 correct (80.60)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.5702\n",
            "Checking accuracy on validation set\n",
            "Got 846 / 1000 correct (84.60)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.5300\n",
            "Checking accuracy on validation set\n",
            "Got 836 / 1000 correct (83.60)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.4486\n",
            "Checking accuracy on validation set\n",
            "Got 857 / 1000 correct (85.70)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 823 / 1000 correct (82.30)\n",
            "---- New lr = 0.00052 and weight_decay = 0.00093 ----\n",
            "766\n",
            "Epoch: 0, Iteration 0, loss = 4.8097\n",
            "Checking accuracy on validation set\n",
            "Got 120 / 1000 correct (12.00)\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 2.0077\n",
            "Checking accuracy on validation set\n",
            "Got 401 / 1000 correct (40.10)\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.5969\n",
            "Checking accuracy on validation set\n",
            "Got 431 / 1000 correct (43.10)\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.2957\n",
            "Checking accuracy on validation set\n",
            "Got 419 / 1000 correct (41.90)\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.3770\n",
            "Checking accuracy on validation set\n",
            "Got 504 / 1000 correct (50.40)\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.6065\n",
            "Checking accuracy on validation set\n",
            "Got 533 / 1000 correct (53.30)\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.1166\n",
            "Checking accuracy on validation set\n",
            "Got 583 / 1000 correct (58.30)\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 1.4100\n",
            "Checking accuracy on validation set\n",
            "Got 592 / 1000 correct (59.20)\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 1.3371\n",
            "Checking accuracy on validation set\n",
            "Got 618 / 1000 correct (61.80)\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 1.2688\n",
            "Checking accuracy on validation set\n",
            "Got 579 / 1000 correct (57.90)\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 1.2119\n",
            "Checking accuracy on validation set\n",
            "Got 621 / 1000 correct (62.10)\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 1.0653\n",
            "Checking accuracy on validation set\n",
            "Got 676 / 1000 correct (67.60)\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 1.0130\n",
            "Checking accuracy on validation set\n",
            "Got 634 / 1000 correct (63.40)\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 0.9105\n",
            "Checking accuracy on validation set\n",
            "Got 712 / 1000 correct (71.20)\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 0.8772\n",
            "Checking accuracy on validation set\n",
            "Got 736 / 1000 correct (73.60)\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 0.8092\n",
            "Checking accuracy on validation set\n",
            "Got 655 / 1000 correct (65.50)\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 0.9544\n",
            "Checking accuracy on validation set\n",
            "Got 718 / 1000 correct (71.80)\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 1.2294\n",
            "Checking accuracy on validation set\n",
            "Got 750 / 1000 correct (75.00)\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 0.7566\n",
            "Checking accuracy on validation set\n",
            "Got 715 / 1000 correct (71.50)\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 1.1225\n",
            "Checking accuracy on validation set\n",
            "Got 688 / 1000 correct (68.80)\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 0.7678\n",
            "Checking accuracy on validation set\n",
            "Got 766 / 1000 correct (76.60)\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 0.5276\n",
            "Checking accuracy on validation set\n",
            "Got 738 / 1000 correct (73.80)\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 1.1735\n",
            "Checking accuracy on validation set\n",
            "Got 744 / 1000 correct (74.40)\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 0.9342\n",
            "Checking accuracy on validation set\n",
            "Got 699 / 1000 correct (69.90)\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 0.7137\n",
            "Checking accuracy on validation set\n",
            "Got 749 / 1000 correct (74.90)\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 0.7256\n",
            "Checking accuracy on validation set\n",
            "Got 759 / 1000 correct (75.90)\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 1.0187\n",
            "Checking accuracy on validation set\n",
            "Got 732 / 1000 correct (73.20)\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 0.9083\n",
            "Checking accuracy on validation set\n",
            "Got 747 / 1000 correct (74.70)\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.5523\n",
            "Checking accuracy on validation set\n",
            "Got 730 / 1000 correct (73.00)\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.7612\n",
            "Checking accuracy on validation set\n",
            "Got 728 / 1000 correct (72.80)\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.7845\n",
            "Checking accuracy on validation set\n",
            "Got 758 / 1000 correct (75.80)\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 0.5943\n",
            "Checking accuracy on validation set\n",
            "Got 718 / 1000 correct (71.80)\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.8133\n",
            "Checking accuracy on validation set\n",
            "Got 780 / 1000 correct (78.00)\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 0.8084\n",
            "Checking accuracy on validation set\n",
            "Got 768 / 1000 correct (76.80)\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.8139\n",
            "Checking accuracy on validation set\n",
            "Got 777 / 1000 correct (77.70)\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.6193\n",
            "Checking accuracy on validation set\n",
            "Got 778 / 1000 correct (77.80)\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 0.4547\n",
            "Checking accuracy on validation set\n",
            "Got 718 / 1000 correct (71.80)\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 0.5483\n",
            "Checking accuracy on validation set\n",
            "Got 783 / 1000 correct (78.30)\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.7278\n",
            "Checking accuracy on validation set\n",
            "Got 786 / 1000 correct (78.60)\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.6827\n",
            "Checking accuracy on validation set\n",
            "Got 770 / 1000 correct (77.00)\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.7242\n",
            "Checking accuracy on validation set\n",
            "Got 778 / 1000 correct (77.80)\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.6623\n",
            "Checking accuracy on validation set\n",
            "Got 800 / 1000 correct (80.00)\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.3890\n",
            "Checking accuracy on validation set\n",
            "Got 776 / 1000 correct (77.60)\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.5336\n",
            "Checking accuracy on validation set\n",
            "Got 820 / 1000 correct (82.00)\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 0.5161\n",
            "Checking accuracy on validation set\n",
            "Got 789 / 1000 correct (78.90)\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.4266\n",
            "Checking accuracy on validation set\n",
            "Got 795 / 1000 correct (79.50)\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.5401\n",
            "Checking accuracy on validation set\n",
            "Got 771 / 1000 correct (77.10)\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.6615\n",
            "Checking accuracy on validation set\n",
            "Got 771 / 1000 correct (77.10)\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.4929\n",
            "Checking accuracy on validation set\n",
            "Got 782 / 1000 correct (78.20)\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.7394\n",
            "Checking accuracy on validation set\n",
            "Got 811 / 1000 correct (81.10)\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.5043\n",
            "Checking accuracy on validation set\n",
            "Got 805 / 1000 correct (80.50)\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.5000\n",
            "Checking accuracy on validation set\n",
            "Got 763 / 1000 correct (76.30)\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.8214\n",
            "Checking accuracy on validation set\n",
            "Got 778 / 1000 correct (77.80)\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.5762\n",
            "Checking accuracy on validation set\n",
            "Got 811 / 1000 correct (81.10)\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.7195\n",
            "Checking accuracy on validation set\n",
            "Got 832 / 1000 correct (83.20)\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.7007\n",
            "Checking accuracy on validation set\n",
            "Got 806 / 1000 correct (80.60)\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.6578\n",
            "Checking accuracy on validation set\n",
            "Got 817 / 1000 correct (81.70)\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.5217\n",
            "Checking accuracy on validation set\n",
            "Got 796 / 1000 correct (79.60)\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.4287\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.4335\n",
            "Checking accuracy on validation set\n",
            "Got 806 / 1000 correct (80.60)\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.4230\n",
            "Checking accuracy on validation set\n",
            "Got 819 / 1000 correct (81.90)\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.6537\n",
            "Checking accuracy on validation set\n",
            "Got 832 / 1000 correct (83.20)\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.5097\n",
            "Checking accuracy on validation set\n",
            "Got 818 / 1000 correct (81.80)\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.5066\n",
            "Checking accuracy on validation set\n",
            "Got 829 / 1000 correct (82.90)\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.5852\n",
            "Checking accuracy on validation set\n",
            "Got 831 / 1000 correct (83.10)\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.4786\n",
            "Checking accuracy on validation set\n",
            "Got 839 / 1000 correct (83.90)\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.6132\n",
            "Checking accuracy on validation set\n",
            "Got 756 / 1000 correct (75.60)\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.6842\n",
            "Checking accuracy on validation set\n",
            "Got 799 / 1000 correct (79.90)\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.6220\n",
            "Checking accuracy on validation set\n",
            "Got 833 / 1000 correct (83.30)\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.5882\n",
            "Checking accuracy on validation set\n",
            "Got 841 / 1000 correct (84.10)\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.6863\n",
            "Checking accuracy on validation set\n",
            "Got 825 / 1000 correct (82.50)\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.4889\n",
            "Checking accuracy on validation set\n",
            "Got 849 / 1000 correct (84.90)\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.4778\n",
            "Checking accuracy on validation set\n",
            "Got 839 / 1000 correct (83.90)\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.5545\n",
            "Checking accuracy on validation set\n",
            "Got 837 / 1000 correct (83.70)\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.3108\n",
            "Checking accuracy on validation set\n",
            "Got 849 / 1000 correct (84.90)\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.2996\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.5607\n",
            "Checking accuracy on validation set\n",
            "Got 843 / 1000 correct (84.30)\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.6222\n",
            "Checking accuracy on validation set\n",
            "Got 826 / 1000 correct (82.60)\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.5995\n",
            "Checking accuracy on validation set\n",
            "Got 830 / 1000 correct (83.00)\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.4971\n",
            "Checking accuracy on validation set\n",
            "Got 842 / 1000 correct (84.20)\n",
            "\n",
            "Checking accuracy on validation set\n",
            "Got 841 / 1000 correct (84.10)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 384
        },
        "id": "pLjNXL95UUyb",
        "outputId": "18901bab-90fb-43c2-f48a-72b2de205b23"
      },
      "source": [
        "opt.plot_convergence()\r\n",
        "print(\"Best Hyperparameters: \", opt.x_opt)\r\n",
        "print(\"Best Accuracy: \", opt.fx_opt)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "[encoded data, 34924 characters]\n",
            "text/plain": [
              "<Figure size 720x360 with 2 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Best Hyperparameters:  [0.00051311 0.00097364]\n",
            "Best Accuracy:  -0.853\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sCHAl38M7-RF",
        "outputId": "b0e03e73-397d-4939-ab6e-536b821cf99d"
      },
      "source": [
        "# define and train the network\r\n",
        "model = ResNet18()\r\n",
        "optimizer = optim.Adam(model.parameters(),lr = 0.000513, weight_decay = 0.000973)\r\n",
        "\r\n",
        "train_part(model, optimizer, epochs = 10)\r\n",
        "\r\n",
        "\r\n",
        "# report test set accuracy\r\n",
        "\r\n",
        "check_accuracy(loader_test, model)\r\n",
        "\r\n",
        "\r\n",
        "# save the model\r\n",
        "torch.save(model.state_dict(), 'model.pt')"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "766\n",
            "Epoch: 0, Iteration 0, loss = 3.8242\n",
            "\n",
            "Epoch: 0, Iteration 100, loss = 1.6075\n",
            "\n",
            "Epoch: 0, Iteration 200, loss = 1.5011\n",
            "\n",
            "Epoch: 0, Iteration 300, loss = 1.3821\n",
            "\n",
            "Epoch: 0, Iteration 400, loss = 1.2997\n",
            "\n",
            "Epoch: 0, Iteration 500, loss = 1.1808\n",
            "\n",
            "Epoch: 0, Iteration 600, loss = 1.6881\n",
            "\n",
            "Epoch: 0, Iteration 700, loss = 1.0739\n",
            "\n",
            "766\n",
            "Epoch: 1, Iteration 0, loss = 1.2621\n",
            "\n",
            "Epoch: 1, Iteration 100, loss = 0.9631\n",
            "\n",
            "Epoch: 1, Iteration 200, loss = 1.3792\n",
            "\n",
            "Epoch: 1, Iteration 300, loss = 1.1428\n",
            "\n",
            "Epoch: 1, Iteration 400, loss = 0.9323\n",
            "\n",
            "Epoch: 1, Iteration 500, loss = 0.7955\n",
            "\n",
            "Epoch: 1, Iteration 600, loss = 1.0677\n",
            "\n",
            "Epoch: 1, Iteration 700, loss = 0.9228\n",
            "\n",
            "766\n",
            "Epoch: 2, Iteration 0, loss = 0.8093\n",
            "\n",
            "Epoch: 2, Iteration 100, loss = 0.7533\n",
            "\n",
            "Epoch: 2, Iteration 200, loss = 0.8682\n",
            "\n",
            "Epoch: 2, Iteration 300, loss = 0.8569\n",
            "\n",
            "Epoch: 2, Iteration 400, loss = 0.7559\n",
            "\n",
            "Epoch: 2, Iteration 500, loss = 0.6908\n",
            "\n",
            "Epoch: 2, Iteration 600, loss = 0.6042\n",
            "\n",
            "Epoch: 2, Iteration 700, loss = 0.5876\n",
            "\n",
            "766\n",
            "Epoch: 3, Iteration 0, loss = 0.9688\n",
            "\n",
            "Epoch: 3, Iteration 100, loss = 1.1008\n",
            "\n",
            "Epoch: 3, Iteration 200, loss = 0.7724\n",
            "\n",
            "Epoch: 3, Iteration 300, loss = 0.6468\n",
            "\n",
            "Epoch: 3, Iteration 400, loss = 0.9111\n",
            "\n",
            "Epoch: 3, Iteration 500, loss = 0.8497\n",
            "\n",
            "Epoch: 3, Iteration 600, loss = 0.6558\n",
            "\n",
            "Epoch: 3, Iteration 700, loss = 0.8008\n",
            "\n",
            "766\n",
            "Epoch: 4, Iteration 0, loss = 0.6815\n",
            "\n",
            "Epoch: 4, Iteration 100, loss = 0.6311\n",
            "\n",
            "Epoch: 4, Iteration 200, loss = 0.5132\n",
            "\n",
            "Epoch: 4, Iteration 300, loss = 0.5730\n",
            "\n",
            "Epoch: 4, Iteration 400, loss = 0.5668\n",
            "\n",
            "Epoch: 4, Iteration 500, loss = 0.4968\n",
            "\n",
            "Epoch: 4, Iteration 600, loss = 0.6016\n",
            "\n",
            "Epoch: 4, Iteration 700, loss = 0.8137\n",
            "\n",
            "766\n",
            "Epoch: 5, Iteration 0, loss = 0.5700\n",
            "\n",
            "Epoch: 5, Iteration 100, loss = 0.6260\n",
            "\n",
            "Epoch: 5, Iteration 200, loss = 0.5672\n",
            "\n",
            "Epoch: 5, Iteration 300, loss = 0.6930\n",
            "\n",
            "Epoch: 5, Iteration 400, loss = 0.6346\n",
            "\n",
            "Epoch: 5, Iteration 500, loss = 0.5215\n",
            "\n",
            "Epoch: 5, Iteration 600, loss = 0.6199\n",
            "\n",
            "Epoch: 5, Iteration 700, loss = 0.3816\n",
            "\n",
            "766\n",
            "Epoch: 6, Iteration 0, loss = 0.7697\n",
            "\n",
            "Epoch: 6, Iteration 100, loss = 0.6668\n",
            "\n",
            "Epoch: 6, Iteration 200, loss = 0.6334\n",
            "\n",
            "Epoch: 6, Iteration 300, loss = 0.7414\n",
            "\n",
            "Epoch: 6, Iteration 400, loss = 0.5848\n",
            "\n",
            "Epoch: 6, Iteration 500, loss = 0.4268\n",
            "\n",
            "Epoch: 6, Iteration 600, loss = 0.7574\n",
            "\n",
            "Epoch: 6, Iteration 700, loss = 0.6004\n",
            "\n",
            "766\n",
            "Epoch: 7, Iteration 0, loss = 0.5996\n",
            "\n",
            "Epoch: 7, Iteration 100, loss = 0.6203\n",
            "\n",
            "Epoch: 7, Iteration 200, loss = 0.5860\n",
            "\n",
            "Epoch: 7, Iteration 300, loss = 0.3861\n",
            "\n",
            "Epoch: 7, Iteration 400, loss = 0.4512\n",
            "\n",
            "Epoch: 7, Iteration 500, loss = 0.6834\n",
            "\n",
            "Epoch: 7, Iteration 600, loss = 0.5661\n",
            "\n",
            "Epoch: 7, Iteration 700, loss = 0.4902\n",
            "\n",
            "766\n",
            "Epoch: 8, Iteration 0, loss = 0.4905\n",
            "\n",
            "Epoch: 8, Iteration 100, loss = 0.5776\n",
            "\n",
            "Epoch: 8, Iteration 200, loss = 0.4862\n",
            "\n",
            "Epoch: 8, Iteration 300, loss = 0.7179\n",
            "\n",
            "Epoch: 8, Iteration 400, loss = 0.4977\n",
            "\n",
            "Epoch: 8, Iteration 500, loss = 0.3454\n",
            "\n",
            "Epoch: 8, Iteration 600, loss = 0.5161\n",
            "\n",
            "Epoch: 8, Iteration 700, loss = 0.5944\n",
            "\n",
            "766\n",
            "Epoch: 9, Iteration 0, loss = 0.3990\n",
            "\n",
            "Epoch: 9, Iteration 100, loss = 0.4425\n",
            "\n",
            "Epoch: 9, Iteration 200, loss = 0.4498\n",
            "\n",
            "Epoch: 9, Iteration 300, loss = 0.7687\n",
            "\n",
            "Epoch: 9, Iteration 400, loss = 0.4281\n",
            "\n",
            "Epoch: 9, Iteration 500, loss = 0.3201\n",
            "\n",
            "Epoch: 9, Iteration 600, loss = 0.6686\n",
            "\n",
            "Epoch: 9, Iteration 700, loss = 0.4585\n",
            "\n",
            "Checking accuracy on test set\n",
            "Got 8381 / 10000 correct (83.81)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6IQdUdufJMV1"
      },
      "source": [
        "## Part 3"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e3GOnhsiJMV1"
      },
      "source": [
        "The code provided below will allow you to visualise the feature maps computed by different layers of your network. Run the code (install matplotlib if necessary) and **answer the following questions**: \n",
        "\n",
        "1. Compare the feature maps from low-level layers to high-level layers, what do you observe? \n",
        "\n",
        "2. Use the training log, reported test set accuracy and the feature maps, analyse the performance of your network. If you think the performance is sufficiently good, explain why; if not, what might be the problem and how can you improve the performance?\n",
        "\n",
        "3. What are the other possible ways to analyse the performance of your network?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Kb1vxCf9JMV1"
      },
      "source": [
        "**YOUR ANSWER FOR PART 3 HERE**\n",
        "\n",
        "A:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SNcmV4BEJMV1"
      },
      "source": [
        "#!pip install matplotlib\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "plt.tight_layout()\n",
        "\n",
        "\n",
        "activation = {}\n",
        "def get_activation(name):\n",
        "    def hook(model, input, output):\n",
        "        activation[name] = output.detach()\n",
        "    return hook\n",
        "\n",
        "vis_labels = ['conv1', 'layer1', 'layer2', 'layer3', 'layer4']\n",
        "\n",
        "for l in vis_labels:\n",
        "\n",
        "    getattr(model, l).register_forward_hook(get_activation(l))\n",
        "    \n",
        "    \n",
        "data, _ = cifar10_test[0]\n",
        "data = data.unsqueeze_(0).to(device = device, dtype = dtype)\n",
        "\n",
        "output = model(data)\n",
        "\n",
        "\n",
        "\n",
        "for idx, l in enumerate(vis_labels):\n",
        "\n",
        "    act = activation[l].squeeze()\n",
        "\n",
        "    if idx < 2:\n",
        "        ncols = 8\n",
        "    else:\n",
        "        ncols = 32\n",
        "        \n",
        "    nrows = act.size(0) // ncols\n",
        "    \n",
        "    fig, axarr = plt.subplots(nrows, ncols)\n",
        "    fig.suptitle(l)\n",
        "\n",
        "\n",
        "    for i in range(nrows):\n",
        "        for j in range(ncols):\n",
        "            axarr[i, j].imshow(act[i * nrows + j].cpu())\n",
        "            axarr[i, j].axis('off')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}