Neural Networks and Optimizers from scratch in NumPy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow.keras.datasets.mnist as mnist\n",
    "\n",
    "import sys\n",
    "sys.path.append('../Neural_Network_from_Scratch')\n",
    "import optimizers as opt\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import plotly.graph_objs as go"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load data from tensorflow\n",
    "data = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train), (x_test, y_test) = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((60000, 28, 28), (60000,), (10000, 28, 28), (10000,))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.shape, y_train.shape, x_test.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x14a62a1c0>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAOZ0lEQVR4nO3dbYxc5XnG8euKbezamMQbB9chLjjgFAg0Jl0ZEBZQobgOqgSoCsSKIkJpnSY4Ca0rQWlV3IpWbpUQUUqRTHExFS+BBIQ/0CTUQpCowWWhBgwEDMY0NmaNWYENIX5Z3/2w42iBnWeXmTMv3vv/k1Yzc+45c24NXD5nznNmHkeEAIx/H+p0AwDag7ADSRB2IAnCDiRB2IEkJrZzY4d5ckzRtHZuEkjlV3pbe2OPR6o1FXbbiyVdJ2mCpH+LiJWl50/RNJ3qc5rZJICC9bGubq3hw3jbEyTdIOnzkk6UtMT2iY2+HoDWauYz+wJJL0TE5ojYK+lOSedV0xaAqjUT9qMk/WLY4621Ze9ie6ntPtt9+7Snic0BaEbLz8ZHxKqI6I2I3kma3OrNAaijmbBvkzRn2ONP1JYB6ELNhP1RSfNsz7V9mKQvSlpbTVsAqtbw0FtE7Le9TNKPNDT0tjoinq6sMwCVamqcPSLul3R/Rb0AaCEulwWSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJpmZxRffzxPJ/4gkfm9nS7T/3F8fUrQ1OPVBc9+hjdxTrU7/uYv3Vaw+rW3u893vFdXcOvl2sn3r38mL9uD9/pFjvhKbCbnuLpN2SBiXtj4jeKpoCUL0q9uy/FxE7K3gdAC3EZ3YgiWbDHpJ+bPsx20tHeoLtpbb7bPft054mNwegUc0exi+MiG22j5T0gO2fR8TDw58QEaskrZKkI9wTTW4PQIOa2rNHxLba7Q5J90paUEVTAKrXcNhtT7M9/eB9SYskbayqMQDVauYwfpake20ffJ3bI+KHlXQ1zkw4YV6xHpMnFeuvnPWRYv2d0+qPCfd8uDxe/JPPlMebO+k/fzm9WP/Hf1lcrK8/+fa6tZf2vVNcd2X/54r1j//k0PtE2nDYI2KzpM9U2AuAFmLoDUiCsANJEHYgCcIOJEHYgST4imsFBs/+bLF+7S03FOufmlT/q5jj2b4YLNb/5vqvFOsT3y4Pf51+97K6tenb9hfXnbyzPDQ3tW99sd6N2LMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs1dg8nOvFOuP/WpOsf6pSf1VtlOp5dtPK9Y3v1X+Kepbjv1+3dqbB8rj5LP++b+L9VY69L7AOjr27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQhCPaN6J4hHviVJ/Ttu11i4FLTi/Wdy0u/9zzhCcPL9af+Pr1H7ing67Z+TvF+qNnlcfRB994s1iP0+v/APGWbxZX1dwlT5SfgPdZH+u0KwZGnMuaPTuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4exeYMPOjxfrg6wPF+ku31x8rf/rM1cV1F/zDN4r1I2/o3HfK8cE1Nc5ue7XtHbY3DlvWY/sB25tqtzOqbBhA9cZyGH+LpPfOen+lpHURMU/SutpjAF1s1LBHxMOS3nsceZ6kNbX7aySdX3FfACrW6G/QzYqI7bX7r0qaVe+JtpdKWipJUzS1wc0BaFbTZ+Nj6Axf3bN8EbEqInojoneSJje7OQANajTs/bZnS1Ltdkd1LQFohUbDvlbSxbX7F0u6r5p2ALTKqJ/Zbd8h6WxJM21vlXS1pJWS7rJ9qaSXJV3YyibHu8Gdrze1/r5djc/v/ukvPVOsv3bjhPILHCjPsY7uMWrYI2JJnRJXxwCHEC6XBZIg7EAShB1IgrADSRB2IAmmbB4HTrji+bq1S04uD5r8+9HrivWzvnBZsT79e48U6+ge7NmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2ceB0rTJr3/thOK6/7f2nWL9ymtuLdb/8sILivX43w/Xrc35+58V11Ubf+Y8A/bsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEUzYnN/BHpxfrt1397WJ97sQpDW/707cuK9bn3bS9WN+/eUvD2x6vmpqyGcD4QNiBJAg7kARhB5Ig7EAShB1IgrADSTDOjqI4Y36xfsTKrcX6HZ/8UcPbPv7BPy7Wf/tv63+PX5IGN21ueNuHqqbG2W2vtr3D9sZhy1bY3mZ7Q+3v3CobBlC9sRzG3yJp8QjLvxsR82t/91fbFoCqjRr2iHhY0kAbegHQQs2coFtm+8naYf6Mek+yvdR2n+2+fdrTxOYANKPRsN8o6VhJ8yVtl/Sdek+MiFUR0RsRvZM0ucHNAWhWQ2GPiP6IGIyIA5JukrSg2rYAVK2hsNuePezhBZI21nsugO4w6ji77TsknS1ppqR+SVfXHs+XFJK2SPpqRJS/fCzG2cejCbOOLNZfuei4urX1V1xXXPdDo+yLvvTSomL9zYWvF+vjUWmcfdRJIiJiyQiLb266KwBtxeWyQBKEHUiCsANJEHYgCcIOJMFXXNExd20tT9k81YcV67+MvcX6H3zj8vqvfe/64rqHKn5KGgBhB7Ig7EAShB1IgrADSRB2IAnCDiQx6rfekNuBheWfkn7xC+Upm0+av6VubbRx9NFcP3BKsT71vr6mXn+8Yc8OJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kwzj7OufekYv35b5bHum86Y02xfuaU8nfKm7En9hXrjwzMLb/AgVF/3TwV9uxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7IeAiXOPLtZfvOTjdWsrLrqzuO4fHr6zoZ6qcFV/b7H+0HWnFesz1pR/dx7vNuqe3fYc2w/afsb207a/VVveY/sB25tqtzNa3y6ARo3lMH6/pOURcaKk0yRdZvtESVdKWhcR8yStqz0G0KVGDXtEbI+Ix2v3d0t6VtJRks6TdPBayjWSzm9VkwCa94E+s9s+RtIpktZLmhURBy8+flXSrDrrLJW0VJKmaGqjfQJo0pjPxts+XNIPJF0eEbuG12JodsgRZ4iMiFUR0RsRvZM0ualmATRuTGG3PUlDQb8tIu6pLe63PbtWny1pR2taBFCFUQ/jbVvSzZKejYhrh5XWSrpY0sra7X0t6XAcmHjMbxXrb/7u7GL9or/7YbH+px+5p1hvpeXby8NjP/vX+sNrPbf8T3HdGQcYWqvSWD6znyHpy5Kesr2htuwqDYX8LtuXSnpZ0oWtaRFAFUYNe0T8VNKIk7tLOqfadgC0CpfLAkkQdiAJwg4kQdiBJAg7kARfcR2jibN/s25tYPW04rpfm/tQsb5ken9DPVVh2baFxfrjN5anbJ75/Y3Fes9uxsq7BXt2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUgizTj73t8v/2zx3j8bKNavOu7+urVFv/F2Qz1VpX/wnbq1M9cuL657/F//vFjveaM8Tn6gWEU3Yc8OJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0mkGWffcn7537XnT767Zdu+4Y1ji/XrHlpUrHuw3o/7Djn+mpfq1ub1ry+uO1isYjxhzw4kQdiBJAg7kARhB5Ig7EAShB1IgrADSTgiyk+w50i6VdIsSSFpVURcZ3uFpD+R9FrtqVdFRP0vfUs6wj1xqpn4FWiV9bFOu2JgxAszxnJRzX5JyyPicdvTJT1m+4Fa7bsR8e2qGgXQOmOZn327pO21+7ttPyvpqFY3BqBaH+gzu+1jJJ0i6eA1mMtsP2l7te0ZddZZarvPdt8+7WmqWQCNG3PYbR8u6QeSLo+IXZJulHSspPka2vN/Z6T1ImJVRPRGRO8kTa6gZQCNGFPYbU/SUNBvi4h7JCki+iNiMCIOSLpJ0oLWtQmgWaOG3bYl3Szp2Yi4dtjy2cOedoGk8nSeADpqLGfjz5D0ZUlP2d5QW3aVpCW252toOG6LpK+2pEMAlRjL2fifShpp3K44pg6gu3AFHZAEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IIlRf0q60o3Zr0l6ediimZJ2tq2BD6Zbe+vWviR6a1SVvR0dER8bqdDWsL9v43ZfRPR2rIGCbu2tW/uS6K1R7eqNw3ggCcIOJNHpsK/q8PZLurW3bu1LordGtaW3jn5mB9A+nd6zA2gTwg4k0ZGw215s+znbL9i+shM91GN7i+2nbG+w3dfhXlbb3mF747BlPbYfsL2pdjviHHsd6m2F7W21926D7XM71Nsc2w/afsb207a/VVve0feu0Fdb3re2f2a3PUHS85I+J2mrpEclLYmIZ9raSB22t0jqjYiOX4Bh+0xJb0m6NSJOqi37J0kDEbGy9g/ljIi4okt6WyHprU5P412brWj28GnGJZ0v6Svq4HtX6OtCteF968SefYGkFyJic0TslXSnpPM60EfXi4iHJQ28Z/F5ktbU7q/R0P8sbVent64QEdsj4vHa/d2SDk4z3tH3rtBXW3Qi7EdJ+sWwx1vVXfO9h6Qf237M9tJONzOCWRGxvXb/VUmzOtnMCEadxrud3jPNeNe8d41Mf94sTtC938KI+Kykz0u6rHa42pVi6DNYN42djmka73YZYZrxX+vke9fo9OfN6kTYt0maM+zxJ2rLukJEbKvd7pB0r7pvKur+gzPo1m53dLifX+umabxHmmZcXfDedXL6806E/VFJ82zPtX2YpC9KWtuBPt7H9rTaiRPZniZpkbpvKuq1ki6u3b9Y0n0d7OVdumUa73rTjKvD713Hpz+PiLb/STpXQ2fkX5T0V53ooU5fn5T0RO3v6U73JukODR3W7dPQuY1LJX1U0jpJmyT9l6SeLurtPyQ9JelJDQVrdod6W6ihQ/QnJW2o/Z3b6feu0Fdb3jculwWS4AQdkARhB5Ig7EAShB1IgrADSRB2IAnCDiTx/65XcTNOWsh5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(x_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 784) (10000, 784)\n"
     ]
    }
   ],
   "source": [
    "# Preprocess data\n",
    "# Reshape (flatten)\n",
    "x_train = x_train.reshape(x_train.shape[0], -1)\n",
    "x_test = x_test.reshape(x_test.shape[0], -1)\n",
    "print(x_train.shape, x_test.shape)\n",
    "\n",
    "# Normalize data within {0,1} + dtype conversion\n",
    "x_train = np.array(x_train/255., dtype=np.float32)\n",
    "x_test = np.array(x_test/255., dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to convert labels to one-hot encodings\n",
    "def one_hot(Y):\n",
    "    num_labels = len(set(Y))\n",
    "    new_Y = []\n",
    "    for label in Y:\n",
    "        encoding = np.zeros(num_labels)\n",
    "        encoding[label] = 1.\n",
    "        new_Y.append(encoding)\n",
    "    return np.array(new_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((60000, 10), (10000, 10))"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train = one_hot(y_train)\n",
    "y_test = one_hot(y_test)\n",
    "y_train.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Layer object to handle weights, biases, activation (if any) of a layer\n",
    "\n",
    "class Layer:\n",
    "    def __init__(self, hidden_units: int, activation:str=None):\n",
    "        self.hidden_units = hidden_units\n",
    "        self.activation = activation\n",
    "        self.W = None\n",
    "        self.b = None\n",
    "        \n",
    "    def initialize_params(self, n_in, hidden_units):\n",
    "        # set seed for reproducibility \n",
    "        np.random.seed(42)\n",
    "        self.W = np.random.randn(n_in, hidden_units) * np.sqrt(2/n_in) \n",
    "        np.random.seed(42)\n",
    "        self.b = np.zeros((1, hidden_units))\n",
    "    \n",
    "    def forward(self, X):\n",
    "        self.input = np.array(X, copy=True)\n",
    "        if self.W is None:\n",
    "            self.initialize_params(self.input.shape[-1], self.hidden_units)\n",
    "\n",
    "        self.Z = X @ self.W + self.b\n",
    "        \n",
    "        if self.activation is not None:\n",
    "            self.A = self.activation_fn(self.Z)\n",
    "            return self.A\n",
    "        return self.Z\n",
    "    \n",
    "    def activation_fn(self, z, derivative=False):\n",
    "        if self.activation == 'relu':\n",
    "            if derivative:\n",
    "                return self.drelu(z)\n",
    "            return self.relu(z)\n",
    "        if self.activation == 'sigmoid':\n",
    "            if derivative:\n",
    "                return self.dsigmoid(z)\n",
    "            return self.sigmoid(z)\n",
    "        if self.activation == 'softmax':\n",
    "            if derivative: \n",
    "                return self.dsoftmax(z)\n",
    "            return self.softmax(z)\n",
    "\n",
    "    @staticmethod\n",
    "    def sigmoid(z):\n",
    "        return 1 / (1 + np.exp(-z))\n",
    "\n",
    "    @staticmethod\n",
    "    def dsigmoid(z):\n",
    "        return Layer.sigmoid(z) * (1-Layer.sigmoid(z))\n",
    "\n",
    "    @staticmethod\n",
    "    def relu(z):\n",
    "        return np.maximum(0, z)\n",
    "\n",
    "    @staticmethod\n",
    "    def drelu(z):\n",
    "        return np.where(z<=0,0,1)\n",
    "\n",
    "    @staticmethod\n",
    "    def softmax(x):  # numerically stable version of softmax \n",
    "        exp = np.exp(x - np.max(x, axis=1, keepdims=True))\n",
    "        return exp / np.sum(exp, axis=1, keepdims=True)\n",
    "\n",
    "    @staticmethod\n",
    "    def dsoftmax(x):\n",
    "        exp = np.exp(x - np.max(x, axis=1, keepdims=True)) \n",
    "        return exps / np.sum(exps, axis=0) * (1 - exps / np.sum(exps, axis=0))\n",
    "\n",
    "    def __repr__(self):\n",
    "        return str(f'''Hidden Units={self.hidden_units}; Activation={self.activation}''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self):\n",
    "        self.layers = dict()\n",
    "        self.cache = dict()\n",
    "        self.grads = dict()\n",
    "        \n",
    "    def add(self, layer):\n",
    "        self.layers[len(self.layers)+1] = layer\n",
    "        \n",
    "    def set_config(self, epochs, learning_rate, optimizer=None):\n",
    "        self.epochs = epochs\n",
    "        self.optimizer = optimizer\n",
    "        self.learning_rate = learning_rate\n",
    "        if not not self.optimizer:\n",
    "            self.optimizer.config(self.layers)\n",
    "            self.optimizer.epochs = self.epochs\n",
    "            self.optimizer.learning_rate = self.learning_rate\n",
    "\n",
    "    def forward(self, x):\n",
    "        for idx, layer in self.layers.items():\n",
    "            x = layer.forward(x)\n",
    "            self.cache[f'W{idx}'] = layer.W\n",
    "            self.cache[f'Z{idx}'] = layer.Z\n",
    "            self.cache[f'A{idx}'] = layer.A\n",
    "        return x\n",
    "\n",
    "    def backward(self, y):\n",
    "        last_layer_idx = max(self.layers.keys())\n",
    "        m = y.shape[0]\n",
    "        # back prop through all dZs \n",
    "        for idx in reversed(range(1, last_layer_idx+1)):\n",
    "            if idx == last_layer_idx:\n",
    "                # e.g. dZ3 = y_pred - y_true for a 3 layer network \n",
    "                self.grads[f'dZ{idx}'] = self.cache[f'A{idx}'] - y\n",
    "            else:\n",
    "                # dZn = dZ(n+1) dot W(n+1) * inverse derivative of activation function of Layer n, with Zn as input \n",
    "                self.grads[f'dZ{idx}'] = self.grads[f'dZ{idx+1}'] @ self.cache[f'W{idx+1}'].T *\\\n",
    "                                        self.layers[idx].activation_fn(self.cache[f'Z{idx}'], derivative=True)\n",
    "            self.grads[f'dW{idx}'] = 1 / m * self.layers[idx].input.T @ self.grads[f'dZ{idx}']\n",
    "            self.grads[f'db{idx}'] = 1 / m * np.sum(self.grads[f'dZ{idx}'], axis=0, keepdims=True)\n",
    "            \n",
    "            assert self.grads[f'dW{idx}'].shape == self.cache[f'W{idx}'].shape\n",
    "\n",
    "    def update_params(self, epoch_num, steps):\n",
    "        for idx in self.layers.keys():\n",
    "            if self.optimizer is None:\n",
    "                self.optimize(idx)\n",
    "            else:\n",
    "                self.optimizer.optimize(idx, self.layers, self.grads, epoch_num, steps)                \n",
    " \n",
    "    def optimize(self, idx):  \n",
    "        # Vanilla minibatch gradient descent\n",
    "        self.layers[idx].W -= self.learning_rate * self.grads[f'dW{idx}']\n",
    "        self.layers[idx].b -= self.learning_rate * self.grads[f'db{idx}']\n",
    "        \n",
    "    def fit(self, x_train, y_train, x_test=None, y_test=None, batch_size=32):\n",
    "        '''Training cycle of the model object'''\n",
    "        losses = []\n",
    "        train_accs = []\n",
    "        val_accs = []\n",
    "\n",
    "        for i in range(1, self.epochs+1):\n",
    "            print(f'Epoch {i}')\n",
    "            batches = self.create_batches(x_train, y_train, batch_size)\n",
    "            epoch_loss = []\n",
    "            steps = 0\n",
    "            \n",
    "            for x, y in batches:\n",
    "                steps += 1\n",
    "                preds = self.forward(x)\n",
    "                loss = self.compute_loss(y, preds)\n",
    "                epoch_loss.append(loss)\n",
    "\n",
    "                # Backward propagation - calculation of gradients \n",
    "                self.backward(y)\n",
    "                \n",
    "                # update weights and biases of each layer\n",
    "                self.update_params(i, steps)\n",
    "                \n",
    "            loss = sum(epoch_loss) / len(epoch_loss)\n",
    "            losses.append(loss)\n",
    "\n",
    "            # Predict with network on x_train\n",
    "            train_preds = self.forward(x_train)\n",
    "            c = np.argmax(train_preds, axis=1) == np.argmax(y_train, axis=1)\n",
    "            train_acc = list(c).count(True) / len(c) * 100\n",
    "            train_accs.append(train_acc)\n",
    "            \n",
    "            # Predict with network on x_test\n",
    "\n",
    "            test_preds = self.forward(x_test)\n",
    "            c = np.argmax(test_preds, axis=1) == np.argmax(y_test, axis=1)\n",
    "            val_acc = list(c).count(True)/len(c) * 100\n",
    "            val_accs.append(val_acc)\n",
    "\n",
    "            print(f'Loss:{loss} Train Acc: {train_acc} Val Acc: {val_acc}')            \n",
    "                \n",
    "        self.history = {'train_loss': losses, 'train_acc': train_accs, 'val_acc': val_accs}\n",
    "\n",
    "    @staticmethod\n",
    "    def compute_loss(Y, Y_hat):\n",
    "        m = Y.shape[0]\n",
    "        L = -1./m * np.sum(Y * np.log(Y_hat))\n",
    "        return L\n",
    "\n",
    "    @staticmethod\n",
    "    def mse(Y, Y_hat):\n",
    "        m = Y.shape[0]\n",
    "        return np.mean((Y- Y_hat)**2)\n",
    "    \n",
    "    @staticmethod\n",
    "    def create_batches(x, y, batch_size):\n",
    "        m = x.shape[0]\n",
    "        num_batches = m / batch_size\n",
    "        batches = []\n",
    "        for i in range(int(num_batches+1)):\n",
    "            batch_x = x[i*batch_size:(i+1)*batch_size]\n",
    "            batch_y = y[i*batch_size:(i+1)*batch_size]\n",
    "            batches.append((batch_x, batch_y))\n",
    "        \n",
    "        # without this, batch sizes that are perfectly divisible will create an \n",
    "        # empty array at index -1\n",
    "        if m % batch_size == 0:\n",
    "            batches.pop(-1)\n",
    "\n",
    "        return batches\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return str(self.layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 20\n",
    "lr = 1e-3\n",
    "batch_size = 128\n",
    "lr_decay = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_model(optimizer=None):\n",
    "    model = Model()\n",
    "    model.add(Layer(128, activation='relu'))\n",
    "    model.add(Layer(64, activation='relu'))\n",
    "    model.add(Layer(10, activation='softmax'))\n",
    "    model.set_config(epochs=epochs, learning_rate=lr, optimizer=optimizer)\n",
    "\n",
    "    model.fit(x_train, y_train, x_test, y_test, batch_size=batch_size)\n",
    "    return model.history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Demon stands for decaying momentum from this paper : https://arxiv.org/pdf/1910.04952v3.pdf\n",
    "# Optimizers : refer to code in link below for implementation\n",
    "# https://github.com/timvvvht/Neural-Networks-and-Optimizers-from-scratch/blob/main/optimizers.py\n",
    "\n",
    "sgdm = opt.SGDM(lr, name='SGDM')\n",
    "demonSGDM = opt.SGDM(lr, demon=True, name='DemonSGDM')\n",
    "qhm = opt.QHM(lr, name='QHM')\n",
    "nesterov = opt.Nesterov(lr, name='Nesterov')\n",
    "demonNesterov = opt.Nesterov(lr, demon=True, name='DemonNesterov')\n",
    "\n",
    "adagrad = opt.Adagrad(lr, name='Adagrad')\n",
    "rmsprop = opt.RMSprop(lr, name='RMSprop')\n",
    "adadelta = opt.Adadelta(lr, name='Adadelta')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adam family\n",
    "# Weight_decay is the implementation in the AdamW paper \n",
    "adam = opt.Adam(lr, name='Adam')\n",
    "adamW = opt.Adam(lr, weight_decay=True, name='AdamW')\n",
    "demonAdam = opt.Adam(lr, demon=True, name='DemonAdam')\n",
    "demonAdamW = opt.Adam(lr, weight_decay=True, demon=True, name='DemonAdamW')\n",
    "\n",
    "NAdam = opt.Nadam(lr, name='NAdam')\n",
    "NAdamW = opt.Nadam(lr, weight_decay=True, name='NAdamW')\n",
    "demonNAdam = opt.Nadam(lr, demon=True, name='DemonNAdam')\n",
    "demonNAdamW = opt.Nadam(lr, weight_decay=True, demon=True, name='DemonNAdamW')\n",
    "\n",
    "adamax = opt.Adamax(lr, name='Adamax')\n",
    "adamaxW = opt.Adamax(lr, weight_decay=True, name='AdamaxW')\n",
    "demonAdamax = opt.Adamax(lr, demon=True, name='DemonAdamax')\n",
    "demonAdamaxW = opt.Adamax(lr, weight_decay=True, demon=True, name='DemonAdamaxW')\n",
    "\n",
    "QHAdam = opt.QHAdam(lr, name='QHAdam')\n",
    "QHAdamW = opt.QHAdam(lr, weight_decay=True, name='QHAdamW')\n",
    "demonQHAdam = opt.QHAdam(lr, demon=True, name='DemonQHAdam')\n",
    "demonQHAdamW = opt.QHAdam(lr, weight_decay=True, demon=True, name='DemonQHAdamW')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "opts1 = [sgdm, demonSGDM, qhm, nesterov, demonNesterov, adagrad, rmsprop, adadelta]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "opts2 = [adam, adamW, demonAdam, demonAdamW, NAdam, NAdamW, demonNAdam, demonNAdamW, \n",
    "         adamax, adamaxW, demonAdamax, demonAdamaxW, QHAdam, QHAdamW, demonQHAdam, demonQHAdamW]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<optimizers.SGDM at 0x14a686160>,\n",
       " <optimizers.SGDM at 0x14a64d5b0>,\n",
       " <optimizers.QHM at 0x14a686220>,\n",
       " <optimizers.Nesterov at 0x14a6869a0>,\n",
       " <optimizers.Nesterov at 0x14a686790>,\n",
       " <optimizers.Adagrad at 0x14a686bb0>,\n",
       " <optimizers.RMSprop at 0x14a686760>,\n",
       " <optimizers.Adadelta at 0x14a686d30>,\n",
       " <optimizers.Adam at 0x14a7219a0>,\n",
       " <optimizers.Adam at 0x10a04a250>,\n",
       " <optimizers.Adam at 0x14a7219d0>,\n",
       " <optimizers.Adam at 0x10a04a2e0>,\n",
       " <optimizers.Nadam at 0x14a721d00>,\n",
       " <optimizers.Nadam at 0x14a721790>,\n",
       " <optimizers.Nadam at 0x14a721670>,\n",
       " <optimizers.Nadam at 0x14a721520>,\n",
       " <optimizers.Adamax at 0x14a721910>,\n",
       " <optimizers.Adamax at 0x14a721df0>,\n",
       " <optimizers.Adamax at 0x14a721880>,\n",
       " <optimizers.Adamax at 0x14a7215b0>,\n",
       " <optimizers.QHAdam at 0x14a7217f0>,\n",
       " <optimizers.QHAdam at 0x14a721820>,\n",
       " <optimizers.QHAdam at 0x14a721b20>,\n",
       " <optimizers.QHAdam at 0x14a721700>]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opts = opts1 + opts2\n",
    "opts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1\n",
      "Loss:1.971548086743485 Train Acc: 60.940000000000005 Val Acc: 61.78\n",
      "Epoch 2\n",
      "Loss:1.250420248838212 Train Acc: 77.28833333333334 Val Acc: 78.38000000000001\n",
      "Epoch 3\n",
      "Loss:0.7758959326613197 Train Acc: 84.01666666666667 Val Acc: 85.24000000000001\n",
      "Epoch 4\n",
      "Loss:0.5239670911303634 Train Acc: 87.92999999999999 Val Acc: 88.99000000000001\n",
      "Epoch 5\n",
      "Loss:0.3382509386365137 Train Acc: 92.72666666666667 Val Acc: 93.05\n",
      "Epoch 6\n",
      "Loss:0.2160378233607533 Train Acc: 94.80333333333333 Val Acc: 94.69999999999999\n",
      "Epoch 7\n",
      "Loss:0.16188467669077286 Train Acc: 96.2 Val Acc: 96.0\n",
      "Epoch 8\n",
      "Loss:0.13054313307258283 Train Acc: 96.65333333333334 Val Acc: 96.34\n",
      "Epoch 9\n",
      "Loss:0.1108617285036437 Train Acc: 97.29166666666667 Val Acc: 96.71\n",
      "Epoch 10\n",
      "Loss:0.09447141691313259 Train Acc: 97.615 Val Acc: 96.84\n",
      "Epoch 11\n",
      "Loss:0.07913496700905819 Train Acc: 97.79666666666667 Val Acc: 96.81\n",
      "Epoch 12\n",
      "Loss:0.06897837086191724 Train Acc: 97.925 Val Acc: 96.74000000000001\n",
      "Epoch 13\n",
      "Loss:0.06206052102064375 Train Acc: 97.99833333333333 Val Acc: 96.85000000000001\n",
      "Epoch 14\n",
      "Loss:0.0563329734093599 Train Acc: 98.30499999999999 Val Acc: 96.97\n",
      "Epoch 15\n",
      "Loss:0.051070260278167856 Train Acc: 98.64333333333335 Val Acc: 97.33000000000001\n",
      "Epoch 16\n",
      "Loss:0.04586898762231048 Train Acc: 98.81666666666666 Val Acc: 97.56\n",
      "Epoch 17\n",
      "Loss:0.04112409724026969 Train Acc: 98.965 Val Acc: 97.50999999999999\n",
      "Epoch 18\n",
      "Loss:0.03782737159145678 Train Acc: 99.04166666666666 Val Acc: 97.58\n",
      "Epoch 19\n",
      "Loss:0.03507443108055851 Train Acc: 99.11833333333333 Val Acc: 97.6\n",
      "Epoch 20\n",
      "Loss:0.03321487363372428 Train Acc: 99.07333333333334 Val Acc: 97.65\n",
      "Epoch 1\n",
      "Loss:1.1695748081723347 Train Acc: 84.84 Val Acc: 86.00999999999999\n",
      "Epoch 2\n",
      "Loss:0.4815219152400152 Train Acc: 88.53166666666667 Val Acc: 89.5\n",
      "Epoch 3\n",
      "Loss:0.3840708349918986 Train Acc: 89.94166666666666 Val Acc: 90.7\n",
      "Epoch 4\n",
      "Loss:0.3437184550649107 Train Acc: 90.66833333333332 Val Acc: 91.35\n",
      "Epoch 5\n",
      "Loss:0.3193405968850166 Train Acc: 91.23 Val Acc: 91.74\n",
      "Epoch 6\n",
      "Loss:0.301813844596415 Train Acc: 91.62666666666667 Val Acc: 92.09\n",
      "Epoch 7\n",
      "Loss:0.2881100803189039 Train Acc: 91.995 Val Acc: 92.47999999999999\n",
      "Epoch 8\n",
      "Loss:0.27695226100039216 Train Acc: 92.34666666666666 Val Acc: 92.67\n",
      "Epoch 9\n",
      "Loss:0.26758527517191877 Train Acc: 92.61666666666667 Val Acc: 92.9\n",
      "Epoch 10\n",
      "Loss:0.259602957411795 Train Acc: 92.82000000000001 Val Acc: 93.12\n",
      "Epoch 11\n",
      "Loss:0.2527050033766659 Train Acc: 93.01666666666667 Val Acc: 93.34\n",
      "Epoch 12\n",
      "Loss:0.24671018318933519 Train Acc: 93.16499999999999 Val Acc: 93.44\n",
      "Epoch 13\n",
      "Loss:0.24152454420953665 Train Acc: 93.29166666666666 Val Acc: 93.55\n",
      "Epoch 14\n",
      "Loss:0.23704692432939622 Train Acc: 93.39 Val Acc: 93.71000000000001\n",
      "Epoch 15\n",
      "Loss:0.23320396536963994 Train Acc: 93.49833333333333 Val Acc: 93.73\n",
      "Epoch 16\n",
      "Loss:0.2299315898850171 Train Acc: 93.58666666666666 Val Acc: 93.8\n",
      "Epoch 17\n",
      "Loss:0.22717982953920027 Train Acc: 93.65666666666667 Val Acc: 93.87\n",
      "Epoch 18\n",
      "Loss:0.22491532022212288 Train Acc: 93.73166666666667 Val Acc: 93.92\n",
      "Epoch 19\n",
      "Loss:0.22311117172146333 Train Acc: 93.78166666666667 Val Acc: 93.95\n",
      "Epoch 20\n",
      "Loss:0.22175059387475463 Train Acc: 93.80333333333334 Val Acc: 93.97\n",
      "Epoch 1\n",
      "Loss:2.150312213917104 Train Acc: 43.265 Val Acc: 43.63\n",
      "Epoch 2\n",
      "Loss:1.7890261757506687 Train Acc: 60.99 Val Acc: 61.870000000000005\n",
      "Epoch 3\n",
      "Loss:1.4460117601283635 Train Acc: 69.90666666666667 Val Acc: 70.96000000000001\n",
      "Epoch 4\n",
      "Loss:1.168107952889529 Train Acc: 75.39 Val Acc: 76.84\n",
      "Epoch 5\n",
      "Loss:0.9724705601734946 Train Acc: 78.975 Val Acc: 80.05\n",
      "Epoch 6\n",
      "Loss:0.8376195252646573 Train Acc: 81.16 Val Acc: 82.39999999999999\n",
      "Epoch 7\n",
      "Loss:0.7421212834307293 Train Acc: 82.87333333333333 Val Acc: 84.07\n",
      "Epoch 8\n",
      "Loss:0.6719863793027627 Train Acc: 84.08 Val Acc: 85.34\n",
      "Epoch 9\n",
      "Loss:0.6186497819406346 Train Acc: 85.05333333333334 Val Acc: 86.32\n",
      "Epoch 10\n",
      "Loss:0.57688431912922 Train Acc: 85.79166666666667 Val Acc: 86.9\n",
      "Epoch 11\n",
      "Loss:0.5433971094005793 Train Acc: 86.40666666666667 Val Acc: 87.42\n",
      "Epoch 12\n",
      "Loss:0.5160228761977372 Train Acc: 86.87 Val Acc: 87.92999999999999\n",
      "Epoch 13\n",
      "Loss:0.4932625978613264 Train Acc: 87.29666666666667 Val Acc: 88.37\n",
      "Epoch 14\n",
      "Loss:0.47404409226387395 Train Acc: 87.63 Val Acc: 88.7\n",
      "Epoch 15\n",
      "Loss:0.457608609215441 Train Acc: 87.90166666666667 Val Acc: 89.03\n",
      "Epoch 16\n",
      "Loss:0.4433997219780487 Train Acc: 88.20166666666667 Val Acc: 89.21\n",
      "Epoch 17\n",
      "Loss:0.430991130208893 Train Acc: 88.41833333333334 Val Acc: 89.38000000000001\n",
      "Epoch 18\n",
      "Loss:0.42005691141774665 Train Acc: 88.62666666666667 Val Acc: 89.53999999999999\n",
      "Epoch 19\n",
      "Loss:0.410339324749434 Train Acc: 88.84333333333333 Val Acc: 89.73\n",
      "Epoch 20\n",
      "Loss:0.4016283660980465 Train Acc: 89.045 Val Acc: 89.92\n",
      "Epoch 1\n",
      "Loss:1.9704602353734222 Train Acc: 60.96666666666667 Val Acc: 61.809999999999995\n",
      "Epoch 2\n",
      "Loss:1.2492045883557268 Train Acc: 77.275 Val Acc: 78.41\n",
      "Epoch 3\n",
      "Loss:0.7752782565332458 Train Acc: 83.99333333333333 Val Acc: 85.2\n",
      "Epoch 4\n",
      "Loss:0.5235083758006135 Train Acc: 87.92 Val Acc: 88.92\n",
      "Epoch 5\n",
      "Loss:0.3309839506914328 Train Acc: 92.805 Val Acc: 93.25\n",
      "Epoch 6\n",
      "Loss:0.213242561200188 Train Acc: 94.88333333333333 Val Acc: 94.82000000000001\n",
      "Epoch 7\n",
      "Loss:0.15996038924505562 Train Acc: 96.18666666666667 Val Acc: 95.88\n",
      "Epoch 8\n",
      "Loss:0.12805185070117284 Train Acc: 96.89666666666666 Val Acc: 96.41999999999999\n",
      "Epoch 9\n",
      "Loss:0.1072331054825482 Train Acc: 97.37666666666667 Val Acc: 96.69\n",
      "Epoch 10\n",
      "Loss:0.09302741390169603 Train Acc: 97.63333333333334 Val Acc: 96.84\n",
      "Epoch 11\n",
      "Loss:0.08170495425120991 Train Acc: 97.85166666666667 Val Acc: 96.99\n",
      "Epoch 12\n",
      "Loss:0.07074861171046833 Train Acc: 98.08 Val Acc: 97.11\n",
      "Epoch 13\n",
      "Loss:0.06088358675288745 Train Acc: 98.24666666666667 Val Acc: 97.06\n",
      "Epoch 14\n",
      "Loss:0.05360022706590098 Train Acc: 98.37666666666667 Val Acc: 97.05\n",
      "Epoch 15\n",
      "Loss:0.047663183301725226 Train Acc: 98.58166666666666 Val Acc: 97.25\n",
      "Epoch 16\n",
      "Loss:0.04280971332381001 Train Acc: 98.75833333333334 Val Acc: 97.39\n",
      "Epoch 17\n",
      "Loss:0.03891703171194897 Train Acc: 98.86833333333334 Val Acc: 97.50999999999999\n",
      "Epoch 18\n",
      "Loss:0.035632458849953565 Train Acc: 99.00833333333333 Val Acc: 97.54\n",
      "Epoch 19\n",
      "Loss:0.03289496956360477 Train Acc: 99.16666666666667 Val Acc: 97.7\n",
      "Epoch 20\n",
      "Loss:0.03069104717512433 Train Acc: 99.225 Val Acc: 97.72999999999999\n",
      "Epoch 1\n",
      "Loss:1.1642016084499796 Train Acc: 84.85333333333334 Val Acc: 86.08\n",
      "Epoch 2\n",
      "Loss:0.4806458785894338 Train Acc: 88.54833333333333 Val Acc: 89.52\n",
      "Epoch 3\n",
      "Loss:0.38362110879260547 Train Acc: 89.93333333333334 Val Acc: 90.74\n",
      "Epoch 4\n",
      "Loss:0.3433857232655801 Train Acc: 90.67666666666668 Val Acc: 91.32000000000001\n",
      "Epoch 5\n",
      "Loss:0.31904082564565445 Train Acc: 91.21666666666667 Val Acc: 91.74\n",
      "Epoch 6\n",
      "Loss:0.3015359503814585 Train Acc: 91.625 Val Acc: 92.06\n",
      "Epoch 7\n",
      "Loss:0.28785468308708484 Train Acc: 91.99000000000001 Val Acc: 92.43\n",
      "Epoch 8\n",
      "Loss:0.27670844594355565 Train Acc: 92.33 Val Acc: 92.7\n",
      "Epoch 9\n",
      "Loss:0.26735741393562656 Train Acc: 92.62166666666667 Val Acc: 92.94\n",
      "Epoch 10\n",
      "Loss:0.2593869526325768 Train Acc: 92.83333333333333 Val Acc: 93.11\n",
      "Epoch 11\n",
      "Loss:0.2525022200321837 Train Acc: 93.00666666666667 Val Acc: 93.36\n",
      "Epoch 12\n",
      "Loss:0.24652098879491402 Train Acc: 93.175 Val Acc: 93.41000000000001\n",
      "Epoch 13\n",
      "Loss:0.24135376910101974 Train Acc: 93.28999999999999 Val Acc: 93.54\n",
      "Epoch 14\n",
      "Loss:0.23688461790934098 Train Acc: 93.4 Val Acc: 93.69\n",
      "Epoch 15\n",
      "Loss:0.233042576178083 Train Acc: 93.49499999999999 Val Acc: 93.75\n",
      "Epoch 16\n",
      "Loss:0.2297726493963629 Train Acc: 93.58833333333332 Val Acc: 93.81\n",
      "Epoch 17\n",
      "Loss:0.22702523953882325 Train Acc: 93.65333333333334 Val Acc: 93.87\n",
      "Epoch 18\n",
      "Loss:0.22477036248217477 Train Acc: 93.73 Val Acc: 93.93\n",
      "Epoch 19\n",
      "Loss:0.22298424103441822 Train Acc: 93.77499999999999 Val Acc: 93.96\n",
      "Epoch 20\n",
      "Loss:0.22165722817133504 Train Acc: 93.79333333333332 Val Acc: 93.99\n",
      "Epoch 1\n",
      "Loss:0.9514762519950675 Train Acc: 85.97 Val Acc: 87.3\n",
      "Epoch 2\n",
      "Loss:0.4852796125032019 Train Acc: 88.715 Val Acc: 89.64\n",
      "Epoch 3\n",
      "Loss:0.40435011652071595 Train Acc: 89.75166666666667 Val Acc: 90.60000000000001\n",
      "Epoch 4\n",
      "Loss:0.3658523918826436 Train Acc: 90.41833333333334 Val Acc: 90.98\n",
      "Epoch 5\n",
      "Loss:0.3416540099021146 Train Acc: 90.94 Val Acc: 91.47\n",
      "Epoch 6\n",
      "Loss:0.32422532419975825 Train Acc: 91.27 Val Acc: 91.79\n",
      "Epoch 7\n",
      "Loss:0.3106259144425639 Train Acc: 91.57499999999999 Val Acc: 91.97\n",
      "Epoch 8\n",
      "Loss:0.29941988316962515 Train Acc: 91.83 Val Acc: 92.22\n",
      "Epoch 9\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:0.28994856193572865 Train Acc: 92.085 Val Acc: 92.46\n",
      "Epoch 10\n",
      "Loss:0.2817776924594709 Train Acc: 92.28333333333333 Val Acc: 92.58999999999999\n",
      "Epoch 11\n",
      "Loss:0.27458352316899004 Train Acc: 92.47 Val Acc: 92.78\n",
      "Epoch 12\n",
      "Loss:0.26814711451079226 Train Acc: 92.64166666666667 Val Acc: 92.96\n",
      "Epoch 13\n",
      "Loss:0.2623137886995826 Train Acc: 92.785 Val Acc: 93.07\n",
      "Epoch 14\n",
      "Loss:0.2569929493961753 Train Acc: 92.93666666666667 Val Acc: 93.17999999999999\n",
      "Epoch 15\n",
      "Loss:0.25209256177888256 Train Acc: 93.035 Val Acc: 93.25\n",
      "Epoch 16\n",
      "Loss:0.24753349497390942 Train Acc: 93.16166666666666 Val Acc: 93.37\n",
      "Epoch 17\n",
      "Loss:0.2432614575322403 Train Acc: 93.27499999999999 Val Acc: 93.52000000000001\n",
      "Epoch 18\n",
      "Loss:0.23923434705047805 Train Acc: 93.37333333333333 Val Acc: 93.60000000000001\n",
      "Epoch 19\n",
      "Loss:0.23542810919125237 Train Acc: 93.46333333333334 Val Acc: 93.66\n",
      "Epoch 20\n",
      "Loss:0.23184784085697988 Train Acc: 93.57 Val Acc: 93.78999999999999\n",
      "Epoch 1\n",
      "Loss:0.3494961630785522 Train Acc: 93.50666666666667 Val Acc: 93.25\n",
      "Epoch 2\n",
      "Loss:0.16426109799705413 Train Acc: 95.82333333333334 Val Acc: 95.52000000000001\n",
      "Epoch 3\n",
      "Loss:0.1268348128420613 Train Acc: 96.825 Val Acc: 96.32\n",
      "Epoch 4\n",
      "Loss:0.10772133084823728 Train Acc: 97.345 Val Acc: 96.66\n",
      "Epoch 5\n",
      "Loss:0.09654084214486326 Train Acc: 97.785 Val Acc: 96.88\n",
      "Epoch 6\n",
      "Loss:0.09060500542980887 Train Acc: 98.01666666666667 Val Acc: 97.08\n",
      "Epoch 7\n",
      "Loss:0.08587229819013457 Train Acc: 98.18666666666667 Val Acc: 97.11999999999999\n",
      "Epoch 8\n",
      "Loss:0.08136149266792489 Train Acc: 98.265 Val Acc: 96.99\n",
      "Epoch 9\n",
      "Loss:0.07906671621906984 Train Acc: 98.34666666666666 Val Acc: 97.03\n",
      "Epoch 10\n",
      "Loss:0.07584279442077525 Train Acc: 98.34166666666667 Val Acc: 96.89\n",
      "Epoch 11\n",
      "Loss:0.07550684014569611 Train Acc: 98.53333333333333 Val Acc: 97.15\n",
      "Epoch 12\n",
      "Loss:0.07328849074872983 Train Acc: 98.65166666666667 Val Acc: 97.24000000000001\n",
      "Epoch 13\n",
      "Loss:0.07222864501038206 Train Acc: 98.62333333333333 Val Acc: 97.14\n",
      "Epoch 14\n",
      "Loss:0.06967227154613703 Train Acc: 98.69 Val Acc: 97.11999999999999\n",
      "Epoch 15\n",
      "Loss:0.06742340212418112 Train Acc: 98.79 Val Acc: 97.16\n",
      "Epoch 16\n",
      "Loss:0.0636177696059172 Train Acc: 98.83333333333333 Val Acc: 97.27\n",
      "Epoch 17\n",
      "Loss:0.06178256694947343 Train Acc: 98.94833333333334 Val Acc: 97.25\n",
      "Epoch 18\n",
      "Loss:0.05962733737163606 Train Acc: 98.90833333333333 Val Acc: 97.17\n",
      "Epoch 19\n",
      "Loss:0.05649972195690309 Train Acc: 98.92 Val Acc: 97.22\n",
      "Epoch 20\n",
      "Loss:0.05534258714054008 Train Acc: 99.04666666666667 Val Acc: 97.33000000000001\n",
      "Epoch 1\n",
      "Loss:0.5122931164125435 Train Acc: 91.13 Val Acc: 91.72\n",
      "Epoch 2\n",
      "Loss:0.24122521956614748 Train Acc: 93.63833333333334 Val Acc: 93.64\n",
      "Epoch 3\n",
      "Loss:0.19096595483785972 Train Acc: 94.87666666666667 Val Acc: 94.75\n",
      "Epoch 4\n",
      "Loss:0.16173750372849796 Train Acc: 95.67 Val Acc: 95.62\n",
      "Epoch 5\n",
      "Loss:0.14253264505955784 Train Acc: 96.12833333333334 Val Acc: 96.00999999999999\n",
      "Epoch 6\n",
      "Loss:0.1292143386631443 Train Acc: 96.54333333333334 Val Acc: 96.25\n",
      "Epoch 7\n",
      "Loss:0.11944756566132668 Train Acc: 96.82166666666666 Val Acc: 96.46000000000001\n",
      "Epoch 8\n",
      "Loss:0.11144714572905384 Train Acc: 97.045 Val Acc: 96.58\n",
      "Epoch 9\n",
      "Loss:0.10507834269378705 Train Acc: 97.20666666666666 Val Acc: 96.69\n",
      "Epoch 10\n",
      "Loss:0.09998210796338704 Train Acc: 97.32666666666667 Val Acc: 96.78\n",
      "Epoch 11\n",
      "Loss:0.09579553266463074 Train Acc: 97.41499999999999 Val Acc: 96.85000000000001\n",
      "Epoch 12\n",
      "Loss:0.09221897151582907 Train Acc: 97.54 Val Acc: 96.89\n",
      "Epoch 13\n",
      "Loss:0.08888134116143143 Train Acc: 97.65833333333333 Val Acc: 96.94\n",
      "Epoch 14\n",
      "Loss:0.085857908128468 Train Acc: 97.725 Val Acc: 97.02\n",
      "Epoch 15\n",
      "Loss:0.08304233492551626 Train Acc: 97.81 Val Acc: 97.05\n",
      "Epoch 16\n",
      "Loss:0.0808786201700466 Train Acc: 97.87 Val Acc: 97.1\n",
      "Epoch 17\n",
      "Loss:0.07904081253451758 Train Acc: 97.92 Val Acc: 97.14\n",
      "Epoch 18\n",
      "Loss:0.07742831208009768 Train Acc: 97.98833333333333 Val Acc: 97.17\n",
      "Epoch 19\n",
      "Loss:0.07582162127056254 Train Acc: 98.05 Val Acc: 97.16\n",
      "Epoch 20\n",
      "Loss:0.07447354000926483 Train Acc: 98.08166666666666 Val Acc: 97.18\n",
      "Epoch 1\n",
      "Loss:0.34624059333092777 Train Acc: 94.435 Val Acc: 94.15\n",
      "Epoch 2\n",
      "Loss:0.14825623941078148 Train Acc: 96.23833333333334 Val Acc: 95.71\n",
      "Epoch 3\n",
      "Loss:0.11145417180169578 Train Acc: 97.12833333333334 Val Acc: 96.46000000000001\n",
      "Epoch 4\n",
      "Loss:0.08935200043307788 Train Acc: 97.67166666666667 Val Acc: 96.72\n",
      "Epoch 5\n",
      "Loss:0.07398113188749977 Train Acc: 98.00666666666666 Val Acc: 96.91\n",
      "Epoch 6\n",
      "Loss:0.06255578133234581 Train Acc: 98.21833333333333 Val Acc: 97.06\n",
      "Epoch 7\n",
      "Loss:0.05367491966493914 Train Acc: 98.42 Val Acc: 97.18\n",
      "Epoch 8\n",
      "Loss:0.04643268179420983 Train Acc: 98.58666666666667 Val Acc: 97.24000000000001\n",
      "Epoch 9\n",
      "Loss:0.0401502688351156 Train Acc: 98.67 Val Acc: 97.26\n",
      "Epoch 10\n",
      "Loss:0.03492799950511688 Train Acc: 98.7 Val Acc: 97.27\n",
      "Epoch 11\n",
      "Loss:0.030460472903013912 Train Acc: 98.89666666666666 Val Acc: 97.35000000000001\n",
      "Epoch 12\n",
      "Loss:0.026486564063315152 Train Acc: 98.96000000000001 Val Acc: 97.32\n",
      "Epoch 13\n",
      "Loss:0.023012764999635223 Train Acc: 99.21833333333333 Val Acc: 97.46000000000001\n",
      "Epoch 14\n",
      "Loss:0.019850895986943798 Train Acc: 99.27333333333334 Val Acc: 97.52\n",
      "Epoch 15\n",
      "Loss:0.017092940164303613 Train Acc: 99.345 Val Acc: 97.55\n",
      "Epoch 16\n",
      "Loss:0.014669258212178982 Train Acc: 99.36333333333334 Val Acc: 97.57000000000001\n",
      "Epoch 17\n",
      "Loss:0.01263727042925123 Train Acc: 99.38166666666667 Val Acc: 97.63\n",
      "Epoch 18\n",
      "Loss:0.010859550462818085 Train Acc: 99.43333333333332 Val Acc: 97.59\n",
      "Epoch 19\n",
      "Loss:0.009267488182499615 Train Acc: 99.47166666666666 Val Acc: 97.67\n",
      "Epoch 20\n",
      "Loss:0.00798217519584946 Train Acc: 99.55666666666667 Val Acc: 97.65\n",
      "Epoch 1\n",
      "Loss:0.34693777097493633 Train Acc: 94.52000000000001 Val Acc: 94.31\n",
      "Epoch 2\n",
      "Loss:0.1488490048492696 Train Acc: 96.09333333333333 Val Acc: 95.62\n",
      "Epoch 3\n",
      "Loss:0.11159587537012475 Train Acc: 97.095 Val Acc: 96.44\n",
      "Epoch 4\n",
      "Loss:0.0891733326214939 Train Acc: 97.72 Val Acc: 96.81\n",
      "Epoch 5\n",
      "Loss:0.07358275141546555 Train Acc: 98.03 Val Acc: 97.0\n",
      "Epoch 6\n",
      "Loss:0.0620658785384294 Train Acc: 98.25500000000001 Val Acc: 97.07000000000001\n",
      "Epoch 7\n",
      "Loss:0.05303920198505946 Train Acc: 98.45166666666667 Val Acc: 97.17\n",
      "Epoch 8\n",
      "Loss:0.04586183019791281 Train Acc: 98.575 Val Acc: 97.23\n",
      "Epoch 9\n",
      "Loss:0.03975036610130255 Train Acc: 98.75166666666667 Val Acc: 97.27\n",
      "Epoch 10\n",
      "Loss:0.03445315512520775 Train Acc: 98.845 Val Acc: 97.23\n",
      "Epoch 11\n",
      "Loss:0.029861953742705956 Train Acc: 98.77333333333334 Val Acc: 97.1\n",
      "Epoch 12\n",
      "Loss:0.02617570811801098 Train Acc: 98.75500000000001 Val Acc: 96.93\n",
      "Epoch 13\n",
      "Loss:0.022752301635561162 Train Acc: 99.01666666666667 Val Acc: 97.09\n",
      "Epoch 14\n",
      "Loss:0.01991587680840194 Train Acc: 99.10333333333334 Val Acc: 97.14\n",
      "Epoch 15\n",
      "Loss:0.017317125591054108 Train Acc: 99.31833333333333 Val Acc: 97.44\n",
      "Epoch 16\n",
      "Loss:0.014870606691445454 Train Acc: 99.33833333333332 Val Acc: 97.44\n",
      "Epoch 17\n",
      "Loss:0.012834034942881241 Train Acc: 99.35000000000001 Val Acc: 97.56\n",
      "Epoch 18\n",
      "Loss:0.010982917268028492 Train Acc: 99.49666666666667 Val Acc: 97.64\n",
      "Epoch 19\n",
      "Loss:0.009360162475636528 Train Acc: 99.435 Val Acc: 97.64\n",
      "Epoch 20\n",
      "Loss:0.007910447361174723 Train Acc: 99.53666666666666 Val Acc: 97.7\n",
      "Epoch 1\n",
      "Loss:0.34525525726787065 Train Acc: 94.59833333333333 Val Acc: 94.27\n",
      "Epoch 2\n",
      "Loss:0.14732157574524915 Train Acc: 96.05166666666666 Val Acc: 95.49\n",
      "Epoch 3\n",
      "Loss:0.11051390974017611 Train Acc: 97.03333333333333 Val Acc: 96.47\n",
      "Epoch 4\n",
      "Loss:0.08838121951405653 Train Acc: 97.60666666666667 Val Acc: 96.78999999999999\n",
      "Epoch 5\n",
      "Loss:0.07312186593982441 Train Acc: 97.955 Val Acc: 96.89999999999999\n",
      "Epoch 6\n",
      "Loss:0.06158788856300335 Train Acc: 98.15166666666667 Val Acc: 97.03\n",
      "Epoch 7\n",
      "Loss:0.05276522536682005 Train Acc: 98.41666666666666 Val Acc: 97.21\n",
      "Epoch 8\n",
      "Loss:0.04539148578620157 Train Acc: 98.65833333333333 Val Acc: 97.25\n",
      "Epoch 9\n",
      "Loss:0.0392544334274763 Train Acc: 98.82666666666667 Val Acc: 97.38\n",
      "Epoch 10\n",
      "Loss:0.03391315881523364 Train Acc: 99.04666666666667 Val Acc: 97.44\n",
      "Epoch 11\n",
      "Loss:0.02937219519817275 Train Acc: 99.14833333333334 Val Acc: 97.50999999999999\n",
      "Epoch 12\n",
      "Loss:0.025526659775357965 Train Acc: 99.17166666666667 Val Acc: 97.46000000000001\n",
      "Epoch 13\n",
      "Loss:0.022093906670588394 Train Acc: 99.21666666666667 Val Acc: 97.48\n",
      "Epoch 14\n",
      "Loss:0.019126039237167356 Train Acc: 99.27333333333334 Val Acc: 97.50999999999999\n",
      "Epoch 15\n",
      "Loss:0.01652421705560063 Train Acc: 99.41666666666666 Val Acc: 97.59\n",
      "Epoch 16\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:0.014178155887311703 Train Acc: 99.43333333333332 Val Acc: 97.58\n",
      "Epoch 17\n",
      "Loss:0.012253706645026983 Train Acc: 99.515 Val Acc: 97.63\n",
      "Epoch 18\n",
      "Loss:0.010595784027540469 Train Acc: 99.50666666666666 Val Acc: 97.58\n",
      "Epoch 19\n",
      "Loss:0.009213784338289014 Train Acc: 99.57166666666667 Val Acc: 97.65\n",
      "Epoch 20\n",
      "Loss:0.008039320337332293 Train Acc: 99.38166666666667 Val Acc: 97.46000000000001\n",
      "Epoch 1\n",
      "Loss:0.3466096925089451 Train Acc: 94.63000000000001 Val Acc: 94.42\n",
      "Epoch 2\n",
      "Loss:0.14803258841396658 Train Acc: 96.11666666666666 Val Acc: 95.5\n",
      "Epoch 3\n",
      "Loss:0.11072932136290548 Train Acc: 97.11833333333333 Val Acc: 96.52\n",
      "Epoch 4\n",
      "Loss:0.0885864731584125 Train Acc: 97.65166666666667 Val Acc: 96.88\n",
      "Epoch 5\n",
      "Loss:0.07329138452387216 Train Acc: 98.02666666666666 Val Acc: 97.11\n",
      "Epoch 6\n",
      "Loss:0.06192593411986874 Train Acc: 98.28833333333333 Val Acc: 97.21\n",
      "Epoch 7\n",
      "Loss:0.052738580067773363 Train Acc: 98.52499999999999 Val Acc: 97.37\n",
      "Epoch 8\n",
      "Loss:0.04525772289855424 Train Acc: 98.72833333333332 Val Acc: 97.39\n",
      "Epoch 9\n",
      "Loss:0.03917564947688579 Train Acc: 98.86166666666666 Val Acc: 97.47\n",
      "Epoch 10\n",
      "Loss:0.03580128426660475 Train Acc: 99.06833333333334 Val Acc: 97.47\n",
      "Epoch 11\n",
      "Loss:0.02903739373850233 Train Acc: 99.17166666666667 Val Acc: 97.49\n",
      "Epoch 12\n",
      "Loss:0.025051853838685707 Train Acc: 99.24833333333333 Val Acc: 97.53\n",
      "Epoch 13\n",
      "Loss:0.021577411668582847 Train Acc: 99.33 Val Acc: 97.52\n",
      "Epoch 14\n",
      "Loss:0.018636467453626585 Train Acc: 99.32666666666667 Val Acc: 97.48\n",
      "Epoch 15\n",
      "Loss:0.016108984884545166 Train Acc: 99.435 Val Acc: 97.52\n",
      "Epoch 16\n",
      "Loss:0.013811973114091295 Train Acc: 99.46333333333334 Val Acc: 97.56\n",
      "Epoch 17\n",
      "Loss:0.011960577389556193 Train Acc: 99.51166666666667 Val Acc: 97.57000000000001\n",
      "Epoch 18\n",
      "Loss:0.010261760977159808 Train Acc: 99.53999999999999 Val Acc: 97.53\n",
      "Epoch 19\n",
      "Loss:0.008815117777721067 Train Acc: 99.52833333333334 Val Acc: 97.46000000000001\n",
      "Epoch 20\n",
      "Loss:0.022477461307888837 Train Acc: 98.425 Val Acc: 96.76\n",
      "Epoch 1\n",
      "Loss:0.321979028553172 Train Acc: 95.02333333333334 Val Acc: 94.82000000000001\n",
      "Epoch 2\n",
      "Loss:0.14077298022238383 Train Acc: 96.375 Val Acc: 95.78999999999999\n",
      "Epoch 3\n",
      "Loss:0.10615261453854963 Train Acc: 97.17666666666666 Val Acc: 96.39999999999999\n",
      "Epoch 4\n",
      "Loss:0.08590244875327319 Train Acc: 97.68833333333333 Val Acc: 96.76\n",
      "Epoch 5\n",
      "Loss:0.07158444184251983 Train Acc: 98.05833333333334 Val Acc: 97.05\n",
      "Epoch 6\n",
      "Loss:0.060547327437613936 Train Acc: 98.34166666666667 Val Acc: 97.09\n",
      "Epoch 7\n",
      "Loss:0.051970217660577106 Train Acc: 98.53666666666666 Val Acc: 97.19\n",
      "Epoch 8\n",
      "Loss:0.04502583817675863 Train Acc: 98.695 Val Acc: 97.3\n",
      "Epoch 9\n",
      "Loss:0.039142914501064545 Train Acc: 98.785 Val Acc: 97.28\n",
      "Epoch 10\n",
      "Loss:0.034078582520819474 Train Acc: 98.91166666666666 Val Acc: 97.27\n",
      "Epoch 11\n",
      "Loss:0.02959670209452659 Train Acc: 99.04166666666666 Val Acc: 97.36\n",
      "Epoch 12\n",
      "Loss:0.025791835713915235 Train Acc: 99.13 Val Acc: 97.39\n",
      "Epoch 13\n",
      "Loss:0.022342333082251115 Train Acc: 99.21 Val Acc: 97.48\n",
      "Epoch 14\n",
      "Loss:0.01939690804295289 Train Acc: 99.31166666666667 Val Acc: 97.5\n",
      "Epoch 15\n",
      "Loss:0.01670604551772691 Train Acc: 99.39666666666666 Val Acc: 97.59\n",
      "Epoch 16\n",
      "Loss:0.014377306149196025 Train Acc: 99.49833333333333 Val Acc: 97.61999999999999\n",
      "Epoch 17\n",
      "Loss:0.01238276999178871 Train Acc: 99.52 Val Acc: 97.71\n",
      "Epoch 18\n",
      "Loss:0.01061498786924163 Train Acc: 99.605 Val Acc: 97.72\n",
      "Epoch 19\n",
      "Loss:0.00909216970155948 Train Acc: 99.63666666666666 Val Acc: 97.67\n",
      "Epoch 20\n",
      "Loss:0.007759226196403027 Train Acc: 99.68666666666667 Val Acc: 97.68\n",
      "Epoch 1\n",
      "Loss:0.3215483091304527 Train Acc: 95.05333333333333 Val Acc: 94.74000000000001\n",
      "Epoch 2\n",
      "Loss:0.13985205819736346 Train Acc: 96.475 Val Acc: 95.82000000000001\n",
      "Epoch 3\n",
      "Loss:0.10520105593215903 Train Acc: 97.235 Val Acc: 96.47\n",
      "Epoch 4\n",
      "Loss:0.08508716107822922 Train Acc: 97.775 Val Acc: 96.81\n",
      "Epoch 5\n",
      "Loss:0.0708798812024602 Train Acc: 98.08666666666667 Val Acc: 97.03\n",
      "Epoch 6\n",
      "Loss:0.06021236681504622 Train Acc: 98.37333333333333 Val Acc: 97.21\n",
      "Epoch 7\n",
      "Loss:0.05174336512756643 Train Acc: 98.56333333333333 Val Acc: 97.31\n",
      "Epoch 8\n",
      "Loss:0.044670462160792464 Train Acc: 98.75333333333333 Val Acc: 97.38\n",
      "Epoch 9\n",
      "Loss:0.038700038689079175 Train Acc: 98.89 Val Acc: 97.47\n",
      "Epoch 10\n",
      "Loss:0.03357192611333592 Train Acc: 98.95 Val Acc: 97.32\n",
      "Epoch 11\n",
      "Loss:0.02931279920034709 Train Acc: 99.00166666666667 Val Acc: 97.3\n",
      "Epoch 12\n",
      "Loss:0.02547647178724195 Train Acc: 99.11666666666666 Val Acc: 97.39999999999999\n",
      "Epoch 13\n",
      "Loss:0.022122846769928615 Train Acc: 99.21833333333333 Val Acc: 97.49\n",
      "Epoch 14\n",
      "Loss:0.019166720964350188 Train Acc: 99.34666666666668 Val Acc: 97.59\n",
      "Epoch 15\n",
      "Loss:0.016596405074034558 Train Acc: 99.43666666666667 Val Acc: 97.63\n",
      "Epoch 16\n",
      "Loss:0.014312159624314778 Train Acc: 99.52 Val Acc: 97.64\n",
      "Epoch 17\n",
      "Loss:0.012266896310830883 Train Acc: 99.55166666666668 Val Acc: 97.65\n",
      "Epoch 18\n",
      "Loss:0.0106314657345602 Train Acc: 99.56666666666666 Val Acc: 97.66\n",
      "Epoch 19\n",
      "Loss:0.009101187444271354 Train Acc: 99.54833333333333 Val Acc: 97.6\n",
      "Epoch 20\n",
      "Loss:0.007846651477564191 Train Acc: 99.53999999999999 Val Acc: 97.61999999999999\n",
      "Epoch 1\n",
      "Loss:0.32014620290691254 Train Acc: 95.02833333333334 Val Acc: 94.8\n",
      "Epoch 2\n",
      "Loss:0.1391438229961789 Train Acc: 96.36666666666667 Val Acc: 95.69\n",
      "Epoch 3\n",
      "Loss:0.10496703897027956 Train Acc: 97.14333333333333 Val Acc: 96.46000000000001\n",
      "Epoch 4\n",
      "Loss:0.08485596418550473 Train Acc: 97.61 Val Acc: 96.86\n",
      "Epoch 5\n",
      "Loss:0.07073215397064359 Train Acc: 97.93166666666666 Val Acc: 97.04\n",
      "Epoch 6\n",
      "Loss:0.060211139006792765 Train Acc: 98.225 Val Acc: 97.16\n",
      "Epoch 7\n",
      "Loss:0.05185836908152471 Train Acc: 98.50333333333333 Val Acc: 97.22\n",
      "Epoch 8\n",
      "Loss:0.04489480716572345 Train Acc: 98.67666666666666 Val Acc: 97.31\n",
      "Epoch 9\n",
      "Loss:0.03891819931618054 Train Acc: 98.875 Val Acc: 97.34\n",
      "Epoch 10\n",
      "Loss:0.03382431813849106 Train Acc: 99.01833333333333 Val Acc: 97.42\n",
      "Epoch 11\n",
      "Loss:0.029380570476330376 Train Acc: 99.11666666666666 Val Acc: 97.42\n",
      "Epoch 12\n",
      "Loss:0.02546365709927213 Train Acc: 99.205 Val Acc: 97.45\n",
      "Epoch 13\n",
      "Loss:0.02217254420912456 Train Acc: 99.29666666666667 Val Acc: 97.5\n",
      "Epoch 14\n",
      "Loss:0.0191751223285452 Train Acc: 99.34833333333334 Val Acc: 97.53\n",
      "Epoch 15\n",
      "Loss:0.016691632642621877 Train Acc: 99.395 Val Acc: 97.54\n",
      "Epoch 16\n",
      "Loss:0.014469152735911534 Train Acc: 99.425 Val Acc: 97.53\n",
      "Epoch 17\n",
      "Loss:0.012606555986105774 Train Acc: 99.43833333333333 Val Acc: 97.50999999999999\n",
      "Epoch 18\n",
      "Loss:0.010928345544080419 Train Acc: 99.46166666666667 Val Acc: 97.46000000000001\n",
      "Epoch 19\n",
      "Loss:0.00947591930587324 Train Acc: 99.42999999999999 Val Acc: 97.34\n",
      "Epoch 20\n",
      "Loss:0.008168001241785745 Train Acc: 99.335 Val Acc: 97.3\n",
      "Epoch 1\n",
      "Loss:0.32098571532793757 Train Acc: 95.08166666666666 Val Acc: 94.71000000000001\n",
      "Epoch 2\n",
      "Loss:0.13938636558401085 Train Acc: 96.365 Val Acc: 95.7\n",
      "Epoch 3\n",
      "Loss:0.10515266917207738 Train Acc: 97.16833333333334 Val Acc: 96.36\n",
      "Epoch 4\n",
      "Loss:0.08536353357077302 Train Acc: 97.60666666666667 Val Acc: 96.72\n",
      "Epoch 5\n",
      "Loss:0.07127104034519345 Train Acc: 97.93833333333333 Val Acc: 96.96000000000001\n",
      "Epoch 6\n",
      "Loss:0.06054816451178312 Train Acc: 98.25333333333333 Val Acc: 97.11999999999999\n",
      "Epoch 7\n",
      "Loss:0.052185108737742014 Train Acc: 98.46833333333333 Val Acc: 97.38\n",
      "Epoch 8\n",
      "Loss:0.045124168747511344 Train Acc: 98.67666666666666 Val Acc: 97.42\n",
      "Epoch 9\n",
      "Loss:0.039160521297913156 Train Acc: 98.81333333333333 Val Acc: 97.42\n",
      "Epoch 10\n",
      "Loss:0.03390293534561339 Train Acc: 98.96166666666667 Val Acc: 97.49\n",
      "Epoch 11\n",
      "Loss:0.029517737306607058 Train Acc: 99.07666666666667 Val Acc: 97.52\n",
      "Epoch 12\n",
      "Loss:0.028454302101426292 Train Acc: 99.14166666666667 Val Acc: 97.53\n",
      "Epoch 13\n",
      "Loss:0.02343097407165649 Train Acc: 99.22 Val Acc: 97.52\n",
      "Epoch 14\n",
      "Loss:0.019694780231643234 Train Acc: 99.29833333333333 Val Acc: 97.53\n",
      "Epoch 15\n",
      "Loss:0.01689303179854729 Train Acc: 99.41666666666666 Val Acc: 97.52\n",
      "Epoch 16\n",
      "Loss:0.014585680053362278 Train Acc: 99.49833333333333 Val Acc: 97.65\n",
      "Epoch 17\n",
      "Loss:0.012609288020485877 Train Acc: 99.505 Val Acc: 97.53\n",
      "Epoch 18\n",
      "Loss:0.010995294697718245 Train Acc: 99.54499999999999 Val Acc: 97.50999999999999\n",
      "Epoch 19\n",
      "Loss:0.00950886967468698 Train Acc: 99.45666666666668 Val Acc: 97.49\n",
      "Epoch 20\n",
      "Loss:0.00830109227883747 Train Acc: 99.48166666666667 Val Acc: 97.52\n",
      "Epoch 1\n",
      "Loss:1.3882072711887488 Train Acc: 86.12666666666667 Val Acc: 87.3\n",
      "Epoch 2\n",
      "Loss:0.4592084660934654 Train Acc: 89.67666666666668 Val Acc: 90.56\n",
      "Epoch 3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:0.3531753695219789 Train Acc: 91.08333333333334 Val Acc: 91.59\n",
      "Epoch 4\n",
      "Loss:0.30761378067950385 Train Acc: 91.97666666666666 Val Acc: 92.35\n",
      "Epoch 5\n",
      "Loss:0.2787598133094216 Train Acc: 92.61 Val Acc: 92.99\n",
      "Epoch 6\n",
      "Loss:0.2571553470956499 Train Acc: 93.14333333333333 Val Acc: 93.47999999999999\n",
      "Epoch 7\n",
      "Loss:0.23938363252245923 Train Acc: 93.60333333333334 Val Acc: 93.81\n",
      "Epoch 8\n",
      "Loss:0.22425914754146434 Train Acc: 94.01833333333333 Val Acc: 94.04\n",
      "Epoch 9\n",
      "Loss:0.2112139834034975 Train Acc: 94.34166666666667 Val Acc: 94.36\n",
      "Epoch 10\n",
      "Loss:0.19980202596668398 Train Acc: 94.64666666666666 Val Acc: 94.62\n",
      "Epoch 11\n",
      "Loss:0.1897602969881926 Train Acc: 94.89166666666667 Val Acc: 94.87\n",
      "Epoch 12\n",
      "Loss:0.18085804369809824 Train Acc: 95.12666666666667 Val Acc: 95.07\n",
      "Epoch 13\n",
      "Loss:0.17287640880367106 Train Acc: 95.34666666666666 Val Acc: 95.21\n",
      "Epoch 14\n",
      "Loss:0.16567168984416727 Train Acc: 95.53666666666668 Val Acc: 95.36\n",
      "Epoch 15\n",
      "Loss:0.15910550063302167 Train Acc: 95.73333333333333 Val Acc: 95.55\n",
      "Epoch 16\n",
      "Loss:0.15308534241829896 Train Acc: 95.895 Val Acc: 95.69\n",
      "Epoch 17\n",
      "Loss:0.14755990532546973 Train Acc: 96.035 Val Acc: 95.81\n",
      "Epoch 18\n",
      "Loss:0.14244969603749383 Train Acc: 96.16833333333334 Val Acc: 95.86\n",
      "Epoch 19\n",
      "Loss:0.13767040007548587 Train Acc: 96.285 Val Acc: 95.96000000000001\n",
      "Epoch 20\n",
      "Loss:0.13322062984899766 Train Acc: 96.36333333333333 Val Acc: 95.98\n",
      "Epoch 1\n",
      "Loss:1.3882104360795142 Train Acc: 86.11999999999999 Val Acc: 87.29\n",
      "Epoch 2\n",
      "Loss:0.4592105337041899 Train Acc: 89.67 Val Acc: 90.56\n",
      "Epoch 3\n",
      "Loss:0.3531785105142862 Train Acc: 91.08500000000001 Val Acc: 91.58\n",
      "Epoch 4\n",
      "Loss:0.30760987848389315 Train Acc: 91.97666666666666 Val Acc: 92.36999999999999\n",
      "Epoch 5\n",
      "Loss:0.278751331061059 Train Acc: 92.61500000000001 Val Acc: 92.99\n",
      "Epoch 6\n",
      "Loss:0.2571209979588024 Train Acc: 93.15166666666667 Val Acc: 93.45\n",
      "Epoch 7\n",
      "Loss:0.23933931148153087 Train Acc: 93.60666666666667 Val Acc: 93.81\n",
      "Epoch 8\n",
      "Loss:0.22422115371468035 Train Acc: 94.01333333333334 Val Acc: 94.02000000000001\n",
      "Epoch 9\n",
      "Loss:0.21114966708904254 Train Acc: 94.34333333333333 Val Acc: 94.35\n",
      "Epoch 10\n",
      "Loss:0.19973252755701543 Train Acc: 94.64166666666667 Val Acc: 94.63000000000001\n",
      "Epoch 11\n",
      "Loss:0.18968541554557108 Train Acc: 94.88666666666667 Val Acc: 94.88\n",
      "Epoch 12\n",
      "Loss:0.1807794584691211 Train Acc: 95.13333333333334 Val Acc: 95.06\n",
      "Epoch 13\n",
      "Loss:0.17280468596515322 Train Acc: 95.355 Val Acc: 95.24000000000001\n",
      "Epoch 14\n",
      "Loss:0.16562218414287835 Train Acc: 95.54333333333334 Val Acc: 95.39999999999999\n",
      "Epoch 15\n",
      "Loss:0.159055606433047 Train Acc: 95.72 Val Acc: 95.52000000000001\n",
      "Epoch 16\n",
      "Loss:0.15305188904549744 Train Acc: 95.89166666666667 Val Acc: 95.7\n",
      "Epoch 17\n",
      "Loss:0.14753542908575676 Train Acc: 96.04166666666667 Val Acc: 95.8\n",
      "Epoch 18\n",
      "Loss:0.14240498180171815 Train Acc: 96.16 Val Acc: 95.84\n",
      "Epoch 19\n",
      "Loss:0.13764856107046797 Train Acc: 96.285 Val Acc: 95.95\n",
      "Epoch 20\n",
      "Loss:0.13319514843603894 Train Acc: 96.37166666666667 Val Acc: 95.97\n",
      "Epoch 1\n",
      "Loss:1.3883124147741253 Train Acc: 86.14500000000001 Val Acc: 87.28\n",
      "Epoch 2\n",
      "Loss:0.45949123790568863 Train Acc: 89.68166666666667 Val Acc: 90.53999999999999\n",
      "Epoch 3\n",
      "Loss:0.3534439083880329 Train Acc: 91.07 Val Acc: 91.59\n",
      "Epoch 4\n",
      "Loss:0.3078793403795391 Train Acc: 92.00333333333333 Val Acc: 92.32000000000001\n",
      "Epoch 5\n",
      "Loss:0.2790411596399793 Train Acc: 92.62 Val Acc: 92.93\n",
      "Epoch 6\n",
      "Loss:0.2574393370229785 Train Acc: 93.13 Val Acc: 93.41000000000001\n",
      "Epoch 7\n",
      "Loss:0.23969405869218105 Train Acc: 93.605 Val Acc: 93.7\n",
      "Epoch 8\n",
      "Loss:0.224667070526092 Train Acc: 94.0 Val Acc: 94.05\n",
      "Epoch 9\n",
      "Loss:0.2116535192380939 Train Acc: 94.32666666666667 Val Acc: 94.31\n",
      "Epoch 10\n",
      "Loss:0.2002798744244759 Train Acc: 94.60499999999999 Val Acc: 94.62\n",
      "Epoch 11\n",
      "Loss:0.19030916865276606 Train Acc: 94.87833333333333 Val Acc: 94.89\n",
      "Epoch 12\n",
      "Loss:0.18141738557863687 Train Acc: 95.11666666666667 Val Acc: 95.05\n",
      "Epoch 13\n",
      "Loss:0.17346255448513787 Train Acc: 95.32333333333334 Val Acc: 95.22\n",
      "Epoch 14\n",
      "Loss:0.16629185887287745 Train Acc: 95.535 Val Acc: 95.35\n",
      "Epoch 15\n",
      "Loss:0.15975611347434696 Train Acc: 95.68333333333334 Val Acc: 95.52000000000001\n",
      "Epoch 16\n",
      "Loss:0.15376500491258624 Train Acc: 95.85833333333333 Val Acc: 95.7\n",
      "Epoch 17\n",
      "Loss:0.14824842481975037 Train Acc: 96.0 Val Acc: 95.78\n",
      "Epoch 18\n",
      "Loss:0.14312130288077973 Train Acc: 96.11 Val Acc: 95.88\n",
      "Epoch 19\n",
      "Loss:0.1382985999249652 Train Acc: 96.21499999999999 Val Acc: 95.92\n",
      "Epoch 20\n",
      "Loss:0.13373551734777622 Train Acc: 96.32166666666667 Val Acc: 95.98\n",
      "Epoch 1\n",
      "Loss:1.388313146417635 Train Acc: 86.14166666666667 Val Acc: 87.28\n",
      "Epoch 2\n",
      "Loss:0.4595008652310962 Train Acc: 89.67833333333334 Val Acc: 90.55\n",
      "Epoch 3\n",
      "Loss:0.3534427880159079 Train Acc: 91.06333333333333 Val Acc: 91.59\n",
      "Epoch 4\n",
      "Loss:0.30786026102640457 Train Acc: 92.00833333333334 Val Acc: 92.32000000000001\n",
      "Epoch 5\n",
      "Loss:0.2790362815683766 Train Acc: 92.61666666666667 Val Acc: 92.93\n",
      "Epoch 6\n",
      "Loss:0.2574407310318479 Train Acc: 93.13 Val Acc: 93.37\n",
      "Epoch 7\n",
      "Loss:0.23970516647318857 Train Acc: 93.605 Val Acc: 93.72\n",
      "Epoch 8\n",
      "Loss:0.22466822287637855 Train Acc: 94.00166666666667 Val Acc: 94.05\n",
      "Epoch 9\n",
      "Loss:0.21164026170648326 Train Acc: 94.32000000000001 Val Acc: 94.35\n",
      "Epoch 10\n",
      "Loss:0.20025922975457744 Train Acc: 94.61166666666668 Val Acc: 94.61\n",
      "Epoch 11\n",
      "Loss:0.19026950326968903 Train Acc: 94.87833333333333 Val Acc: 94.88\n",
      "Epoch 12\n",
      "Loss:0.18139148549912748 Train Acc: 95.11833333333334 Val Acc: 95.05\n",
      "Epoch 13\n",
      "Loss:0.17345627954546589 Train Acc: 95.32833333333333 Val Acc: 95.21\n",
      "Epoch 14\n",
      "Loss:0.16629832640282505 Train Acc: 95.52333333333334 Val Acc: 95.35\n",
      "Epoch 15\n",
      "Loss:0.15977370837422528 Train Acc: 95.67999999999999 Val Acc: 95.55\n",
      "Epoch 16\n",
      "Loss:0.1537985816096089 Train Acc: 95.86666666666666 Val Acc: 95.73\n",
      "Epoch 17\n",
      "Loss:0.1482718861193596 Train Acc: 96.00333333333333 Val Acc: 95.78999999999999\n",
      "Epoch 18\n",
      "Loss:0.1431364029359199 Train Acc: 96.12166666666667 Val Acc: 95.88\n",
      "Epoch 19\n",
      "Loss:0.13832805650804503 Train Acc: 96.21833333333333 Val Acc: 95.92\n",
      "Epoch 20\n",
      "Loss:0.13377258620601346 Train Acc: 96.325 Val Acc: 95.98\n",
      "Epoch 1\n",
      "Loss:0.3312074805181328 Train Acc: 93.94333333333333 Val Acc: 93.82000000000001\n",
      "Epoch 2\n",
      "Loss:0.13891918601218176 Train Acc: 95.95666666666666 Val Acc: 95.65\n",
      "Epoch 3\n",
      "Loss:0.10466199651117808 Train Acc: 96.75 Val Acc: 96.2\n",
      "Epoch 4\n",
      "Loss:0.08467346411283565 Train Acc: 97.23666666666666 Val Acc: 96.55\n",
      "Epoch 5\n",
      "Loss:0.07074424357688919 Train Acc: 97.70333333333333 Val Acc: 96.91\n",
      "Epoch 6\n",
      "Loss:0.060021896890597275 Train Acc: 97.98 Val Acc: 96.99\n",
      "Epoch 7\n",
      "Loss:0.05152446912384484 Train Acc: 98.37333333333333 Val Acc: 97.24000000000001\n",
      "Epoch 8\n",
      "Loss:0.04448890285823793 Train Acc: 98.56166666666667 Val Acc: 97.35000000000001\n",
      "Epoch 9\n",
      "Loss:0.03850024877047829 Train Acc: 98.655 Val Acc: 97.39999999999999\n",
      "Epoch 10\n",
      "Loss:0.03335411880561014 Train Acc: 98.83666666666666 Val Acc: 97.39999999999999\n",
      "Epoch 11\n",
      "Loss:0.02896606742055557 Train Acc: 98.96833333333333 Val Acc: 97.38\n",
      "Epoch 12\n",
      "Loss:0.02512201143983417 Train Acc: 99.045 Val Acc: 97.36\n",
      "Epoch 13\n",
      "Loss:0.021713996789965856 Train Acc: 99.16833333333334 Val Acc: 97.41\n",
      "Epoch 14\n",
      "Loss:0.018918071173568373 Train Acc: 99.22833333333332 Val Acc: 97.46000000000001\n",
      "Epoch 15\n",
      "Loss:0.0163515310147612 Train Acc: 99.405 Val Acc: 97.50999999999999\n",
      "Epoch 16\n",
      "Loss:0.014238729290158507 Train Acc: 99.45833333333334 Val Acc: 97.47\n",
      "Epoch 17\n",
      "Loss:0.012364823509698268 Train Acc: 99.565 Val Acc: 97.54\n",
      "Epoch 18\n",
      "Loss:0.010682971190995991 Train Acc: 99.63833333333334 Val Acc: 97.57000000000001\n",
      "Epoch 19\n",
      "Loss:0.0092051766274365 Train Acc: 99.66166666666668 Val Acc: 97.6\n",
      "Epoch 20\n",
      "Loss:0.007995936860663891 Train Acc: 99.71666666666667 Val Acc: 97.69\n",
      "Epoch 1\n",
      "Loss:0.33095683988718727 Train Acc: 94.19166666666666 Val Acc: 94.03\n",
      "Epoch 2\n",
      "Loss:0.13856957846965118 Train Acc: 96.055 Val Acc: 95.76\n",
      "Epoch 3\n",
      "Loss:0.10461709811607098 Train Acc: 96.7 Val Acc: 96.2\n",
      "Epoch 4\n",
      "Loss:0.08449143616535054 Train Acc: 97.31 Val Acc: 96.57\n",
      "Epoch 5\n",
      "Loss:0.07047598288044583 Train Acc: 97.75166666666667 Val Acc: 96.89\n",
      "Epoch 6\n",
      "Loss:0.05967693540684103 Train Acc: 98.02833333333332 Val Acc: 97.02\n",
      "Epoch 7\n",
      "Loss:0.051128995878849635 Train Acc: 98.42833333333333 Val Acc: 97.28999999999999\n",
      "Epoch 8\n",
      "Loss:0.04412926526559461 Train Acc: 98.58333333333333 Val Acc: 97.33000000000001\n",
      "Epoch 9\n",
      "Loss:0.038041487821788754 Train Acc: 98.73166666666665 Val Acc: 97.39999999999999\n",
      "Epoch 10\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss:0.03300715642795326 Train Acc: 98.85000000000001 Val Acc: 97.39999999999999\n",
      "Epoch 11\n",
      "Loss:0.02866261822065572 Train Acc: 98.99333333333334 Val Acc: 97.46000000000001\n",
      "Epoch 12\n",
      "Loss:0.024774177824879656 Train Acc: 99.09666666666666 Val Acc: 97.47\n",
      "Epoch 13\n",
      "Loss:0.021464150281812106 Train Acc: 99.21833333333333 Val Acc: 97.55\n",
      "Epoch 14\n",
      "Loss:0.018443837829816855 Train Acc: 99.26166666666667 Val Acc: 97.50999999999999\n",
      "Epoch 15\n",
      "Loss:0.01598987057260612 Train Acc: 99.41499999999999 Val Acc: 97.50999999999999\n",
      "Epoch 16\n",
      "Loss:0.013841694731086965 Train Acc: 99.48 Val Acc: 97.56\n",
      "Epoch 17\n",
      "Loss:0.011926170272867487 Train Acc: 99.555 Val Acc: 97.61999999999999\n",
      "Epoch 18\n",
      "Loss:0.010288358322347235 Train Acc: 99.63666666666666 Val Acc: 97.64\n",
      "Epoch 19\n",
      "Loss:0.008914418972177516 Train Acc: 99.68 Val Acc: 97.68\n",
      "Epoch 20\n",
      "Loss:0.007675716733578058 Train Acc: 99.73333333333333 Val Acc: 97.69\n",
      "Epoch 1\n",
      "Loss:0.3312229890195404 Train Acc: 94.08333333333333 Val Acc: 93.96\n",
      "Epoch 2\n",
      "Loss:0.13967710416486043 Train Acc: 96.14 Val Acc: 95.74000000000001\n",
      "Epoch 3\n",
      "Loss:0.10565062317469798 Train Acc: 96.73666666666666 Val Acc: 96.19\n",
      "Epoch 4\n",
      "Loss:0.08571282886676496 Train Acc: 97.37166666666667 Val Acc: 96.63000000000001\n",
      "Epoch 5\n",
      "Loss:0.07156913296227567 Train Acc: 97.72166666666666 Val Acc: 96.85000000000001\n",
      "Epoch 6\n",
      "Loss:0.06084103259316544 Train Acc: 98.01833333333333 Val Acc: 96.94\n",
      "Epoch 7\n",
      "Loss:0.05217486681376716 Train Acc: 98.26666666666667 Val Acc: 97.11999999999999\n",
      "Epoch 8\n",
      "Loss:0.045084260424835064 Train Acc: 98.53 Val Acc: 97.34\n",
      "Epoch 9\n",
      "Loss:0.03889312559196875 Train Acc: 98.73666666666666 Val Acc: 97.47\n",
      "Epoch 10\n",
      "Loss:0.03368330561181899 Train Acc: 98.87 Val Acc: 97.34\n",
      "Epoch 11\n",
      "Loss:0.029296835701112245 Train Acc: 99.02833333333334 Val Acc: 97.41\n",
      "Epoch 12\n",
      "Loss:0.025392575369040177 Train Acc: 99.07166666666667 Val Acc: 97.43\n",
      "Epoch 13\n",
      "Loss:0.02201199908496061 Train Acc: 99.15333333333334 Val Acc: 97.34\n",
      "Epoch 14\n",
      "Loss:0.01910859037765261 Train Acc: 99.25500000000001 Val Acc: 97.36\n",
      "Epoch 15\n",
      "Loss:0.016514705153122637 Train Acc: 99.34333333333333 Val Acc: 97.46000000000001\n",
      "Epoch 16\n",
      "Loss:0.014337399537915105 Train Acc: 99.40333333333334 Val Acc: 97.47\n",
      "Epoch 17\n",
      "Loss:0.012368945887024328 Train Acc: 99.50833333333333 Val Acc: 97.58\n",
      "Epoch 18\n",
      "Loss:0.010774414527658411 Train Acc: 99.615 Val Acc: 97.61999999999999\n",
      "Epoch 19\n",
      "Loss:0.009308040196023536 Train Acc: 99.675 Val Acc: 97.64\n",
      "Epoch 20\n",
      "Loss:0.008034600475817836 Train Acc: 99.71 Val Acc: 97.68\n",
      "Epoch 1\n",
      "Loss:0.33094005687262273 Train Acc: 94.14666666666666 Val Acc: 94.07\n",
      "Epoch 2\n",
      "Loss:0.1383157801769264 Train Acc: 96.02000000000001 Val Acc: 95.64\n",
      "Epoch 3\n",
      "Loss:0.10377877640376668 Train Acc: 96.64166666666667 Val Acc: 96.04\n",
      "Epoch 4\n",
      "Loss:0.08380856558500191 Train Acc: 97.35666666666667 Val Acc: 96.43\n",
      "Epoch 5\n",
      "Loss:0.0698866870412897 Train Acc: 97.81 Val Acc: 96.77\n",
      "Epoch 6\n",
      "Loss:0.05912955673185904 Train Acc: 98.21666666666667 Val Acc: 96.97\n",
      "Epoch 7\n",
      "Loss:0.050764391754564916 Train Acc: 98.48666666666666 Val Acc: 97.14\n",
      "Epoch 8\n",
      "Loss:0.04370266655130094 Train Acc: 98.66833333333334 Val Acc: 97.26\n",
      "Epoch 9\n",
      "Loss:0.037806470051593155 Train Acc: 98.79833333333333 Val Acc: 97.26\n",
      "Epoch 10\n",
      "Loss:0.032850125201994526 Train Acc: 98.92 Val Acc: 97.36\n",
      "Epoch 11\n",
      "Loss:0.028368370712161837 Train Acc: 99.0 Val Acc: 97.37\n",
      "Epoch 12\n",
      "Loss:0.02461771552220165 Train Acc: 99.175 Val Acc: 97.43\n",
      "Epoch 13\n",
      "Loss:0.021344548002018123 Train Acc: 99.22833333333332 Val Acc: 97.39999999999999\n",
      "Epoch 14\n",
      "Loss:0.018464835423373127 Train Acc: 99.35666666666667 Val Acc: 97.35000000000001\n",
      "Epoch 15\n",
      "Loss:0.016000172706288226 Train Acc: 99.42999999999999 Val Acc: 97.48\n",
      "Epoch 16\n",
      "Loss:0.013786769980986378 Train Acc: 99.515 Val Acc: 97.55\n",
      "Epoch 17\n",
      "Loss:0.01203997681917739 Train Acc: 99.60333333333334 Val Acc: 97.64\n",
      "Epoch 18\n",
      "Loss:0.010355039319235582 Train Acc: 99.64666666666666 Val Acc: 97.66\n",
      "Epoch 19\n",
      "Loss:0.008944099347227667 Train Acc: 99.685 Val Acc: 97.7\n",
      "Epoch 20\n",
      "Loss:0.007729186848953028 Train Acc: 99.74 Val Acc: 97.68\n"
     ]
    }
   ],
   "source": [
    "opts_history = {i.name: run_model(i) for i in opts}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1\n",
      "Loss:2.1503119682928302 Train Acc: 43.27166666666667 Val Acc: 43.63\n",
      "Epoch 2\n",
      "Loss:1.789025401127598 Train Acc: 60.99166666666667 Val Acc: 61.86000000000001\n",
      "Epoch 3\n",
      "Loss:1.4460175021900223 Train Acc: 69.90666666666667 Val Acc: 70.96000000000001\n",
      "Epoch 4\n",
      "Loss:1.1681158313491378 Train Acc: 75.38833333333334 Val Acc: 76.83\n",
      "Epoch 5\n",
      "Loss:0.9724806898954285 Train Acc: 78.97333333333333 Val Acc: 80.05\n",
      "Epoch 6\n",
      "Loss:0.8376267271694037 Train Acc: 81.15833333333333 Val Acc: 82.41000000000001\n",
      "Epoch 7\n",
      "Loss:0.7421259956904581 Train Acc: 82.87166666666667 Val Acc: 84.07\n",
      "Epoch 8\n",
      "Loss:0.6719898616919099 Train Acc: 84.08 Val Acc: 85.34\n",
      "Epoch 9\n",
      "Loss:0.6186536204666072 Train Acc: 85.055 Val Acc: 86.32\n",
      "Epoch 10\n",
      "Loss:0.5768878117558198 Train Acc: 85.79166666666667 Val Acc: 86.89\n",
      "Epoch 11\n",
      "Loss:0.5434002606364619 Train Acc: 86.40833333333333 Val Acc: 87.41\n",
      "Epoch 12\n",
      "Loss:0.5160244313197698 Train Acc: 86.87 Val Acc: 87.92999999999999\n",
      "Epoch 13\n",
      "Loss:0.4932644358199433 Train Acc: 87.29666666666667 Val Acc: 88.37\n",
      "Epoch 14\n",
      "Loss:0.4740456128150996 Train Acc: 87.63 Val Acc: 88.69\n",
      "Epoch 15\n",
      "Loss:0.4576100884986992 Train Acc: 87.91 Val Acc: 89.03\n",
      "Epoch 16\n",
      "Loss:0.4434002522618662 Train Acc: 88.205 Val Acc: 89.21\n",
      "Epoch 17\n",
      "Loss:0.4309912004325757 Train Acc: 88.42 Val Acc: 89.38000000000001\n",
      "Epoch 18\n",
      "Loss:0.4200562273562605 Train Acc: 88.62833333333333 Val Acc: 89.53999999999999\n",
      "Epoch 19\n",
      "Loss:0.4103384811479114 Train Acc: 88.84 Val Acc: 89.73\n",
      "Epoch 20\n",
      "Loss:0.40162777020010226 Train Acc: 89.04166666666666 Val Acc: 89.92\n"
     ]
    }
   ],
   "source": [
    "opts_history['Gradient_Descent'] = (run_model(None))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<optimizers.SGDM at 0x14a686160>,\n",
       " <optimizers.SGDM at 0x14a64d5b0>,\n",
       " <optimizers.QHM at 0x14a686220>,\n",
       " <optimizers.Nesterov at 0x14a6869a0>,\n",
       " <optimizers.Nesterov at 0x14a686790>,\n",
       " <optimizers.Adagrad at 0x14a686bb0>,\n",
       " <optimizers.RMSprop at 0x14a686760>,\n",
       " <optimizers.Adadelta at 0x14a686d30>,\n",
       " <optimizers.Adam at 0x14a7219a0>,\n",
       " <optimizers.Adam at 0x10a04a250>,\n",
       " <optimizers.Adam at 0x14a7219d0>,\n",
       " <optimizers.Adam at 0x10a04a2e0>,\n",
       " <optimizers.Nadam at 0x14a721d00>,\n",
       " <optimizers.Nadam at 0x14a721790>,\n",
       " <optimizers.Nadam at 0x14a721670>,\n",
       " <optimizers.Nadam at 0x14a721520>,\n",
       " <optimizers.Adamax at 0x14a721910>,\n",
       " <optimizers.Adamax at 0x14a721df0>,\n",
       " <optimizers.Adamax at 0x14a721880>,\n",
       " <optimizers.Adamax at 0x14a7215b0>,\n",
       " <optimizers.QHAdam at 0x14a7217f0>,\n",
       " <optimizers.QHAdam at 0x14a721820>,\n",
       " <optimizers.QHAdam at 0x14a721b20>,\n",
       " <optimizers.QHAdam at 0x14a721700>]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "name": "Gradient_Descent",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          2.1503119682928302,
          1.789025401127598,
          1.4460175021900223,
          1.1681158313491378,
          0.9724806898954285,
          0.8376267271694037,
          0.7421259956904581,
          0.6719898616919099,
          0.6186536204666072,
          0.5768878117558198,
          0.5434002606364619,
          0.5160244313197698,
          0.4932644358199433,
          0.4740456128150996,
          0.4576100884986992,
          0.4434002522618662,
          0.4309912004325757,
          0.4200562273562605,
          0.4103384811479114,
          0.40162777020010226
         ]
        },
        {
         "name": "SGDM",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          1.971548086743485,
          1.250420248838212,
          0.7758959326613197,
          0.5239670911303634,
          0.3382509386365137,
          0.2160378233607533,
          0.16188467669077286,
          0.13054313307258283,
          0.1108617285036437,
          0.09447141691313259,
          0.07913496700905819,
          0.06897837086191724,
          0.06206052102064375,
          0.0563329734093599,
          0.051070260278167856,
          0.04586898762231048,
          0.04112409724026969,
          0.03782737159145678,
          0.03507443108055851,
          0.03321487363372428
         ]
        },
        {
         "name": "Nesterov",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          1.9704602353734222,
          1.2492045883557268,
          0.7752782565332458,
          0.5235083758006135,
          0.3309839506914328,
          0.213242561200188,
          0.15996038924505562,
          0.12805185070117284,
          0.1072331054825482,
          0.09302741390169603,
          0.08170495425120991,
          0.07074861171046833,
          0.06088358675288745,
          0.05360022706590098,
          0.047663183301725226,
          0.04280971332381001,
          0.03891703171194897,
          0.035632458849953565,
          0.03289496956360477,
          0.03069104717512433
         ]
        },
        {
         "name": "RMSprop",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          0.3494961630785522,
          0.16426109799705413,
          0.1268348128420613,
          0.10772133084823728,
          0.09654084214486326,
          0.09060500542980887,
          0.08587229819013457,
          0.08136149266792489,
          0.07906671621906984,
          0.07584279442077525,
          0.07550684014569611,
          0.07328849074872983,
          0.07222864501038206,
          0.06967227154613703,
          0.06742340212418112,
          0.0636177696059172,
          0.06178256694947343,
          0.05962733737163606,
          0.05649972195690309,
          0.05534258714054008
         ]
        },
        {
         "name": "Adam",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          0.34624059333092777,
          0.14825623941078148,
          0.11145417180169578,
          0.08935200043307788,
          0.07398113188749977,
          0.06255578133234581,
          0.05367491966493914,
          0.04643268179420983,
          0.0401502688351156,
          0.03492799950511688,
          0.030460472903013912,
          0.026486564063315152,
          0.023012764999635223,
          0.019850895986943798,
          0.017092940164303613,
          0.014669258212178982,
          0.01263727042925123,
          0.010859550462818085,
          0.009267488182499615,
          0.00798217519584946
         ]
        },
        {
         "name": "Adagrad",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          0.9514762519950675,
          0.4852796125032019,
          0.40435011652071595,
          0.3658523918826436,
          0.3416540099021146,
          0.32422532419975825,
          0.3106259144425639,
          0.29941988316962515,
          0.28994856193572865,
          0.2817776924594709,
          0.27458352316899004,
          0.26814711451079226,
          0.2623137886995826,
          0.2569929493961753,
          0.25209256177888256,
          0.24753349497390942,
          0.2432614575322403,
          0.23923434705047805,
          0.23542810919125237,
          0.23184784085697988
         ]
        },
        {
         "name": "QHAdamW",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          0.33095683988718727,
          0.13856957846965118,
          0.10461709811607098,
          0.08449143616535054,
          0.07047598288044583,
          0.05967693540684103,
          0.051128995878849635,
          0.04412926526559461,
          0.038041487821788754,
          0.03300715642795326,
          0.02866261822065572,
          0.024774177824879656,
          0.021464150281812106,
          0.018443837829816855,
          0.01598987057260612,
          0.013841694731086965,
          0.011926170272867487,
          0.010288358322347235,
          0.008914418972177516,
          0.007675716733578058
         ]
        },
        {
         "name": "NAdam",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          0.321979028553172,
          0.14077298022238383,
          0.10615261453854963,
          0.08590244875327319,
          0.07158444184251983,
          0.060547327437613936,
          0.051970217660577106,
          0.04502583817675863,
          0.039142914501064545,
          0.034078582520819474,
          0.02959670209452659,
          0.025791835713915235,
          0.022342333082251115,
          0.01939690804295289,
          0.01670604551772691,
          0.014377306149196025,
          0.01238276999178871,
          0.01061498786924163,
          0.00909216970155948,
          0.007759226196403027
         ]
        }
       ],
       "layout": {
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Loss"
        },
        "xaxis": {
         "title": {
          "text": "Epochs"
         }
        },
        "yaxis": {
         "title": {
          "text": "Loss"
         }
        }
       }
      },
      "text/html": [
       "<div>                            <div id=\"dfca80bb-be22-469d-be92-8811f3d63ad2\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"dfca80bb-be22-469d-be92-8811f3d63ad2\")) {                    Plotly.newPlot(                        \"dfca80bb-be22-469d-be92-8811f3d63ad2\",                        [{\"name\": \"Gradient_Descent\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [2.1503119682928302, 1.789025401127598, 1.4460175021900223, 1.1681158313491378, 0.9724806898954285, 0.8376267271694037, 0.7421259956904581, 0.6719898616919099, 0.6186536204666072, 0.5768878117558198, 0.5434002606364619, 0.5160244313197698, 0.4932644358199433, 0.4740456128150996, 0.4576100884986992, 0.4434002522618662, 0.4309912004325757, 0.4200562273562605, 0.4103384811479114, 0.40162777020010226]}, {\"name\": \"SGDM\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [1.971548086743485, 1.250420248838212, 0.7758959326613197, 0.5239670911303634, 0.3382509386365137, 0.2160378233607533, 0.16188467669077286, 0.13054313307258283, 0.1108617285036437, 0.09447141691313259, 0.07913496700905819, 0.06897837086191724, 0.06206052102064375, 0.0563329734093599, 0.051070260278167856, 0.04586898762231048, 0.04112409724026969, 0.03782737159145678, 0.03507443108055851, 0.03321487363372428]}, {\"name\": \"Nesterov\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [1.9704602353734222, 1.2492045883557268, 0.7752782565332458, 0.5235083758006135, 0.3309839506914328, 0.213242561200188, 0.15996038924505562, 0.12805185070117284, 0.1072331054825482, 0.09302741390169603, 0.08170495425120991, 0.07074861171046833, 0.06088358675288745, 0.05360022706590098, 0.047663183301725226, 0.04280971332381001, 0.03891703171194897, 0.035632458849953565, 0.03289496956360477, 0.03069104717512433]}, {\"name\": \"RMSprop\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [0.3494961630785522, 0.16426109799705413, 0.1268348128420613, 0.10772133084823728, 0.09654084214486326, 0.09060500542980887, 0.08587229819013457, 0.08136149266792489, 0.07906671621906984, 0.07584279442077525, 0.07550684014569611, 0.07328849074872983, 0.07222864501038206, 0.06967227154613703, 0.06742340212418112, 0.0636177696059172, 0.06178256694947343, 0.05962733737163606, 0.05649972195690309, 0.05534258714054008]}, {\"name\": \"Adam\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [0.34624059333092777, 0.14825623941078148, 0.11145417180169578, 0.08935200043307788, 0.07398113188749977, 0.06255578133234581, 0.05367491966493914, 0.04643268179420983, 0.0401502688351156, 0.03492799950511688, 0.030460472903013912, 0.026486564063315152, 0.023012764999635223, 0.019850895986943798, 0.017092940164303613, 0.014669258212178982, 0.01263727042925123, 0.010859550462818085, 0.009267488182499615, 0.00798217519584946]}, {\"name\": \"Adagrad\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [0.9514762519950675, 0.4852796125032019, 0.40435011652071595, 0.3658523918826436, 0.3416540099021146, 0.32422532419975825, 0.3106259144425639, 0.29941988316962515, 0.28994856193572865, 0.2817776924594709, 0.27458352316899004, 0.26814711451079226, 0.2623137886995826, 0.2569929493961753, 0.25209256177888256, 0.24753349497390942, 0.2432614575322403, 0.23923434705047805, 0.23542810919125237, 0.23184784085697988]}, {\"name\": \"QHAdamW\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [0.33095683988718727, 0.13856957846965118, 0.10461709811607098, 0.08449143616535054, 0.07047598288044583, 0.05967693540684103, 0.051128995878849635, 0.04412926526559461, 0.038041487821788754, 0.03300715642795326, 0.02866261822065572, 0.024774177824879656, 0.021464150281812106, 0.018443837829816855, 0.01598987057260612, 0.013841694731086965, 0.011926170272867487, 0.010288358322347235, 0.008914418972177516, 0.007675716733578058]}, {\"name\": \"NAdam\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [0.321979028553172, 0.14077298022238383, 0.10615261453854963, 0.08590244875327319, 0.07158444184251983, 0.060547327437613936, 0.051970217660577106, 0.04502583817675863, 0.039142914501064545, 0.034078582520819474, 0.02959670209452659, 0.025791835713915235, 0.022342333082251115, 0.01939690804295289, 0.01670604551772691, 0.014377306149196025, 0.01238276999178871, 0.01061498786924163, 0.00909216970155948, 0.007759226196403027]}],                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Loss\"}, \"xaxis\": {\"title\": {\"text\": \"Epochs\"}}, \"yaxis\": {\"title\": {\"text\": \"Loss\"}}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('dfca80bb-be22-469d-be92-8811f3d63ad2');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from plotly.subplots import make_subplots\n",
    "\n",
    "num_epochs = [i for i in range(epochs)]\n",
    "fig = go.Figure(layout_title_text=\"Loss\")\n",
    "\n",
    "lst = ['Gradient_Descent', 'SGDM','Nesterov', 'RMSprop', 'Adam', 'Adagrad', 'QHAdamW', 'NAdam']\n",
    "\n",
    "\n",
    "for i in lst:\n",
    "    fig.add_trace(go.Scatter(x=num_epochs, y=opts_history[i]['train_loss'], name=i))\n",
    "    \n",
    "fig.update_xaxes(title_text='Epochs')\n",
    "fig.update_yaxes(title_text='Loss')\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.plotly.v1+json": {
       "config": {
        "plotlyServerURL": "https://plot.ly"
       },
       "data": [
        {
         "name": "Gradient_Descent",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          43.63,
          61.86000000000001,
          70.96000000000001,
          76.83,
          80.05,
          82.41000000000001,
          84.07,
          85.34,
          86.32,
          86.89,
          87.41,
          87.92999999999999,
          88.37,
          88.69,
          89.03,
          89.21,
          89.38000000000001,
          89.53999999999999,
          89.73,
          89.92
         ]
        },
        {
         "name": "SGDM",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          61.78,
          78.38000000000001,
          85.24000000000001,
          88.99000000000001,
          93.05,
          94.69999999999999,
          96,
          96.34,
          96.71,
          96.84,
          96.81,
          96.74000000000001,
          96.85000000000001,
          96.97,
          97.33000000000001,
          97.56,
          97.50999999999999,
          97.58,
          97.6,
          97.65
         ]
        },
        {
         "name": "Nesterov",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          61.809999999999995,
          78.41,
          85.2,
          88.92,
          93.25,
          94.82000000000001,
          95.88,
          96.41999999999999,
          96.69,
          96.84,
          96.99,
          97.11,
          97.06,
          97.05,
          97.25,
          97.39,
          97.50999999999999,
          97.54,
          97.7,
          97.72999999999999
         ]
        },
        {
         "name": "RMSprop",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          93.25,
          95.52000000000001,
          96.32,
          96.66,
          96.88,
          97.08,
          97.11999999999999,
          96.99,
          97.03,
          96.89,
          97.15,
          97.24000000000001,
          97.14,
          97.11999999999999,
          97.16,
          97.27,
          97.25,
          97.17,
          97.22,
          97.33000000000001
         ]
        },
        {
         "name": "Adam",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          94.15,
          95.71,
          96.46000000000001,
          96.72,
          96.91,
          97.06,
          97.18,
          97.24000000000001,
          97.26,
          97.27,
          97.35000000000001,
          97.32,
          97.46000000000001,
          97.52,
          97.55,
          97.57000000000001,
          97.63,
          97.59,
          97.67,
          97.65
         ]
        },
        {
         "name": "Adagrad",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          87.3,
          89.64,
          90.60000000000001,
          90.98,
          91.47,
          91.79,
          91.97,
          92.22,
          92.46,
          92.58999999999999,
          92.78,
          92.96,
          93.07,
          93.17999999999999,
          93.25,
          93.37,
          93.52000000000001,
          93.60000000000001,
          93.66,
          93.78999999999999
         ]
        },
        {
         "name": "QHAdamW",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          94.03,
          95.76,
          96.2,
          96.57,
          96.89,
          97.02,
          97.28999999999999,
          97.33000000000001,
          97.39999999999999,
          97.39999999999999,
          97.46000000000001,
          97.47,
          97.55,
          97.50999999999999,
          97.50999999999999,
          97.56,
          97.61999999999999,
          97.64,
          97.68,
          97.69
         ]
        },
        {
         "name": "NAdam",
         "type": "scatter",
         "x": [
          0,
          1,
          2,
          3,
          4,
          5,
          6,
          7,
          8,
          9,
          10,
          11,
          12,
          13,
          14,
          15,
          16,
          17,
          18,
          19
         ],
         "y": [
          94.82000000000001,
          95.78999999999999,
          96.39999999999999,
          96.76,
          97.05,
          97.09,
          97.19,
          97.3,
          97.28,
          97.27,
          97.36,
          97.39,
          97.48,
          97.5,
          97.59,
          97.61999999999999,
          97.71,
          97.72,
          97.67,
          97.68
         ]
        }
       ],
       "layout": {
        "template": {
         "data": {
          "bar": [
           {
            "error_x": {
             "color": "#2a3f5f"
            },
            "error_y": {
             "color": "#2a3f5f"
            },
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "bar"
           }
          ],
          "barpolar": [
           {
            "marker": {
             "line": {
              "color": "#E5ECF6",
              "width": 0.5
             }
            },
            "type": "barpolar"
           }
          ],
          "carpet": [
           {
            "aaxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "baxis": {
             "endlinecolor": "#2a3f5f",
             "gridcolor": "white",
             "linecolor": "white",
             "minorgridcolor": "white",
             "startlinecolor": "#2a3f5f"
            },
            "type": "carpet"
           }
          ],
          "choropleth": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "choropleth"
           }
          ],
          "contour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "contour"
           }
          ],
          "contourcarpet": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "contourcarpet"
           }
          ],
          "heatmap": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmap"
           }
          ],
          "heatmapgl": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "heatmapgl"
           }
          ],
          "histogram": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "histogram"
           }
          ],
          "histogram2d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2d"
           }
          ],
          "histogram2dcontour": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "histogram2dcontour"
           }
          ],
          "mesh3d": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "type": "mesh3d"
           }
          ],
          "parcoords": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "parcoords"
           }
          ],
          "pie": [
           {
            "automargin": true,
            "type": "pie"
           }
          ],
          "scatter": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter"
           }
          ],
          "scatter3d": [
           {
            "line": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatter3d"
           }
          ],
          "scattercarpet": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattercarpet"
           }
          ],
          "scattergeo": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergeo"
           }
          ],
          "scattergl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattergl"
           }
          ],
          "scattermapbox": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scattermapbox"
           }
          ],
          "scatterpolar": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolar"
           }
          ],
          "scatterpolargl": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterpolargl"
           }
          ],
          "scatterternary": [
           {
            "marker": {
             "colorbar": {
              "outlinewidth": 0,
              "ticks": ""
             }
            },
            "type": "scatterternary"
           }
          ],
          "surface": [
           {
            "colorbar": {
             "outlinewidth": 0,
             "ticks": ""
            },
            "colorscale": [
             [
              0,
              "#0d0887"
             ],
             [
              0.1111111111111111,
              "#46039f"
             ],
             [
              0.2222222222222222,
              "#7201a8"
             ],
             [
              0.3333333333333333,
              "#9c179e"
             ],
             [
              0.4444444444444444,
              "#bd3786"
             ],
             [
              0.5555555555555556,
              "#d8576b"
             ],
             [
              0.6666666666666666,
              "#ed7953"
             ],
             [
              0.7777777777777778,
              "#fb9f3a"
             ],
             [
              0.8888888888888888,
              "#fdca26"
             ],
             [
              1,
              "#f0f921"
             ]
            ],
            "type": "surface"
           }
          ],
          "table": [
           {
            "cells": {
             "fill": {
              "color": "#EBF0F8"
             },
             "line": {
              "color": "white"
             }
            },
            "header": {
             "fill": {
              "color": "#C8D4E3"
             },
             "line": {
              "color": "white"
             }
            },
            "type": "table"
           }
          ]
         },
         "layout": {
          "annotationdefaults": {
           "arrowcolor": "#2a3f5f",
           "arrowhead": 0,
           "arrowwidth": 1
          },
          "coloraxis": {
           "colorbar": {
            "outlinewidth": 0,
            "ticks": ""
           }
          },
          "colorscale": {
           "diverging": [
            [
             0,
             "#8e0152"
            ],
            [
             0.1,
             "#c51b7d"
            ],
            [
             0.2,
             "#de77ae"
            ],
            [
             0.3,
             "#f1b6da"
            ],
            [
             0.4,
             "#fde0ef"
            ],
            [
             0.5,
             "#f7f7f7"
            ],
            [
             0.6,
             "#e6f5d0"
            ],
            [
             0.7,
             "#b8e186"
            ],
            [
             0.8,
             "#7fbc41"
            ],
            [
             0.9,
             "#4d9221"
            ],
            [
             1,
             "#276419"
            ]
           ],
           "sequential": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ],
           "sequentialminus": [
            [
             0,
             "#0d0887"
            ],
            [
             0.1111111111111111,
             "#46039f"
            ],
            [
             0.2222222222222222,
             "#7201a8"
            ],
            [
             0.3333333333333333,
             "#9c179e"
            ],
            [
             0.4444444444444444,
             "#bd3786"
            ],
            [
             0.5555555555555556,
             "#d8576b"
            ],
            [
             0.6666666666666666,
             "#ed7953"
            ],
            [
             0.7777777777777778,
             "#fb9f3a"
            ],
            [
             0.8888888888888888,
             "#fdca26"
            ],
            [
             1,
             "#f0f921"
            ]
           ]
          },
          "colorway": [
           "#636efa",
           "#EF553B",
           "#00cc96",
           "#ab63fa",
           "#FFA15A",
           "#19d3f3",
           "#FF6692",
           "#B6E880",
           "#FF97FF",
           "#FECB52"
          ],
          "font": {
           "color": "#2a3f5f"
          },
          "geo": {
           "bgcolor": "white",
           "lakecolor": "white",
           "landcolor": "#E5ECF6",
           "showlakes": true,
           "showland": true,
           "subunitcolor": "white"
          },
          "hoverlabel": {
           "align": "left"
          },
          "hovermode": "closest",
          "mapbox": {
           "style": "light"
          },
          "paper_bgcolor": "white",
          "plot_bgcolor": "#E5ECF6",
          "polar": {
           "angularaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "radialaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "scene": {
           "xaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "yaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           },
           "zaxis": {
            "backgroundcolor": "#E5ECF6",
            "gridcolor": "white",
            "gridwidth": 2,
            "linecolor": "white",
            "showbackground": true,
            "ticks": "",
            "zerolinecolor": "white"
           }
          },
          "shapedefaults": {
           "line": {
            "color": "#2a3f5f"
           }
          },
          "ternary": {
           "aaxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "baxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           },
           "bgcolor": "#E5ECF6",
           "caxis": {
            "gridcolor": "white",
            "linecolor": "white",
            "ticks": ""
           }
          },
          "title": {
           "x": 0.05
          },
          "xaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          },
          "yaxis": {
           "automargin": true,
           "gridcolor": "white",
           "linecolor": "white",
           "ticks": "",
           "title": {
            "standoff": 15
           },
           "zerolinecolor": "white",
           "zerolinewidth": 2
          }
         }
        },
        "title": {
         "text": "Validation Accuracy"
        },
        "xaxis": {
         "title": {
          "text": "Epochs"
         }
        },
        "yaxis": {
         "title": {
          "text": "Accuracy"
         }
        }
       }
      },
      "text/html": [
       "<div>                            <div id=\"881dbe0f-a284-4b1b-84af-0faee7c120ca\" class=\"plotly-graph-div\" style=\"height:525px; width:100%;\"></div>            <script type=\"text/javascript\">                require([\"plotly\"], function(Plotly) {                    window.PLOTLYENV=window.PLOTLYENV || {};                                    if (document.getElementById(\"881dbe0f-a284-4b1b-84af-0faee7c120ca\")) {                    Plotly.newPlot(                        \"881dbe0f-a284-4b1b-84af-0faee7c120ca\",                        [{\"name\": \"Gradient_Descent\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [43.63, 61.86000000000001, 70.96000000000001, 76.83, 80.05, 82.41000000000001, 84.07, 85.34, 86.32, 86.89, 87.41, 87.92999999999999, 88.37, 88.69, 89.03, 89.21, 89.38000000000001, 89.53999999999999, 89.73, 89.92]}, {\"name\": \"SGDM\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [61.78, 78.38000000000001, 85.24000000000001, 88.99000000000001, 93.05, 94.69999999999999, 96.0, 96.34, 96.71, 96.84, 96.81, 96.74000000000001, 96.85000000000001, 96.97, 97.33000000000001, 97.56, 97.50999999999999, 97.58, 97.6, 97.65]}, {\"name\": \"Nesterov\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [61.809999999999995, 78.41, 85.2, 88.92, 93.25, 94.82000000000001, 95.88, 96.41999999999999, 96.69, 96.84, 96.99, 97.11, 97.06, 97.05, 97.25, 97.39, 97.50999999999999, 97.54, 97.7, 97.72999999999999]}, {\"name\": \"RMSprop\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [93.25, 95.52000000000001, 96.32, 96.66, 96.88, 97.08, 97.11999999999999, 96.99, 97.03, 96.89, 97.15, 97.24000000000001, 97.14, 97.11999999999999, 97.16, 97.27, 97.25, 97.17, 97.22, 97.33000000000001]}, {\"name\": \"Adam\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [94.15, 95.71, 96.46000000000001, 96.72, 96.91, 97.06, 97.18, 97.24000000000001, 97.26, 97.27, 97.35000000000001, 97.32, 97.46000000000001, 97.52, 97.55, 97.57000000000001, 97.63, 97.59, 97.67, 97.65]}, {\"name\": \"Adagrad\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [87.3, 89.64, 90.60000000000001, 90.98, 91.47, 91.79, 91.97, 92.22, 92.46, 92.58999999999999, 92.78, 92.96, 93.07, 93.17999999999999, 93.25, 93.37, 93.52000000000001, 93.60000000000001, 93.66, 93.78999999999999]}, {\"name\": \"QHAdamW\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [94.03, 95.76, 96.2, 96.57, 96.89, 97.02, 97.28999999999999, 97.33000000000001, 97.39999999999999, 97.39999999999999, 97.46000000000001, 97.47, 97.55, 97.50999999999999, 97.50999999999999, 97.56, 97.61999999999999, 97.64, 97.68, 97.69]}, {\"name\": \"NAdam\", \"type\": \"scatter\", \"x\": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], \"y\": [94.82000000000001, 95.78999999999999, 96.39999999999999, 96.76, 97.05, 97.09, 97.19, 97.3, 97.28, 97.27, 97.36, 97.39, 97.48, 97.5, 97.59, 97.61999999999999, 97.71, 97.72, 97.67, 97.68]}],                        {\"template\": {\"data\": {\"bar\": [{\"error_x\": {\"color\": \"#2a3f5f\"}, \"error_y\": {\"color\": \"#2a3f5f\"}, \"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"bar\"}], \"barpolar\": [{\"marker\": {\"line\": {\"color\": \"#E5ECF6\", \"width\": 0.5}}, \"type\": \"barpolar\"}], \"carpet\": [{\"aaxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"baxis\": {\"endlinecolor\": \"#2a3f5f\", \"gridcolor\": \"white\", \"linecolor\": \"white\", \"minorgridcolor\": \"white\", \"startlinecolor\": \"#2a3f5f\"}, \"type\": \"carpet\"}], \"choropleth\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"choropleth\"}], \"contour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"contour\"}], \"contourcarpet\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"contourcarpet\"}], \"heatmap\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmap\"}], \"heatmapgl\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"heatmapgl\"}], \"histogram\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"histogram\"}], \"histogram2d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2d\"}], \"histogram2dcontour\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"histogram2dcontour\"}], \"mesh3d\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"type\": \"mesh3d\"}], \"parcoords\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"parcoords\"}], \"pie\": [{\"automargin\": true, \"type\": \"pie\"}], \"scatter\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter\"}], \"scatter3d\": [{\"line\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatter3d\"}], \"scattercarpet\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattercarpet\"}], \"scattergeo\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergeo\"}], \"scattergl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattergl\"}], \"scattermapbox\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scattermapbox\"}], \"scatterpolar\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolar\"}], \"scatterpolargl\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterpolargl\"}], \"scatterternary\": [{\"marker\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"type\": \"scatterternary\"}], \"surface\": [{\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}, \"colorscale\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"type\": \"surface\"}], \"table\": [{\"cells\": {\"fill\": {\"color\": \"#EBF0F8\"}, \"line\": {\"color\": \"white\"}}, \"header\": {\"fill\": {\"color\": \"#C8D4E3\"}, \"line\": {\"color\": \"white\"}}, \"type\": \"table\"}]}, \"layout\": {\"annotationdefaults\": {\"arrowcolor\": \"#2a3f5f\", \"arrowhead\": 0, \"arrowwidth\": 1}, \"coloraxis\": {\"colorbar\": {\"outlinewidth\": 0, \"ticks\": \"\"}}, \"colorscale\": {\"diverging\": [[0, \"#8e0152\"], [0.1, \"#c51b7d\"], [0.2, \"#de77ae\"], [0.3, \"#f1b6da\"], [0.4, \"#fde0ef\"], [0.5, \"#f7f7f7\"], [0.6, \"#e6f5d0\"], [0.7, \"#b8e186\"], [0.8, \"#7fbc41\"], [0.9, \"#4d9221\"], [1, \"#276419\"]], \"sequential\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]], \"sequentialminus\": [[0.0, \"#0d0887\"], [0.1111111111111111, \"#46039f\"], [0.2222222222222222, \"#7201a8\"], [0.3333333333333333, \"#9c179e\"], [0.4444444444444444, \"#bd3786\"], [0.5555555555555556, \"#d8576b\"], [0.6666666666666666, \"#ed7953\"], [0.7777777777777778, \"#fb9f3a\"], [0.8888888888888888, \"#fdca26\"], [1.0, \"#f0f921\"]]}, \"colorway\": [\"#636efa\", \"#EF553B\", \"#00cc96\", \"#ab63fa\", \"#FFA15A\", \"#19d3f3\", \"#FF6692\", \"#B6E880\", \"#FF97FF\", \"#FECB52\"], \"font\": {\"color\": \"#2a3f5f\"}, \"geo\": {\"bgcolor\": \"white\", \"lakecolor\": \"white\", \"landcolor\": \"#E5ECF6\", \"showlakes\": true, \"showland\": true, \"subunitcolor\": \"white\"}, \"hoverlabel\": {\"align\": \"left\"}, \"hovermode\": \"closest\", \"mapbox\": {\"style\": \"light\"}, \"paper_bgcolor\": \"white\", \"plot_bgcolor\": \"#E5ECF6\", \"polar\": {\"angularaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"radialaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"scene\": {\"xaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"yaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}, \"zaxis\": {\"backgroundcolor\": \"#E5ECF6\", \"gridcolor\": \"white\", \"gridwidth\": 2, \"linecolor\": \"white\", \"showbackground\": true, \"ticks\": \"\", \"zerolinecolor\": \"white\"}}, \"shapedefaults\": {\"line\": {\"color\": \"#2a3f5f\"}}, \"ternary\": {\"aaxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"baxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}, \"bgcolor\": \"#E5ECF6\", \"caxis\": {\"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\"}}, \"title\": {\"x\": 0.05}, \"xaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}, \"yaxis\": {\"automargin\": true, \"gridcolor\": \"white\", \"linecolor\": \"white\", \"ticks\": \"\", \"title\": {\"standoff\": 15}, \"zerolinecolor\": \"white\", \"zerolinewidth\": 2}}}, \"title\": {\"text\": \"Validation Accuracy\"}, \"xaxis\": {\"title\": {\"text\": \"Epochs\"}}, \"yaxis\": {\"title\": {\"text\": \"Accuracy\"}}},                        {\"responsive\": true}                    ).then(function(){\n",
       "                            \n",
       "var gd = document.getElementById('881dbe0f-a284-4b1b-84af-0faee7c120ca');\n",
       "var x = new MutationObserver(function (mutations, observer) {{\n",
       "        var display = window.getComputedStyle(gd).display;\n",
       "        if (!display || display === 'none') {{\n",
       "            console.log([gd, 'removed!']);\n",
       "            Plotly.purge(gd);\n",
       "            observer.disconnect();\n",
       "        }}\n",
       "}});\n",
       "\n",
       "// Listen for the removal of the full notebook cells\n",
       "var notebookContainer = gd.closest('#notebook-container');\n",
       "if (notebookContainer) {{\n",
       "    x.observe(notebookContainer, {childList: true});\n",
       "}}\n",
       "\n",
       "// Listen for the clearing of the current output cell\n",
       "var outputEl = gd.closest('.output');\n",
       "if (outputEl) {{\n",
       "    x.observe(outputEl, {childList: true});\n",
       "}}\n",
       "\n",
       "                        })                };                });            </script>        </div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = go.Figure(layout_title_text=\"Validation Accuracy\")\n",
    "\n",
    "for i in lst:\n",
    "    fig.add_trace(go.Scatter(x=num_epochs, y=opts_history[i]['val_acc'], name=i))\n",
    "    \n",
    "fig.update_xaxes(title_text='Epochs')\n",
    "fig.update_yaxes(title_text='Accuracy')\n",
    "fig.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}