experiment13.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mmenpo 0.8.1 has requirement matplotlib<2.0,>=1.4, but you'll have matplotlib 3.0.2 which is incompatible.\u001b[0m\n",
      "\u001b[31mmenpo 0.8.1 has requirement pillow<5.0,>=3.0, but you'll have pillow 5.4.1 which is incompatible.\u001b[0m\n",
      "\u001b[31mmenpo 0.8.1 has requirement scipy<1.0,>=0.16, but you'll have scipy 1.2.0 which is incompatible.\u001b[0m\n",
      "\u001b[33mYou are using pip version 10.0.1, however version 19.2.3 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install -q -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from PIL import Image\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils.data as D\n",
    "from torch.optim.lr_scheduler import ExponentialLR\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "\n",
    "from torchvision import transforms\n",
    "\n",
    "from ignite.engine import Events\n",
    "from scripts.ignite import create_supervised_evaluator, create_supervised_trainer\n",
    "from ignite.metrics import Loss, Accuracy\n",
    "from ignite.contrib.handlers.tqdm_logger import ProgressBar\n",
    "from ignite.handlers import  EarlyStopping, ModelCheckpoint\n",
    "from ignite.contrib.handlers import LinearCyclicalScheduler, CosineAnnealingScheduler\n",
    "\n",
    "import random\n",
    "\n",
    "from tqdm import tqdm_notebook\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from efficientnet_pytorch import EfficientNet, utils as enet_utils\n",
    "\n",
    "from scripts.evaluate import eval_model\n",
    "from scripts.transforms import gen_transform_train, gen_transform_validation\n",
    "from scripts.plates_leak import apply_plates_leak\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define dataset and model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "img_dir = '../input/rxrxai'\n",
    "path_data = '../input/rxrxaicsv'\n",
    "device = 'cuda'\n",
    "batch_size = 4\n",
    "torch.manual_seed(0)\n",
    "model_name = 'efficientnet-b4'\n",
    "init_lr = 3e-4\n",
    "end_lr = 1e-7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ImagesDS(D.Dataset):\n",
    "    transform_validation = gen_transform_validation(crop_size=448)\n",
    "    \n",
    "    def __init__(self, df, img_dir=img_dir, mode='train', validation=False, channels=[1,2,3,4,5,6]):\n",
    "        self.records = df.to_records(index=False)\n",
    "        self.mode = mode\n",
    "        self.img_dir = img_dir\n",
    "        self.len = df.shape[0]\n",
    "        self.validation = validation\n",
    "        self.channels = channels\n",
    "\n",
    "    def _get_img_path(self, index, channel, site):\n",
    "        experiment, well, plate = self.records[index].experiment, self.records[index].well, self.records[index].plate\n",
    "        return '/'.join([self.img_dir,self.mode,experiment,f'Plate{plate}',f'{well}_s{site}_w{channel}.png'])\n",
    "        \n",
    "    @staticmethod\n",
    "    def _load_img_as_tensor(file_name, transform):\n",
    "        with Image.open(file_name) as img:\n",
    "            return transform(img)\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        transform1 = ImagesDS.transform_validation if self.validation else gen_transform_train()\n",
    "        transform2 = ImagesDS.transform_validation if self.validation else gen_transform_train()\n",
    "        \n",
    "        paths1 = [self._get_img_path(index, ch, 1) for ch in self.channels]\n",
    "        paths2 = [self._get_img_path(index, ch, 2) for ch in self.channels]\n",
    "        \n",
    "        img1 = torch.cat([self._load_img_as_tensor(img_path, transform1) for img_path in paths1])\n",
    "        img2 = torch.cat([self._load_img_as_tensor(img_path, transform2) for img_path in paths2])\n",
    "        \n",
    "        if random.random() > 0.5 and not self.validation:\n",
    "            img1, img2 = img2, img1\n",
    "        \n",
    "        if self.mode == 'train':\n",
    "            return img1, img2, int(self.records[index].sirna)\n",
    "        else:\n",
    "            return img1, img2, self.records[index].id_code\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataframes for training, cross-validation, and testing\n",
    "df = pd.read_csv(path_data+'/train.csv')\n",
    "df_train, df_val = train_test_split(df, test_size = 0.1, random_state=42)\n",
    "df_test = pd.read_csv(path_data+'/test.csv')\n",
    "\n",
    "# pytorch training dataset & loader\n",
    "ds = ImagesDS(df_train, mode='train', validation=False)\n",
    "loader = D.DataLoader(ds, batch_size=batch_size, shuffle=True, num_workers=4)\n",
    "\n",
    "# pytorch cross-validation dataset & loader\n",
    "ds_val = ImagesDS(df_val, mode='train', validation=True)\n",
    "val_loader = D.DataLoader(ds_val, batch_size=batch_size, shuffle=True, num_workers=4)\n",
    "\n",
    "# pytorch test dataset & loader\n",
    "ds_test = ImagesDS(df_test, mode='test', validation=True)\n",
    "tloader = D.DataLoader(ds_test, batch_size=1, shuffle=False, num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading: \"http://storage.googleapis.com/public-models/efficientnet/efficientnet-b4-6ed6700e.pth\" to /root/.cache/torch/checkpoints/efficientnet-b4-6ed6700e.pth\n",
      "100%|██████████| 77999237/77999237 [00:01<00:00, 45792473.40it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded pretrained weights for efficientnet-b4\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "EfficientNetTwoInputs(\n",
       "  (resnet): EfficientNet(\n",
       "    (_conv_stem): Conv2dStaticSamePadding(\n",
       "      6, 48, kernel_size=(3, 3), stride=(2, 2), bias=False\n",
       "      (static_padding): ZeroPad2d(padding=(0, 1, 0, 1), value=0.0)\n",
       "    )\n",
       "    (_bn0): BatchNorm2d(48, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "    (_blocks): ModuleList(\n",
       "      (0): MBConvBlock(\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          48, 48, kernel_size=(3, 3), stride=[1, 1], groups=48, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(48, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          48, 12, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          12, 48, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          48, 24, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(24, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): MBConvBlock(\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          24, 24, kernel_size=(3, 3), stride=(1, 1), groups=24, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(24, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          24, 6, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          6, 24, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          24, 24, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(24, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          24, 144, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(144, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          144, 144, kernel_size=(3, 3), stride=[2, 2], groups=144, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(0, 1, 0, 1), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(144, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          144, 6, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          6, 144, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          144, 32, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(32, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          32, 192, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(192, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          192, 192, kernel_size=(3, 3), stride=(1, 1), groups=192, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(192, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          192, 8, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          8, 192, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(32, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (4): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          32, 192, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(192, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          192, 192, kernel_size=(3, 3), stride=(1, 1), groups=192, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(192, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          192, 8, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          8, 192, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(32, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (5): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          32, 192, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(192, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          192, 192, kernel_size=(3, 3), stride=(1, 1), groups=192, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(192, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          192, 8, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          8, 192, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          192, 32, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(32, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (6): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          32, 192, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(192, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          192, 192, kernel_size=(5, 5), stride=[2, 2], groups=192, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(1, 2, 1, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(192, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          192, 8, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          8, 192, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          192, 56, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(56, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (7): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          56, 336, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(336, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          336, 336, kernel_size=(5, 5), stride=(1, 1), groups=336, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(336, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          336, 14, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          14, 336, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          336, 56, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(56, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (8): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          56, 336, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(336, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          336, 336, kernel_size=(5, 5), stride=(1, 1), groups=336, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(336, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          336, 14, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          14, 336, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          336, 56, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(56, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (9): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          56, 336, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(336, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          336, 336, kernel_size=(5, 5), stride=(1, 1), groups=336, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(336, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          336, 14, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          14, 336, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          336, 56, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(56, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (10): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          56, 336, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(336, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          336, 336, kernel_size=(3, 3), stride=[2, 2], groups=336, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(0, 1, 0, 1), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(336, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          336, 14, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          14, 336, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          336, 112, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(112, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (11): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          672, 672, kernel_size=(3, 3), stride=(1, 1), groups=672, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          672, 28, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          28, 672, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(112, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (12): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          672, 672, kernel_size=(3, 3), stride=(1, 1), groups=672, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          672, 28, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          28, 672, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(112, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (13): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          672, 672, kernel_size=(3, 3), stride=(1, 1), groups=672, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          672, 28, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          28, 672, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(112, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (14): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          672, 672, kernel_size=(3, 3), stride=(1, 1), groups=672, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          672, 28, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          28, 672, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(112, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (15): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          672, 672, kernel_size=(3, 3), stride=(1, 1), groups=672, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          672, 28, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          28, 672, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          672, 112, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(112, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (16): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          112, 672, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          672, 672, kernel_size=(5, 5), stride=[1, 1], groups=672, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(672, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          672, 28, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          28, 672, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          672, 160, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(160, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (17): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          160, 960, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(960, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          960, 960, kernel_size=(5, 5), stride=(1, 1), groups=960, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(960, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          960, 40, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          40, 960, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          960, 160, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(160, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (18): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          160, 960, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(960, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          960, 960, kernel_size=(5, 5), stride=(1, 1), groups=960, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(960, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          960, 40, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          40, 960, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          960, 160, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(160, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (19): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          160, 960, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(960, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          960, 960, kernel_size=(5, 5), stride=(1, 1), groups=960, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(960, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          960, 40, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          40, 960, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          960, 160, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(160, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (20): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          160, 960, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(960, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          960, 960, kernel_size=(5, 5), stride=(1, 1), groups=960, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(960, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          960, 40, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          40, 960, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          960, 160, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(160, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (21): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          160, 960, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(960, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          960, 960, kernel_size=(5, 5), stride=(1, 1), groups=960, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(960, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          960, 40, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          40, 960, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          960, 160, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(160, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (22): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          160, 960, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(960, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          960, 960, kernel_size=(5, 5), stride=[2, 2], groups=960, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(1, 2, 1, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(960, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          960, 40, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          40, 960, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          960, 272, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(272, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (23): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          1632, 1632, kernel_size=(5, 5), stride=(1, 1), groups=1632, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          1632, 68, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          68, 1632, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          1632, 272, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(272, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (24): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          1632, 1632, kernel_size=(5, 5), stride=(1, 1), groups=1632, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          1632, 68, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          68, 1632, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          1632, 272, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(272, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (25): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          1632, 1632, kernel_size=(5, 5), stride=(1, 1), groups=1632, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          1632, 68, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          68, 1632, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          1632, 272, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(272, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (26): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          1632, 1632, kernel_size=(5, 5), stride=(1, 1), groups=1632, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          1632, 68, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          68, 1632, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          1632, 272, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(272, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (27): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          1632, 1632, kernel_size=(5, 5), stride=(1, 1), groups=1632, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          1632, 68, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          68, 1632, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          1632, 272, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(272, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (28): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          1632, 1632, kernel_size=(5, 5), stride=(1, 1), groups=1632, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          1632, 68, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          68, 1632, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          1632, 272, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(272, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (29): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          1632, 1632, kernel_size=(5, 5), stride=(1, 1), groups=1632, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(2, 2, 2, 2), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          1632, 68, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          68, 1632, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          1632, 272, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(272, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (30): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          272, 1632, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          1632, 1632, kernel_size=(3, 3), stride=[1, 1], groups=1632, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(1632, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          1632, 68, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          68, 1632, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          1632, 448, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(448, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (31): MBConvBlock(\n",
       "        (_expand_conv): Conv2dStaticSamePadding(\n",
       "          448, 2688, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn0): BatchNorm2d(2688, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_depthwise_conv): Conv2dStaticSamePadding(\n",
       "          2688, 2688, kernel_size=(3, 3), stride=(1, 1), groups=2688, bias=False\n",
       "          (static_padding): ZeroPad2d(padding=(1, 1, 1, 1), value=0.0)\n",
       "        )\n",
       "        (_bn1): BatchNorm2d(2688, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "        (_se_reduce): Conv2dStaticSamePadding(\n",
       "          2688, 112, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_se_expand): Conv2dStaticSamePadding(\n",
       "          112, 2688, kernel_size=(1, 1), stride=(1, 1)\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_project_conv): Conv2dStaticSamePadding(\n",
       "          2688, 448, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "          (static_padding): Identity()\n",
       "        )\n",
       "        (_bn2): BatchNorm2d(448, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (_conv_head): Conv2dStaticSamePadding(\n",
       "      448, 1792, kernel_size=(1, 1), stride=(1, 1), bias=False\n",
       "      (static_padding): Identity()\n",
       "    )\n",
       "    (_bn1): BatchNorm2d(1792, eps=0.001, momentum=0.010000000000000009, affine=True, track_running_stats=True)\n",
       "    (_fc): Identity()\n",
       "  )\n",
       "  (fc): Linear(in_features=3584, out_features=1108, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class EfficientNetTwoInputs(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(EfficientNetTwoInputs, self).__init__()\n",
    "        self.classes = 1108\n",
    "        \n",
    "        model = EfficientNet.from_pretrained(model_name, num_classes=1108) \n",
    "        num_ftrs = model._fc.in_features\n",
    "        model._fc = nn.Identity()\n",
    "        \n",
    "        # accept 6 channels\n",
    "        trained_kernel = model._conv_stem.weight\n",
    "        new_conv = enet_utils.Conv2dStaticSamePadding(6, 48, kernel_size=(3, 3), stride=(2, 2), bias=False, image_size=512)\n",
    "        with torch.no_grad():\n",
    "            new_conv.weight[:,:] = torch.stack([torch.mean(trained_kernel, 1)]*6, dim=1)\n",
    "        model._conv_stem = new_conv\n",
    "        \n",
    "        self.resnet = model\n",
    "        self.fc = nn.Linear(num_ftrs * 2, self.classes)\n",
    "\n",
    "    def forward(self, x1, x2):\n",
    "        x1_out = self.resnet(x1)\n",
    "        x2_out = self.resnet(x2)\n",
    "   \n",
    "        N, _, _, _ = x1.size()\n",
    "        x1_out = x1_out.view(N, -1)\n",
    "        x2_out = x2_out.view(N, -1)\n",
    "        \n",
    "        out = torch.cat((x1_out, x2_out), 1)\n",
    "        out = self.fc(out)\n",
    "\n",
    "        return out \n",
    "    \n",
    "model = EfficientNetTwoInputs()\n",
    "model.load_state_dict(torch.load('./enet4/Model_efficientnet-b4_89.pth'))\n",
    "model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=init_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics = {\n",
    "    'loss': Loss(criterion),\n",
    "    'accuracy': Accuracy(),\n",
    "}\n",
    "\n",
    "trainer = create_supervised_trainer(model, optimizer, criterion, device=device)\n",
    "val_evaluator = create_supervised_evaluator(model, metrics=metrics, device=device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### EarlyStopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# handler = EarlyStopping(patience=30, score_function=lambda engine: engine.state.metrics['accuracy'], trainer=trainer)\n",
    "# val_evaluator.add_event_handler(Events.COMPLETED, handler)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LR Scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "scheduler = CosineAnnealingScheduler(optimizer, 'lr', init_lr, end_lr, len(loader))\n",
    "trainer.add_event_handler(Events.ITERATION_STARTED, scheduler)\n",
    "\n",
    "@trainer.on(Events.ITERATION_COMPLETED)\n",
    "def print_lr(engine):\n",
    "    epoch = engine.state.epoch\n",
    "    iteration = engine.state.iteration\n",
    "    \n",
    "    if epoch < 2 and iteration % 100 == 0:\n",
    "        print(f'Iteration {iteration} | LR {optimizer.param_groups[0][\"lr\"]}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Compute and display metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "@trainer.on(Events.EPOCH_COMPLETED)\n",
    "def compute_and_display_val_metrics(engine):\n",
    "    epoch = engine.state.epoch\n",
    "    metrics = val_evaluator.run(val_loader).metrics\n",
    "    print(\"Validation Results - Epoch: {} | Average Loss: {:.4f} | Accuracy: {:.4f} \"\n",
    "          .format(engine.state.epoch, metrics['loss'], metrics['accuracy']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Save best epoch only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "!mkdir -p models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_saved_model_path(epoch):\n",
    "    return f'models/Model_{model_name}_{epoch + 49}.pth'\n",
    "\n",
    "best_acc = 0.\n",
    "best_epoch = 1\n",
    "best_epoch_file = ''\n",
    "\n",
    "@trainer.on(Events.EPOCH_COMPLETED)\n",
    "def save_best_epoch_only(engine):\n",
    "    epoch = engine.state.epoch\n",
    "\n",
    "    global best_acc\n",
    "    global best_epoch\n",
    "    global best_epoch_file\n",
    "    best_acc = 0. if epoch == 1 else best_acc\n",
    "    best_epoch = 1 if epoch == 1 else best_epoch\n",
    "    best_epoch_file = '' if epoch == 1 else best_epoch_file\n",
    "\n",
    "    metrics = val_evaluator.run(val_loader).metrics\n",
    "\n",
    "    if metrics['accuracy'] > best_acc:\n",
    "        prev_best_epoch_file = get_saved_model_path(best_epoch)\n",
    "        if os.path.exists(prev_best_epoch_file):\n",
    "            os.remove(prev_best_epoch_file)\n",
    "            \n",
    "        best_acc = metrics['accuracy']\n",
    "        best_epoch = epoch\n",
    "        best_epoch_file = get_saved_model_path(best_epoch)\n",
    "        print(f'\\nEpoch: {best_epoch} - New best accuracy! Accuracy: {best_acc}\\n\\n\\n')\n",
    "        torch.save(model.state_dict(), best_epoch_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Progress bar - uncomment when testing in notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pbar = ProgressBar(bar_format='')\n",
    "# pbar.attach(trainer, output_transform=lambda x: {'loss': x})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training started\n",
      "\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-0a7083a51a99>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Training started\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, data, max_epochs)\u001b[0m\n\u001b[1;32m    357\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    358\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Engine run is terminating due to exception: %s.\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 359\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    360\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    361\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36m_handle_exception\u001b[0;34m(self, e)\u001b[0m\n\u001b[1;32m    322\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEXCEPTION_RAISED\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 324\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    325\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    326\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, data, max_epochs)\u001b[0m\n\u001b[1;32m    344\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    345\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEPOCH_STARTED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 346\u001b[0;31m                 \u001b[0mhours\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msecs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_once_on_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    347\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Epoch[%s] Complete. Time taken: %02d:%02d:%02d\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhours\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msecs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    348\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_terminate\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36m_run_once_on_dataset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    311\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    312\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_logger\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Current run is terminating due to exception: %s.\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 313\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_handle_exception\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    314\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    315\u001b[0m         \u001b[0mtime_taken\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart_time\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36m_handle_exception\u001b[0;34m(self, e)\u001b[0m\n\u001b[1;32m    322\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mEXCEPTION_RAISED\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    323\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 324\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    325\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    326\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmax_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/ignite/engine/engine.py\u001b[0m in \u001b[0;36m_run_once_on_dataset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    303\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0miteration\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    304\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mITERATION_STARTED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 305\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_process_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    306\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fire_event\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEvents\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mITERATION_COMPLETED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    307\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_terminate\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_terminate_single_epoch\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/floyd/home/scripts/ignite.py\u001b[0m in \u001b[0;36m_update\u001b[0;34m(engine, batch)\u001b[0m\n\u001b[1;32m     51\u001b[0m         \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloss_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     52\u001b[0m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 53\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     54\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0moutput_transform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m                 \u001b[0;31m# Decay the first and second moment running average coefficient\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m                 \u001b[0mexp_avg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m                 \u001b[0mexp_avg_sq\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbeta2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maddcmul_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mbeta2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mamsgrad\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "print('Training started\\n')\n",
    "trainer.run(loader, max_epochs=8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_preds, _ = eval_model(model, tloader, best_epoch_file, path_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "apply_plates_leak(all_preds)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "2b62ae829edc4d60acf1d9a9e1d598d8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "7740dfb227e54da8b1510dac2d094406": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "921a9c670b6e4a2db86c75a7ff5d9ee6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9dfcb7497f8842af817750eec565b8b9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_921a9c670b6e4a2db86c75a7ff5d9ee6",
       "placeholder": "​",
       "style": "IPY_MODEL_2b62ae829edc4d60acf1d9a9e1d598d8",
       "value": " 94% 2151/2283 [22:45&lt;01:23,  1.58it/s]"
      }
     },
     "d2df0eb5abab4e3895ec792681cfa8d2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": "initial"
      }
     },
     "e3ff3ae302394523bb5b28ee009842d5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_ff74a4321a59419cb24e116db9dd1e3e",
        "IPY_MODEL_9dfcb7497f8842af817750eec565b8b9"
       ],
       "layout": "IPY_MODEL_7740dfb227e54da8b1510dac2d094406"
      }
     },
     "fad7703039454db7af5d7fb4bce65003": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ff74a4321a59419cb24e116db9dd1e3e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "Loss: 128.54232788085938",
       "description_tooltip": null,
       "layout": "IPY_MODEL_fad7703039454db7af5d7fb4bce65003",
       "max": 2283,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_d2df0eb5abab4e3895ec792681cfa8d2",
       "value": 2151
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}