example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "from torch.utils import data\n",
    "from torch import nn \n",
    "import copy\n",
    "\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from time import time\n",
    "from sklearn.metrics import roc_auc_score, average_precision_score, f1_score, roc_curve, confusion_matrix, precision_score, recall_score, auc\n",
    "from sklearn.model_selection import KFold\n",
    "torch.manual_seed(1)    # reproducible torch:2 np:3\n",
    "np.random.seed(1)\n",
    "\n",
    "from config import BIN_config_DBPE\n",
    "from models import BIN_Interaction_Flat\n",
    "from stream import BIN_Data_Encoder\n",
    "\n",
    "use_cuda = torch.cuda.is_available()\n",
    "device = torch.device(\"cuda:0\" if use_cuda else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(data_generator, model):\n",
    "    y_pred = []\n",
    "    y_label = []\n",
    "    model.eval()\n",
    "    loss_accumulate = 0.0\n",
    "    count = 0.0\n",
    "    for i, (d, p, d_mask, p_mask, label) in enumerate(data_generator):\n",
    "        score = model(d.long().cuda(), p.long().cuda(), d_mask.long().cuda(), p_mask.long().cuda())\n",
    "        \n",
    "        m = torch.nn.Sigmoid()\n",
    "        logits = torch.squeeze(m(score))\n",
    "        loss_fct = torch.nn.BCELoss()            \n",
    "        \n",
    "        label = Variable(torch.from_numpy(np.array(label)).float()).cuda()\n",
    "\n",
    "        loss = loss_fct(logits, label)\n",
    "        \n",
    "        loss_accumulate += loss\n",
    "        count += 1\n",
    "        \n",
    "        logits = logits.detach().cpu().numpy()\n",
    "        \n",
    "        label_ids = label.to('cpu').numpy()\n",
    "        y_label = y_label + label_ids.flatten().tolist()\n",
    "        y_pred = y_pred + logits.flatten().tolist()\n",
    "        \n",
    "    loss = loss_accumulate/count\n",
    "    \n",
    "    fpr, tpr, thresholds = roc_curve(y_label, y_pred)\n",
    "\n",
    "    precision = tpr / (tpr + fpr)\n",
    "\n",
    "    f1 = 2 * precision * tpr / (tpr + precision + 0.00001)\n",
    "\n",
    "    thred_optim = thresholds[5:][np.argmax(f1[5:])]\n",
    "\n",
    "    print(\"optimal threshold: \" + str(thred_optim))\n",
    "\n",
    "    y_pred_s = [1 if i else 0 for i in (y_pred >= thred_optim)]\n",
    "\n",
    "    auc_k = auc(fpr, tpr)\n",
    "    print(\"AUROC:\" + str(auc_k))\n",
    "    print(\"AUPRC: \"+ str(average_precision_score(y_label, y_pred)))\n",
    "\n",
    "    cm1 = confusion_matrix(y_label, y_pred_s)\n",
    "    print('Confusion Matrix : \\n', cm1)\n",
    "    print('Recall : ', recall_score(y_label, y_pred_s))\n",
    "    print('Precision : ', precision_score(y_label, y_pred_s))\n",
    "\n",
    "    total1=sum(sum(cm1))\n",
    "    #####from confusion matrix calculate accuracy\n",
    "    accuracy1=(cm1[0,0]+cm1[1,1])/total1\n",
    "    print ('Accuracy : ', accuracy1)\n",
    "\n",
    "    sensitivity1 = cm1[0,0]/(cm1[0,0]+cm1[0,1])\n",
    "    print('Sensitivity : ', sensitivity1 )\n",
    "\n",
    "    specificity1 = cm1[1,1]/(cm1[1,0]+cm1[1,1])\n",
    "    print('Specificity : ', specificity1)\n",
    "\n",
    "    outputs = np.asarray([1 if i else 0 for i in (np.asarray(y_pred) >= 0.5)])\n",
    "    return roc_auc_score(y_label, y_pred), average_precision_score(y_label, y_pred), f1_score(y_label, outputs), y_pred, loss.item()\n",
    "\n",
    "\n",
    "def main(fold_n, lr):\n",
    "    config = BIN_config_DBPE()\n",
    "    \n",
    "    lr = lr\n",
    "    BATCH_SIZE = config['batch_size']\n",
    "    train_epoch = 100\n",
    "    \n",
    "    loss_history = []\n",
    "    \n",
    "    model = BIN_Interaction_Flat(**config)\n",
    "    \n",
    "    model = model.cuda()\n",
    "\n",
    "    if torch.cuda.device_count() > 1:\n",
    "        print(\"Let's use\", torch.cuda.device_count(), \"GPUs!\")\n",
    "        model = nn.DataParallel(model, dim = 0)\n",
    "            \n",
    "    opt = torch.optim.Adam(model.parameters(), lr = lr)\n",
    "    #opt = torch.optim.SGD(model.parameters(), lr = lr, momentum=0.9)\n",
    "    \n",
    "    print('--- Data Preparation ---')\n",
    "    \n",
    "    params = {'batch_size': BATCH_SIZE,\n",
    "              'shuffle': True,\n",
    "              'num_workers': 6, \n",
    "              'drop_last': True}\n",
    "\n",
    "    dataFolder = './dataset/BindingDB'\n",
    "    df_train = pd.read_csv(dataFolder + '/train.csv')\n",
    "    df_val = pd.read_csv(dataFolder + '/val.csv')\n",
    "    df_test = pd.read_csv(dataFolder + '/test.csv')\n",
    "    \n",
    "    training_set = BIN_Data_Encoder(df_train.index.values, df_train.Label.values, df_train)\n",
    "    training_generator = data.DataLoader(training_set, **params)\n",
    "\n",
    "    validation_set = BIN_Data_Encoder(df_val.index.values, df_val.Label.values, df_val)\n",
    "    validation_generator = data.DataLoader(validation_set, **params)\n",
    "    \n",
    "    testing_set = BIN_Data_Encoder(df_test.index.values, df_test.Label.values, df_test)\n",
    "    testing_generator = data.DataLoader(testing_set, **params)\n",
    "    \n",
    "    # early stopping\n",
    "    max_auc = 0\n",
    "    model_max = copy.deepcopy(model)\n",
    "    \n",
    "    print('--- Go for Training ---')\n",
    "    torch.backends.cudnn.benchmark = True\n",
    "    for epo in range(train_epoch):\n",
    "        model.train()\n",
    "        for i, (d, p, d_mask, p_mask, label) in enumerate(training_generator):\n",
    "            score = model(d.long().cuda(), p.long().cuda(), d_mask.long().cuda(), p_mask.long().cuda())\n",
    "\n",
    "            label = Variable(torch.from_numpy(np.array(label)).float()).cuda()\n",
    "            \n",
    "            loss_fct = torch.nn.BCELoss()\n",
    "            m = torch.nn.Sigmoid()\n",
    "            n = torch.squeeze(m(score))\n",
    "            \n",
    "            loss = loss_fct(n, label)\n",
    "            loss_history.append(loss)\n",
    "            \n",
    "            opt.zero_grad()\n",
    "            loss.backward()\n",
    "            opt.step()\n",
    "            \n",
    "            if (i % 100 == 0):\n",
    "                print('Training at Epoch ' + str(epo + 1) + ' iteration ' + str(i) + ' with loss ' + str(loss.cpu().detach().numpy()))\n",
    "            \n",
    "        # every epoch test\n",
    "        with torch.set_grad_enabled(False):\n",
    "            auc, auprc, f1, logits, loss = test(validation_generator, model)\n",
    "            if auc > max_auc:\n",
    "                model_max = copy.deepcopy(model)\n",
    "                max_auc = auc\n",
    "            \n",
    "            print('Validation at Epoch '+ str(epo + 1) + ' , AUROC: '+ str(auc) + ' , AUPRC: ' + str(auprc) + ' , F1: '+str(f1))\n",
    "    \n",
    "    print('--- Go for Testing ---')\n",
    "    try:\n",
    "        with torch.set_grad_enabled(False):\n",
    "            auc, auprc, f1, logits, loss = test(testing_generator, model_max)\n",
    "            print('Testing AUROC: ' + str(auc) + ' , AUPRC: ' + str(auprc) + ' , F1: '+str(f1) + ' , Test loss: '+str(loss))\n",
    "    except:\n",
    "        print('testing failed')\n",
    "    return model_max, loss_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- Data Preparation ---\n",
      "--- Go for Training ---\n",
      "Training at Epoch 1 iteration 0 with loss 0.6687578\n",
      "Training at Epoch 1 iteration 100 with loss 0.6131128\n",
      "Training at Epoch 1 iteration 200 with loss 0.64457905\n",
      "Training at Epoch 1 iteration 300 with loss 0.63140243\n",
      "Training at Epoch 1 iteration 400 with loss 0.56660235\n",
      "Training at Epoch 1 iteration 500 with loss 0.6013969\n",
      "Training at Epoch 1 iteration 600 with loss 0.5783651\n",
      "Training at Epoch 1 iteration 700 with loss 0.6953592\n",
      "optimal threshold: 0.45820438861846924\n",
      "AUROC:0.8287971388191859\n",
      "AUPRC: 0.413317105511778\n",
      "Confusion Matrix : \n",
      " [[8115 3260]\n",
      " [ 346 1559]]\n",
      "Recall :  0.8183727034120735\n",
      "Precision :  0.3235111018883586\n",
      "Accuracy :  0.7284638554216868\n",
      "Sensitivity :  0.7134065934065934\n",
      "Specificity :  0.8183727034120735\n",
      "Validation at Epoch 1 , AUROC: 0.8287971388191859 , AUPRC: 0.413317105511778 , F1: 0.482941371315005\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 2 iteration 0 with loss 0.54751515\n",
      "Training at Epoch 2 iteration 100 with loss 0.5923611\n",
      "Training at Epoch 2 iteration 200 with loss 0.5209317\n",
      "Training at Epoch 2 iteration 300 with loss 0.6359924\n",
      "Training at Epoch 2 iteration 400 with loss 0.59243274\n",
      "Training at Epoch 2 iteration 500 with loss 0.4722705\n",
      "Training at Epoch 2 iteration 600 with loss 0.51201665\n",
      "Training at Epoch 2 iteration 700 with loss 0.6073925\n",
      "optimal threshold: 0.4336393177509308\n",
      "AUROC:0.8572182268213191\n",
      "AUPRC: 0.4650067567599871\n",
      "Confusion Matrix : \n",
      " [[7742 3634]\n",
      " [ 223 1681]]\n",
      "Recall :  0.8828781512605042\n",
      "Precision :  0.316274694261524\n",
      "Accuracy :  0.7095632530120481\n",
      "Sensitivity :  0.6805555555555556\n",
      "Specificity :  0.8828781512605042\n",
      "Validation at Epoch 2 , AUROC: 0.8572182268213191 , AUPRC: 0.4650067567599871 , F1: 0.5097054886211513\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 3 iteration 0 with loss 0.76211536\n",
      "Training at Epoch 3 iteration 100 with loss 0.48241425\n",
      "Training at Epoch 3 iteration 200 with loss 0.52534354\n",
      "Training at Epoch 3 iteration 300 with loss 0.40709087\n",
      "Training at Epoch 3 iteration 400 with loss 0.48139185\n",
      "Training at Epoch 3 iteration 500 with loss 0.43261158\n",
      "Training at Epoch 3 iteration 600 with loss 0.46849614\n",
      "Training at Epoch 3 iteration 700 with loss 0.49497002\n",
      "optimal threshold: 0.3645516037940979\n",
      "AUROC:0.8667227687060824\n",
      "AUPRC: 0.4934045405528476\n",
      "Confusion Matrix : \n",
      " [[7394 3983]\n",
      " [ 163 1740]]\n",
      "Recall :  0.9143457698370994\n",
      "Precision :  0.30403634457452383\n",
      "Accuracy :  0.6878012048192771\n",
      "Sensitivity :  0.6499077085347631\n",
      "Specificity :  0.9143457698370994\n",
      "Validation at Epoch 3 , AUROC: 0.8667227687060824 , AUPRC: 0.4934045405528476 , F1: 0.5245202558635395\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 4 iteration 0 with loss 0.4655727\n",
      "Training at Epoch 4 iteration 100 with loss 0.35244012\n",
      "Training at Epoch 4 iteration 200 with loss 0.508357\n",
      "Training at Epoch 4 iteration 300 with loss 0.35495508\n",
      "Training at Epoch 4 iteration 400 with loss 0.5489132\n",
      "Training at Epoch 4 iteration 500 with loss 0.52321494\n",
      "Training at Epoch 4 iteration 600 with loss 0.40749574\n",
      "Training at Epoch 4 iteration 700 with loss 0.4385683\n",
      "optimal threshold: 0.42705199122428894\n",
      "AUROC:0.8803783437293622\n",
      "AUPRC: 0.5408543180672061\n",
      "Confusion Matrix : \n",
      " [[8011 3367]\n",
      " [ 186 1716]]\n",
      "Recall :  0.9022082018927445\n",
      "Precision :  0.3375959079283887\n",
      "Accuracy :  0.7324548192771084\n",
      "Sensitivity :  0.7040780453506768\n",
      "Specificity :  0.9022082018927445\n",
      "Validation at Epoch 4 , AUROC: 0.8803783437293622 , AUPRC: 0.5408543180672061 , F1: 0.527124019362377\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 5 iteration 0 with loss 0.39542985\n",
      "Training at Epoch 5 iteration 100 with loss 0.37256885\n",
      "Training at Epoch 5 iteration 200 with loss 0.513319\n",
      "Training at Epoch 5 iteration 300 with loss 0.44392866\n",
      "Training at Epoch 5 iteration 400 with loss 0.46204135\n",
      "Training at Epoch 5 iteration 500 with loss 0.44193769\n",
      "Training at Epoch 5 iteration 600 with loss 0.43468052\n",
      "Training at Epoch 5 iteration 700 with loss 0.44528905\n",
      "optimal threshold: 0.40333762764930725\n",
      "AUROC:0.8831680648261414\n",
      "AUPRC: 0.5546666726699543\n",
      "Confusion Matrix : \n",
      " [[7743 3633]\n",
      " [ 145 1759]]\n",
      "Recall :  0.9238445378151261\n",
      "Precision :  0.3262240356083086\n",
      "Accuracy :  0.715512048192771\n",
      "Sensitivity :  0.6806434599156118\n",
      "Specificity :  0.9238445378151261\n",
      "Validation at Epoch 5 , AUROC: 0.8831680648261414 , AUPRC: 0.5546666726699543 , F1: 0.5274688796680499\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 6 iteration 0 with loss 0.4210677\n",
      "Training at Epoch 6 iteration 100 with loss 0.42599458\n",
      "Training at Epoch 6 iteration 200 with loss 0.42392832\n",
      "Training at Epoch 6 iteration 300 with loss 0.3536567\n",
      "Training at Epoch 6 iteration 400 with loss 0.4607436\n",
      "Training at Epoch 6 iteration 500 with loss 0.4136529\n",
      "Training at Epoch 6 iteration 600 with loss 0.31365365\n",
      "Training at Epoch 6 iteration 700 with loss 0.39677495\n",
      "optimal threshold: 0.4591626524925232\n",
      "AUROC:0.8853871419918663\n",
      "AUPRC: 0.5452667710185966\n",
      "Confusion Matrix : \n",
      " [[8095 3280]\n",
      " [ 182 1723]]\n",
      "Recall :  0.9044619422572179\n",
      "Precision :  0.344393363981611\n",
      "Accuracy :  0.7393072289156627\n",
      "Sensitivity :  0.7116483516483516\n",
      "Specificity :  0.9044619422572179\n",
      "Validation at Epoch 6 , AUROC: 0.8853871419918663 , AUPRC: 0.5452667710185966 , F1: 0.5179968701095462\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 7 iteration 0 with loss 0.3864047\n",
      "Training at Epoch 7 iteration 100 with loss 0.34454185\n",
      "Training at Epoch 7 iteration 200 with loss 0.33634776\n",
      "Training at Epoch 7 iteration 300 with loss 0.40733021\n",
      "Training at Epoch 7 iteration 400 with loss 0.47611696\n",
      "Training at Epoch 7 iteration 500 with loss 0.31163216\n",
      "Training at Epoch 7 iteration 600 with loss 0.37158978\n",
      "Training at Epoch 7 iteration 700 with loss 0.35472864\n",
      "optimal threshold: 0.42141830921173096\n",
      "AUROC:0.8910468114562603\n",
      "AUPRC: 0.5551987051426074\n",
      "Confusion Matrix : \n",
      " [[8180 3195]\n",
      " [ 177 1728]]\n",
      "Recall :  0.9070866141732283\n",
      "Precision :  0.3510054844606947\n",
      "Accuracy :  0.7460843373493976\n",
      "Sensitivity :  0.7191208791208791\n",
      "Specificity :  0.9070866141732283\n",
      "Validation at Epoch 7 , AUROC: 0.8910468114562603 , AUPRC: 0.5551987051426074 , F1: 0.5338148391332895\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 8 iteration 0 with loss 0.31566584\n",
      "Training at Epoch 8 iteration 100 with loss 0.6017214\n",
      "Training at Epoch 8 iteration 200 with loss 0.26420036\n",
      "Training at Epoch 8 iteration 300 with loss 0.60125715\n",
      "Training at Epoch 8 iteration 400 with loss 0.31816602\n",
      "Training at Epoch 8 iteration 500 with loss 0.3947599\n",
      "Training at Epoch 8 iteration 600 with loss 0.37606537\n",
      "Training at Epoch 8 iteration 700 with loss 0.289298\n",
      "optimal threshold: 0.3461473286151886\n",
      "AUROC:0.8903770533793686\n",
      "AUPRC: 0.5540269700130303\n",
      "Confusion Matrix : \n",
      " [[8228 3148]\n",
      " [ 175 1729]]\n",
      "Recall :  0.9080882352941176\n",
      "Precision :  0.3545212220627435\n",
      "Accuracy :  0.7497740963855422\n",
      "Sensitivity :  0.7232770745428974\n",
      "Specificity :  0.9080882352941176\n",
      "Validation at Epoch 8 , AUROC: 0.8903770533793686 , AUPRC: 0.5540269700130303 , F1: 0.5514018691588785\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 9 iteration 0 with loss 0.31785774\n",
      "Training at Epoch 9 iteration 100 with loss 0.38460368\n",
      "Training at Epoch 9 iteration 200 with loss 0.28542277\n",
      "Training at Epoch 9 iteration 300 with loss 0.42280567\n",
      "Training at Epoch 9 iteration 400 with loss 0.31990814\n",
      "Training at Epoch 9 iteration 500 with loss 0.32363963\n",
      "Training at Epoch 9 iteration 600 with loss 0.2845815\n",
      "Training at Epoch 9 iteration 700 with loss 0.3063144\n",
      "optimal threshold: 0.4382968544960022\n",
      "AUROC:0.8915550595238094\n",
      "AUPRC: 0.527278122436895\n",
      "Confusion Matrix : \n",
      " [[8376 3000]\n",
      " [ 179 1725]]\n",
      "Recall :  0.9059873949579832\n",
      "Precision :  0.36507936507936506\n",
      "Accuracy :  0.7606174698795181\n",
      "Sensitivity :  0.7362869198312236\n",
      "Specificity :  0.9059873949579832\n",
      "Validation at Epoch 9 , AUROC: 0.8915550595238094 , AUPRC: 0.527278122436895 , F1: 0.5411610364053788\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 10 iteration 0 with loss 0.41566882\n",
      "Training at Epoch 10 iteration 100 with loss 0.56288254\n",
      "Training at Epoch 10 iteration 200 with loss 0.55108416\n",
      "Training at Epoch 10 iteration 300 with loss 0.28254604\n",
      "Training at Epoch 10 iteration 400 with loss 0.32634223\n",
      "Training at Epoch 10 iteration 500 with loss 0.27280563\n",
      "Training at Epoch 10 iteration 600 with loss 0.36122933\n",
      "Training at Epoch 10 iteration 700 with loss 0.3377106\n",
      "optimal threshold: 0.4543067216873169\n",
      "AUROC:0.8958291551061351\n",
      "AUPRC: 0.5490557328794263\n",
      "Confusion Matrix : \n",
      " [[8601 2775]\n",
      " [ 194 1710]]\n",
      "Recall :  0.898109243697479\n",
      "Precision :  0.38127090301003347\n",
      "Accuracy :  0.7764307228915662\n",
      "Sensitivity :  0.7560654008438819\n",
      "Specificity :  0.898109243697479\n",
      "Validation at Epoch 10 , AUROC: 0.8958291551061351 , AUPRC: 0.5490557328794263 , F1: 0.5470763131813677\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 11 iteration 0 with loss 0.2973517\n",
      "Training at Epoch 11 iteration 100 with loss 0.20083396\n",
      "Training at Epoch 11 iteration 200 with loss 0.41760522\n",
      "Training at Epoch 11 iteration 300 with loss 0.46515882\n",
      "Training at Epoch 11 iteration 400 with loss 0.45584506\n",
      "Training at Epoch 11 iteration 500 with loss 0.5080781\n",
      "Training at Epoch 11 iteration 600 with loss 0.211982\n",
      "Training at Epoch 11 iteration 700 with loss 0.50351095\n",
      "optimal threshold: 0.45910003781318665\n",
      "AUROC:0.8961517327131275\n",
      "AUPRC: 0.5427569600298573\n",
      "Confusion Matrix : \n",
      " [[8604 2772]\n",
      " [ 203 1701]]\n",
      "Recall :  0.8933823529411765\n",
      "Precision :  0.38028169014084506\n",
      "Accuracy :  0.7759789156626506\n",
      "Sensitivity :  0.7563291139240507\n",
      "Specificity :  0.8933823529411765\n",
      "Validation at Epoch 11 , AUROC: 0.8961517327131275 , AUPRC: 0.5427569600298573 , F1: 0.5462046204620463\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 12 iteration 0 with loss 0.6123441\n",
      "Training at Epoch 12 iteration 100 with loss 0.31953248\n",
      "Training at Epoch 12 iteration 200 with loss 0.3130611\n",
      "Training at Epoch 12 iteration 300 with loss 0.41254798\n",
      "Training at Epoch 12 iteration 400 with loss 0.38823605\n",
      "Training at Epoch 12 iteration 500 with loss 0.426561\n",
      "Training at Epoch 12 iteration 600 with loss 0.36083424\n",
      "Training at Epoch 12 iteration 700 with loss 0.36723775\n",
      "optimal threshold: 0.4585101306438446\n",
      "AUROC:0.8977755857089672\n",
      "AUPRC: 0.5584784988191449\n",
      "Confusion Matrix : \n",
      " [[8480 2896]\n",
      " [ 174 1730]]\n",
      "Recall :  0.9086134453781513\n",
      "Precision :  0.37397319498486814\n",
      "Accuracy :  0.7688253012048193\n",
      "Sensitivity :  0.7454289732770746\n",
      "Specificity :  0.9086134453781513\n",
      "Validation at Epoch 12 , AUROC: 0.8977755857089672 , AUPRC: 0.5584784988191449 , F1: 0.5430548849187188\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 13 iteration 0 with loss 0.23632544\n",
      "Training at Epoch 13 iteration 100 with loss 0.4007699\n",
      "Training at Epoch 13 iteration 200 with loss 0.42466667\n",
      "Training at Epoch 13 iteration 300 with loss 0.49687007\n",
      "Training at Epoch 13 iteration 400 with loss 0.24691118\n",
      "Training at Epoch 13 iteration 500 with loss 0.55031896\n",
      "Training at Epoch 13 iteration 600 with loss 0.4321213\n",
      "Training at Epoch 13 iteration 700 with loss 0.42407632\n",
      "optimal threshold: 0.4131617844104767\n",
      "AUROC:0.9008087681347524\n",
      "AUPRC: 0.5736807002570349\n",
      "Confusion Matrix : \n",
      " [[8309 3066]\n",
      " [ 143 1762]]\n",
      "Recall :  0.9249343832020998\n",
      "Precision :  0.36495443247721626\n",
      "Accuracy :  0.7583584337349397\n",
      "Sensitivity :  0.7304615384615385\n",
      "Specificity :  0.9249343832020998\n",
      "Validation at Epoch 13 , AUROC: 0.9008087681347524 , AUPRC: 0.5736807002570349 , F1: 0.5468164794007492\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 14 iteration 0 with loss 0.21237463\n",
      "Training at Epoch 14 iteration 100 with loss 0.2414099\n",
      "Training at Epoch 14 iteration 200 with loss 0.37060747\n",
      "Training at Epoch 14 iteration 300 with loss 0.41737154\n",
      "Training at Epoch 14 iteration 400 with loss 0.2818727\n",
      "Training at Epoch 14 iteration 500 with loss 0.3416828\n",
      "Training at Epoch 14 iteration 600 with loss 0.18360442\n",
      "Training at Epoch 14 iteration 700 with loss 0.4197511\n",
      "optimal threshold: 0.4353664815425873\n",
      "AUROC:0.9046537336679068\n",
      "AUPRC: 0.5812357910763775\n",
      "Confusion Matrix : \n",
      " [[8455 2920]\n",
      " [ 164 1741]]\n",
      "Recall :  0.9139107611548556\n",
      "Precision :  0.37352499463634414\n",
      "Accuracy :  0.7677710843373494\n",
      "Sensitivity :  0.7432967032967033\n",
      "Specificity :  0.9139107611548556\n",
      "Validation at Epoch 14 , AUROC: 0.9046537336679068 , AUPRC: 0.5812357910763775 , F1: 0.5501544464314746\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 15 iteration 0 with loss 0.31794614\n",
      "Training at Epoch 15 iteration 100 with loss 0.24009345\n",
      "Training at Epoch 15 iteration 200 with loss 0.23622327\n",
      "Training at Epoch 15 iteration 300 with loss 0.21803865\n",
      "Training at Epoch 15 iteration 400 with loss 0.303263\n",
      "Training at Epoch 15 iteration 500 with loss 0.29343677\n",
      "Training at Epoch 15 iteration 600 with loss 0.38064715\n",
      "Training at Epoch 15 iteration 700 with loss 0.32521772\n",
      "optimal threshold: 0.4205222427845001\n",
      "AUROC:0.902560602299992\n",
      "AUPRC: 0.5805558614281385\n",
      "Confusion Matrix : \n",
      " [[8342 3034]\n",
      " [ 165 1739]]\n",
      "Recall :  0.9133403361344538\n",
      "Precision :  0.3643410852713178\n",
      "Accuracy :  0.7591114457831325\n",
      "Sensitivity :  0.7332981715893109\n",
      "Specificity :  0.9133403361344538\n",
      "Validation at Epoch 15 , AUROC: 0.902560602299992 , AUPRC: 0.5805558614281385 , F1: 0.5469133780907156\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 16 iteration 0 with loss 0.39882445\n",
      "Training at Epoch 16 iteration 100 with loss 0.23038784\n",
      "Training at Epoch 16 iteration 200 with loss 0.18663248\n",
      "Training at Epoch 16 iteration 300 with loss 0.4433686\n",
      "Training at Epoch 16 iteration 400 with loss 0.2816658\n",
      "Training at Epoch 16 iteration 500 with loss 0.25306726\n",
      "Training at Epoch 16 iteration 600 with loss 0.44300866\n",
      "Training at Epoch 16 iteration 700 with loss 0.40727574\n",
      "optimal threshold: 0.5114461779594421\n",
      "AUROC:0.9045015316323265\n",
      "AUPRC: 0.5770786452162552\n",
      "Confusion Matrix : \n",
      " [[8698 2679]\n",
      " [ 190 1713]]\n",
      "Recall :  0.9001576458223857\n",
      "Precision :  0.3900273224043716\n",
      "Accuracy :  0.7839608433734939\n",
      "Sensitivity :  0.764524918695614\n",
      "Specificity :  0.9001576458223857\n",
      "Validation at Epoch 16 , AUROC: 0.9045015316323265 , AUPRC: 0.5770786452162552 , F1: 0.5384374019454031\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 17 iteration 0 with loss 0.22627155\n",
      "Training at Epoch 17 iteration 100 with loss 0.37241656\n",
      "Training at Epoch 17 iteration 200 with loss 0.46175522\n",
      "Training at Epoch 17 iteration 300 with loss 0.36947444\n",
      "Training at Epoch 17 iteration 400 with loss 0.37070817\n",
      "Training at Epoch 17 iteration 500 with loss 0.18922567\n",
      "Training at Epoch 17 iteration 600 with loss 0.6054473\n",
      "Training at Epoch 17 iteration 700 with loss 0.27242285\n",
      "optimal threshold: 0.4137243628501892\n",
      "AUROC:0.9067702393545884\n",
      "AUPRC: 0.598114700816768\n",
      "Confusion Matrix : \n",
      " [[8856 2521]\n",
      " [ 203 1700]]\n",
      "Recall :  0.8933263268523384\n",
      "Precision :  0.4027481639421938\n",
      "Accuracy :  0.7948795180722892\n",
      "Sensitivity :  0.7784125867979257\n",
      "Specificity :  0.8933263268523384\n",
      "Validation at Epoch 17 , AUROC: 0.9067702393545884 , AUPRC: 0.598114700816768 , F1: 0.5757363253856943\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 18 iteration 0 with loss 0.2671366\n",
      "Training at Epoch 18 iteration 100 with loss 0.2229254\n",
      "Training at Epoch 18 iteration 200 with loss 0.40808788\n",
      "Training at Epoch 18 iteration 300 with loss 0.275303\n",
      "Training at Epoch 18 iteration 400 with loss 0.25729007\n",
      "Training at Epoch 18 iteration 500 with loss 0.15258569\n",
      "Training at Epoch 18 iteration 600 with loss 0.22036697\n",
      "Training at Epoch 18 iteration 700 with loss 0.43161917\n",
      "optimal threshold: 0.4544684588909149\n",
      "AUROC:0.9068110946669824\n",
      "AUPRC: 0.5895787699368359\n",
      "Confusion Matrix : \n",
      " [[8682 2696]\n",
      " [ 175 1727]]\n",
      "Recall :  0.9079915878023134\n",
      "Precision :  0.3904589645037305\n",
      "Accuracy :  0.7838102409638554\n",
      "Sensitivity :  0.763051502900334\n",
      "Specificity :  0.9079915878023134\n",
      "Validation at Epoch 18 , AUROC: 0.9068110946669824 , AUPRC: 0.5895787699368359 , F1: 0.5577623303541873\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 19 iteration 0 with loss 0.47100067\n",
      "Training at Epoch 19 iteration 100 with loss 0.22283724\n",
      "Training at Epoch 19 iteration 200 with loss 0.19516581\n",
      "Training at Epoch 19 iteration 300 with loss 0.2928453\n",
      "Training at Epoch 19 iteration 400 with loss 0.3894276\n",
      "Training at Epoch 19 iteration 500 with loss 0.23239008\n",
      "Training at Epoch 19 iteration 600 with loss 0.2685936\n",
      "Training at Epoch 19 iteration 700 with loss 0.23185325\n",
      "optimal threshold: 0.45064476132392883\n",
      "AUROC:0.9085653609757353\n",
      "AUPRC: 0.5860346718056736\n",
      "Confusion Matrix : \n",
      " [[8866 2512]\n",
      " [ 203 1699]]\n",
      "Recall :  0.8932702418506835\n",
      "Precision :  0.40346710995013063\n",
      "Accuracy :  0.7955572289156626\n",
      "Sensitivity :  0.7792230620495694\n",
      "Specificity :  0.8932702418506835\n",
      "Validation at Epoch 19 , AUROC: 0.9085653609757353 , AUPRC: 0.5860346718056736 , F1: 0.5689300411522634\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 20 iteration 0 with loss 0.33571684\n",
      "Training at Epoch 20 iteration 100 with loss 0.26386994\n",
      "Training at Epoch 20 iteration 200 with loss 0.23417406\n",
      "Training at Epoch 20 iteration 300 with loss 0.19838974\n",
      "Training at Epoch 20 iteration 400 with loss 0.29120523\n",
      "Training at Epoch 20 iteration 500 with loss 0.21226735\n",
      "Training at Epoch 20 iteration 600 with loss 0.31986398\n",
      "Training at Epoch 20 iteration 700 with loss 0.293334\n",
      "optimal threshold: 0.38383591175079346\n",
      "AUROC:0.9054635237533831\n",
      "AUPRC: 0.5773089904065806\n",
      "Confusion Matrix : \n",
      " [[8543 2833]\n",
      " [ 168 1736]]\n",
      "Recall :  0.9117647058823529\n",
      "Precision :  0.3799518494200044\n",
      "Accuracy :  0.7740210843373494\n",
      "Sensitivity :  0.7509669479606188\n",
      "Specificity :  0.9117647058823529\n",
      "Validation at Epoch 20 , AUROC: 0.9054635237533831 , AUPRC: 0.5773089904065806 , F1: 0.564469425992165\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 21 iteration 0 with loss 0.29901457\n",
      "Training at Epoch 21 iteration 100 with loss 0.19063595\n",
      "Training at Epoch 21 iteration 200 with loss 0.20020232\n",
      "Training at Epoch 21 iteration 300 with loss 0.47887883\n",
      "Training at Epoch 21 iteration 400 with loss 0.13492385\n",
      "Training at Epoch 21 iteration 500 with loss 0.26537076\n",
      "Training at Epoch 21 iteration 600 with loss 0.4418953\n",
      "Training at Epoch 21 iteration 700 with loss 0.17565244\n",
      "optimal threshold: 0.33264514803886414\n",
      "AUROC:0.9106911092496071\n",
      "AUPRC: 0.5885841896599382\n",
      "Confusion Matrix : \n",
      " [[8661 2715]\n",
      " [ 151 1753]]\n",
      "Recall :  0.9206932773109243\n",
      "Precision :  0.3923455684870188\n",
      "Accuracy :  0.7841867469879518\n",
      "Sensitivity :  0.7613396624472574\n",
      "Specificity :  0.9206932773109243\n",
      "Validation at Epoch 21 , AUROC: 0.9106911092496071 , AUPRC: 0.5885841896599382 , F1: 0.5853306902144529\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 22 iteration 0 with loss 0.2628677\n",
      "Training at Epoch 22 iteration 100 with loss 0.27212954\n",
      "Training at Epoch 22 iteration 200 with loss 0.19531035\n",
      "Training at Epoch 22 iteration 300 with loss 0.4682579\n",
      "Training at Epoch 22 iteration 400 with loss 0.3740888\n",
      "Training at Epoch 22 iteration 500 with loss 0.20119388\n",
      "Training at Epoch 22 iteration 600 with loss 0.2853523\n",
      "Training at Epoch 22 iteration 700 with loss 0.19811621\n",
      "optimal threshold: 0.3855345845222473\n",
      "AUROC:0.9109943584956808\n",
      "AUPRC: 0.6007356001959462\n",
      "Confusion Matrix : \n",
      " [[8760 2617]\n",
      " [ 170 1733]]\n",
      "Recall :  0.9106673673147662\n",
      "Precision :  0.39839080459770115\n",
      "Accuracy :  0.7901355421686747\n",
      "Sensitivity :  0.7699745099762679\n",
      "Specificity :  0.9106673673147662\n",
      "Validation at Epoch 22 , AUROC: 0.9109943584956808 , AUPRC: 0.6007356001959462 , F1: 0.5785353974604279\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 23 iteration 0 with loss 0.21706407\n",
      "Training at Epoch 23 iteration 100 with loss 0.49184704\n",
      "Training at Epoch 23 iteration 200 with loss 0.27468246\n",
      "Training at Epoch 23 iteration 300 with loss 0.13125288\n",
      "Training at Epoch 23 iteration 400 with loss 0.5377386\n",
      "Training at Epoch 23 iteration 500 with loss 0.18604329\n",
      "Training at Epoch 23 iteration 600 with loss 0.4365265\n",
      "Training at Epoch 23 iteration 700 with loss 0.22505657\n",
      "optimal threshold: 0.3632024824619293\n",
      "AUROC:0.9101758484248129\n",
      "AUPRC: 0.5972548018860016\n",
      "Confusion Matrix : \n",
      " [[8706 2670]\n",
      " [ 156 1748]]\n",
      "Recall :  0.9180672268907563\n",
      "Precision :  0.39565414214576733\n",
      "Accuracy :  0.7871987951807229\n",
      "Sensitivity :  0.765295358649789\n",
      "Specificity :  0.9180672268907563\n",
      "Validation at Epoch 23 , AUROC: 0.9101758484248129 , AUPRC: 0.5972548018860016 , F1: 0.5827134254339356\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 24 iteration 0 with loss 0.17102584\n",
      "Training at Epoch 24 iteration 100 with loss 0.26296663\n",
      "Training at Epoch 24 iteration 200 with loss 0.25515085\n",
      "Training at Epoch 24 iteration 300 with loss 0.5022435\n",
      "Training at Epoch 24 iteration 400 with loss 0.1599897\n",
      "Training at Epoch 24 iteration 500 with loss 0.19347279\n",
      "Training at Epoch 24 iteration 600 with loss 0.4645342\n",
      "Training at Epoch 24 iteration 700 with loss 0.35451558\n",
      "optimal threshold: 0.2882738709449768\n",
      "AUROC:0.9134637319691233\n",
      "AUPRC: 0.607508042053388\n",
      "Confusion Matrix : \n",
      " [[8697 2680]\n",
      " [ 150 1753]]\n",
      "Recall :  0.9211770888071467\n",
      "Precision :  0.39544326641100835\n",
      "Accuracy :  0.7868975903614458\n",
      "Sensitivity :  0.764437022062055\n",
      "Specificity :  0.9211770888071467\n",
      "Validation at Epoch 24 , AUROC: 0.9134637319691233 , AUPRC: 0.607508042053388 , F1: 0.5940520446096654\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 25 iteration 0 with loss 0.21491355\n",
      "Training at Epoch 25 iteration 100 with loss 0.1948645\n",
      "Training at Epoch 25 iteration 200 with loss 0.41792238\n",
      "Training at Epoch 25 iteration 300 with loss 0.23288174\n",
      "Training at Epoch 25 iteration 400 with loss 0.18583676\n",
      "Training at Epoch 25 iteration 500 with loss 0.4230837\n",
      "Training at Epoch 25 iteration 600 with loss 0.16417354\n",
      "Training at Epoch 25 iteration 700 with loss 0.11278347\n",
      "optimal threshold: 0.463826060295105\n",
      "AUROC:0.9113371655115422\n",
      "AUPRC: 0.6063670648006071\n",
      "Confusion Matrix : \n",
      " [[8668 2710]\n",
      " [ 159 1743]]\n",
      "Recall :  0.916403785488959\n",
      "Precision :  0.3914215135863463\n",
      "Accuracy :  0.7839608433734939\n",
      "Sensitivity :  0.7618210581824574\n",
      "Specificity :  0.916403785488959\n",
      "Validation at Epoch 25 , AUROC: 0.9113371655115422 , AUPRC: 0.6063670648006071 , F1: 0.5554119547657512\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 26 iteration 0 with loss 0.22283229\n",
      "Training at Epoch 26 iteration 100 with loss 0.101939745\n",
      "Training at Epoch 26 iteration 200 with loss 0.18984878\n",
      "Training at Epoch 26 iteration 300 with loss 0.2869362\n",
      "Training at Epoch 26 iteration 400 with loss 0.18705682\n",
      "Training at Epoch 26 iteration 500 with loss 0.32020798\n",
      "Training at Epoch 26 iteration 600 with loss 0.23628417\n",
      "Training at Epoch 26 iteration 700 with loss 0.17509283\n",
      "optimal threshold: 0.2693442702293396\n",
      "AUROC:0.9087558562569972\n",
      "AUPRC: 0.5609668836044935\n",
      "Confusion Matrix : \n",
      " [[8478 2900]\n",
      " [ 144 1758]]\n",
      "Recall :  0.9242902208201893\n",
      "Precision :  0.37741519965650494\n",
      "Accuracy :  0.7707831325301204\n",
      "Sensitivity :  0.7451221655827035\n",
      "Specificity :  0.9242902208201893\n",
      "Validation at Epoch 26 , AUROC: 0.9087558562569972 , AUPRC: 0.5609668836044935 , F1: 0.5874406717780212\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 27 iteration 0 with loss 0.21620536\n",
      "Training at Epoch 27 iteration 100 with loss 0.17158127\n",
      "Training at Epoch 27 iteration 200 with loss 0.12793052\n",
      "Training at Epoch 27 iteration 300 with loss 0.25551188\n",
      "Training at Epoch 27 iteration 400 with loss 0.2561753\n",
      "Training at Epoch 27 iteration 500 with loss 0.26108965\n",
      "Training at Epoch 27 iteration 600 with loss 0.104477026\n",
      "Training at Epoch 27 iteration 700 with loss 0.24071035\n",
      "optimal threshold: 0.3852746784687042\n",
      "AUROC:0.9108640786219551\n",
      "AUPRC: 0.5812797309614258\n",
      "Confusion Matrix : \n",
      " [[8878 2498]\n",
      " [ 170 1734]]\n",
      "Recall :  0.9107142857142857\n",
      "Precision :  0.40973534971644615\n",
      "Accuracy :  0.7990963855421687\n",
      "Sensitivity :  0.7804149085794655\n",
      "Specificity :  0.9107142857142857\n",
      "Validation at Epoch 27 , AUROC: 0.9108640786219551 , AUPRC: 0.5812797309614258 , F1: 0.5869106263194933\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 28 iteration 0 with loss 0.23876543\n",
      "Training at Epoch 28 iteration 100 with loss 0.5640094\n",
      "Training at Epoch 28 iteration 200 with loss 0.358315\n",
      "Training at Epoch 28 iteration 300 with loss 0.15396619\n",
      "Training at Epoch 28 iteration 400 with loss 0.38123047\n",
      "Training at Epoch 28 iteration 500 with loss 0.24560913\n",
      "Training at Epoch 28 iteration 600 with loss 0.22264881\n",
      "Training at Epoch 28 iteration 700 with loss 0.23591828\n",
      "optimal threshold: 0.4661433696746826\n",
      "AUROC:0.9104694297828156\n",
      "AUPRC: 0.5750345310478633\n",
      "Confusion Matrix : \n",
      " [[9072 2303]\n",
      " [ 203 1702]]\n",
      "Recall :  0.8934383202099737\n",
      "Precision :  0.42496878901373286\n",
      "Accuracy :  0.8112951807228915\n",
      "Sensitivity :  0.7975384615384615\n",
      "Specificity :  0.8934383202099737\n",
      "Validation at Epoch 28 , AUROC: 0.9104694297828156 , AUPRC: 0.5750345310478633 , F1: 0.5807345807345807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 29 iteration 0 with loss 0.28994402\n",
      "Training at Epoch 29 iteration 100 with loss 0.19070208\n",
      "Training at Epoch 29 iteration 200 with loss 0.26405713\n",
      "Training at Epoch 29 iteration 300 with loss 0.10757662\n",
      "Training at Epoch 29 iteration 400 with loss 0.24794139\n",
      "Training at Epoch 29 iteration 500 with loss 0.6263391\n",
      "Training at Epoch 29 iteration 600 with loss 0.30488357\n",
      "Training at Epoch 29 iteration 700 with loss 0.2652835\n",
      "optimal threshold: 0.3344365656375885\n",
      "AUROC:0.9108640324536986\n",
      "AUPRC: 0.5821197651785299\n",
      "Confusion Matrix : \n",
      " [[8821 2555]\n",
      " [ 160 1744]]\n",
      "Recall :  0.9159663865546218\n",
      "Precision :  0.4056757385438474\n",
      "Accuracy :  0.7955572289156626\n",
      "Sensitivity :  0.7754043600562588\n",
      "Specificity :  0.9159663865546218\n",
      "Validation at Epoch 29 , AUROC: 0.9108640324536986 , AUPRC: 0.5821197651785299 , F1: 0.5890533071848815\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 30 iteration 0 with loss 0.20439121\n",
      "Training at Epoch 30 iteration 100 with loss 0.2577244\n",
      "Training at Epoch 30 iteration 200 with loss 0.25527805\n",
      "Training at Epoch 30 iteration 300 with loss 0.22229595\n",
      "Training at Epoch 30 iteration 400 with loss 0.263974\n",
      "Training at Epoch 30 iteration 500 with loss 0.2337648\n",
      "Training at Epoch 30 iteration 600 with loss 0.1730138\n",
      "Training at Epoch 30 iteration 700 with loss 0.1560278\n",
      "optimal threshold: 0.4170575439929962\n",
      "AUROC:0.9109614705586876\n",
      "AUPRC: 0.5742304519670994\n",
      "Confusion Matrix : \n",
      " [[9012 2364]\n",
      " [ 180 1724]]\n",
      "Recall :  0.9054621848739496\n",
      "Precision :  0.4217221135029354\n",
      "Accuracy :  0.808433734939759\n",
      "Sensitivity :  0.7921940928270043\n",
      "Specificity :  0.9054621848739496\n",
      "Validation at Epoch 30 , AUROC: 0.9109614705586876 , AUPRC: 0.5742304519670994 , F1: 0.589125461903924\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 31 iteration 0 with loss 0.22755206\n",
      "Training at Epoch 31 iteration 100 with loss 0.254847\n",
      "Training at Epoch 31 iteration 200 with loss 0.27933338\n",
      "Training at Epoch 31 iteration 300 with loss 0.17644404\n",
      "Training at Epoch 31 iteration 400 with loss 0.32923543\n",
      "Training at Epoch 31 iteration 500 with loss 0.16113636\n",
      "Training at Epoch 31 iteration 600 with loss 0.22072572\n",
      "Training at Epoch 31 iteration 700 with loss 0.18967415\n",
      "optimal threshold: 0.45373106002807617\n",
      "AUROC:0.9128353246625655\n",
      "AUPRC: 0.6033176061844878\n",
      "Confusion Matrix : \n",
      " [[9202 2174]\n",
      " [ 198 1706]]\n",
      "Recall :  0.8960084033613446\n",
      "Precision :  0.43969072164948453\n",
      "Accuracy :  0.8213855421686747\n",
      "Sensitivity :  0.8088959212376934\n",
      "Specificity :  0.8960084033613446\n",
      "Validation at Epoch 31 , AUROC: 0.9128353246625655 , AUPRC: 0.6033176061844878 , F1: 0.5968605066000714\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 32 iteration 0 with loss 0.19717982\n",
      "Training at Epoch 32 iteration 100 with loss 0.23027214\n",
      "Training at Epoch 32 iteration 200 with loss 0.20068187\n",
      "Training at Epoch 32 iteration 300 with loss 0.18876544\n",
      "Training at Epoch 32 iteration 400 with loss 0.08227168\n",
      "Training at Epoch 32 iteration 500 with loss 0.16377918\n",
      "Training at Epoch 32 iteration 600 with loss 0.22060898\n",
      "Training at Epoch 32 iteration 700 with loss 0.30823395\n",
      "optimal threshold: 0.40188437700271606\n",
      "AUROC:0.9165380217029229\n",
      "AUPRC: 0.6167060871736325\n",
      "Confusion Matrix : \n",
      " [[9053 2325]\n",
      " [ 190 1712]]\n",
      "Recall :  0.900105152471083\n",
      "Precision :  0.4240772851127075\n",
      "Accuracy :  0.8106174698795181\n",
      "Sensitivity :  0.795658287924064\n",
      "Specificity :  0.900105152471083\n",
      "Validation at Epoch 32 , AUROC: 0.9165380217029229 , AUPRC: 0.6167060871736325 , F1: 0.5975237753454155\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 33 iteration 0 with loss 0.13174736\n",
      "Training at Epoch 33 iteration 100 with loss 0.18693846\n",
      "Training at Epoch 33 iteration 200 with loss 0.1629883\n",
      "Training at Epoch 33 iteration 300 with loss 0.19293681\n",
      "Training at Epoch 33 iteration 400 with loss 0.23909621\n",
      "Training at Epoch 33 iteration 500 with loss 0.17575642\n",
      "Training at Epoch 33 iteration 600 with loss 0.2652252\n",
      "Training at Epoch 33 iteration 700 with loss 0.11586655\n",
      "optimal threshold: 0.5239027738571167\n",
      "AUROC:0.9092290587656199\n",
      "AUPRC: 0.5903693279471188\n",
      "Confusion Matrix : \n",
      " [[8705 2672]\n",
      " [ 171 1732]]\n",
      "Recall :  0.9101418812401472\n",
      "Precision :  0.3932788374205268\n",
      "Accuracy :  0.7859186746987952\n",
      "Sensitivity :  0.7651401951305264\n",
      "Specificity :  0.9101418812401472\n",
      "Validation at Epoch 33 , AUROC: 0.9092290587656199 , AUPRC: 0.5903693279471188 , F1: 0.5417445482866045\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 34 iteration 0 with loss 0.11344886\n",
      "Training at Epoch 34 iteration 100 with loss 0.14818227\n",
      "Training at Epoch 34 iteration 200 with loss 0.22837068\n",
      "Training at Epoch 34 iteration 300 with loss 0.43818888\n",
      "Training at Epoch 34 iteration 400 with loss 0.26976496\n",
      "Training at Epoch 34 iteration 500 with loss 0.22655885\n",
      "Training at Epoch 34 iteration 600 with loss 0.15259041\n",
      "Training at Epoch 34 iteration 700 with loss 0.16960043\n",
      "optimal threshold: 0.4028904438018799\n",
      "AUROC:0.911840352570353\n",
      "AUPRC: 0.5954258754669081\n",
      "Confusion Matrix : \n",
      " [[8674 2702]\n",
      " [ 145 1759]]\n",
      "Recall :  0.9238445378151261\n",
      "Precision :  0.3943062093700964\n",
      "Accuracy :  0.7856174698795181\n",
      "Sensitivity :  0.7624824191279888\n",
      "Specificity :  0.9238445378151261\n",
      "Validation at Epoch 34 , AUROC: 0.911840352570353 , AUPRC: 0.5954258754669081 , F1: 0.5694676933377971\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 35 iteration 0 with loss 0.17519647\n",
      "Training at Epoch 35 iteration 100 with loss 0.19098657\n",
      "Training at Epoch 35 iteration 200 with loss 0.14564592\n",
      "Training at Epoch 35 iteration 300 with loss 0.44688997\n",
      "Training at Epoch 35 iteration 400 with loss 0.2603044\n",
      "Training at Epoch 35 iteration 500 with loss 0.38341966\n",
      "Training at Epoch 35 iteration 600 with loss 0.17595163\n",
      "Training at Epoch 35 iteration 700 with loss 0.316229\n",
      "optimal threshold: 0.24454373121261597\n",
      "AUROC:0.9127006749429729\n",
      "AUPRC: 0.5924927159273463\n",
      "Confusion Matrix : \n",
      " [[8816 2560]\n",
      " [ 148 1756]]\n",
      "Recall :  0.9222689075630253\n",
      "Precision :  0.4068582020389249\n",
      "Accuracy :  0.7960843373493975\n",
      "Sensitivity :  0.7749648382559775\n",
      "Specificity :  0.9222689075630253\n",
      "Validation at Epoch 35 , AUROC: 0.9127006749429729 , AUPRC: 0.5924927159273463 , F1: 0.6013488197826902\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 36 iteration 0 with loss 0.35493138\n",
      "Training at Epoch 36 iteration 100 with loss 0.2250959\n",
      "Training at Epoch 36 iteration 200 with loss 0.10896918\n",
      "Training at Epoch 36 iteration 300 with loss 0.28244892\n",
      "Training at Epoch 36 iteration 400 with loss 0.23110895\n",
      "Training at Epoch 36 iteration 500 with loss 0.15515316\n",
      "Training at Epoch 36 iteration 600 with loss 0.34797087\n",
      "Training at Epoch 36 iteration 700 with loss 0.2778519\n",
      "optimal threshold: 0.3984920382499695\n",
      "AUROC:0.9123455451196842\n",
      "AUPRC: 0.5866598029514696\n",
      "Confusion Matrix : \n",
      " [[8812 2567]\n",
      " [ 151 1750]]\n",
      "Recall :  0.920568122041031\n",
      "Precision :  0.40537410238591615\n",
      "Accuracy :  0.7953313253012049\n",
      "Sensitivity :  0.774408999033307\n",
      "Specificity :  0.920568122041031\n",
      "Validation at Epoch 36 , AUROC: 0.9123455451196842 , AUPRC: 0.5866598029514696 , F1: 0.5786163522012578\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 37 iteration 0 with loss 0.166769\n",
      "Training at Epoch 37 iteration 100 with loss 0.27890673\n",
      "Training at Epoch 37 iteration 200 with loss 0.42016882\n",
      "Training at Epoch 37 iteration 300 with loss 0.37894714\n",
      "Training at Epoch 37 iteration 400 with loss 0.14671478\n",
      "Training at Epoch 37 iteration 500 with loss 0.09071389\n",
      "Training at Epoch 37 iteration 600 with loss 0.09720045\n",
      "Training at Epoch 37 iteration 700 with loss 0.1629264\n",
      "optimal threshold: 0.3523436188697815\n",
      "AUROC:0.9136365460807686\n",
      "AUPRC: 0.60268823878385\n",
      "Confusion Matrix : \n",
      " [[9078 2299]\n",
      " [ 184 1719]]\n",
      "Recall :  0.9033105622700999\n",
      "Precision :  0.42782478845196614\n",
      "Accuracy :  0.8130271084337349\n",
      "Sensitivity :  0.7979256394480091\n",
      "Specificity :  0.9033105622700999\n",
      "Validation at Epoch 37 , AUROC: 0.9136365460807686 , AUPRC: 0.60268823878385 , F1: 0.597828087612737\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 38 iteration 0 with loss 0.17168719\n",
      "Training at Epoch 38 iteration 100 with loss 0.08409122\n",
      "Training at Epoch 38 iteration 200 with loss 0.30940405\n",
      "Training at Epoch 38 iteration 300 with loss 0.28998053\n",
      "Training at Epoch 38 iteration 400 with loss 0.17720369\n",
      "Training at Epoch 38 iteration 500 with loss 0.10070393\n",
      "Training at Epoch 38 iteration 600 with loss 0.104349434\n",
      "Training at Epoch 38 iteration 700 with loss 0.13716705\n",
      "optimal threshold: 0.3629407286643982\n",
      "AUROC:0.9151213458748477\n",
      "AUPRC: 0.6061262348980168\n",
      "Confusion Matrix : \n",
      " [[8810 2566]\n",
      " [ 158 1746]]\n",
      "Recall :  0.917016806722689\n",
      "Precision :  0.4049165120593692\n",
      "Accuracy :  0.7948795180722892\n",
      "Sensitivity :  0.77443741209564\n",
      "Specificity :  0.917016806722689\n",
      "Validation at Epoch 38 , AUROC: 0.9151213458748477 , AUPRC: 0.6061262348980168 , F1: 0.5850269237450061\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 39 iteration 0 with loss 0.1751421\n",
      "Training at Epoch 39 iteration 100 with loss 0.16079381\n",
      "Training at Epoch 39 iteration 200 with loss 0.13620313\n",
      "Training at Epoch 39 iteration 300 with loss 0.21045798\n",
      "Training at Epoch 39 iteration 400 with loss 0.13983428\n",
      "Training at Epoch 39 iteration 500 with loss 0.23556246\n",
      "Training at Epoch 39 iteration 600 with loss 0.24318141\n",
      "Training at Epoch 39 iteration 700 with loss 0.2539115\n",
      "optimal threshold: 0.46905699372291565\n",
      "AUROC:0.9154095512150008\n",
      "AUPRC: 0.6044002496710622\n",
      "Confusion Matrix : \n",
      " [[8930 2446]\n",
      " [ 171 1733]]\n",
      "Recall :  0.9101890756302521\n",
      "Precision :  0.4146925101698971\n",
      "Accuracy :  0.8029367469879518\n",
      "Sensitivity :  0.784985935302391\n",
      "Specificity :  0.9101890756302521\n",
      "Validation at Epoch 39 , AUROC: 0.9154095512150008 , AUPRC: 0.6044002496710622 , F1: 0.5725280240923539\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 40 iteration 0 with loss 0.11817467\n",
      "Training at Epoch 40 iteration 100 with loss 0.31318253\n",
      "Training at Epoch 40 iteration 200 with loss 0.24785385\n",
      "Training at Epoch 40 iteration 300 with loss 0.19847886\n",
      "Training at Epoch 40 iteration 400 with loss 0.2393257\n",
      "Training at Epoch 40 iteration 500 with loss 0.48297238\n",
      "Training at Epoch 40 iteration 600 with loss 0.11668785\n",
      "Training at Epoch 40 iteration 700 with loss 0.13986191\n",
      "optimal threshold: 0.39785751700401306\n",
      "AUROC:0.9122583830473118\n",
      "AUPRC: 0.5943822939008725\n",
      "Confusion Matrix : \n",
      " [[8853 2523]\n",
      " [ 150 1754]]\n",
      "Recall :  0.9212184873949579\n",
      "Precision :  0.41010053776011224\n",
      "Accuracy :  0.7987198795180723\n",
      "Sensitivity :  0.7782172995780591\n",
      "Specificity :  0.9212184873949579\n",
      "Validation at Epoch 40 , AUROC: 0.9122583830473118 , AUPRC: 0.5943822939008725 , F1: 0.5816882232811437\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 41 iteration 0 with loss 0.13606364\n",
      "Training at Epoch 41 iteration 100 with loss 0.105128\n",
      "Training at Epoch 41 iteration 200 with loss 0.1020614\n",
      "Training at Epoch 41 iteration 300 with loss 0.23711663\n",
      "Training at Epoch 41 iteration 400 with loss 0.19557843\n",
      "Training at Epoch 41 iteration 500 with loss 0.17180447\n",
      "Training at Epoch 41 iteration 600 with loss 0.09500571\n",
      "Training at Epoch 41 iteration 700 with loss 0.35678384\n",
      "optimal threshold: 0.4147492051124573\n",
      "AUROC:0.9136768796389864\n",
      "AUPRC: 0.5944112938153195\n",
      "Confusion Matrix : \n",
      " [[8820 2556]\n",
      " [ 152 1752]]\n",
      "Recall :  0.9201680672268907\n",
      "Precision :  0.40668523676880225\n",
      "Accuracy :  0.7960843373493975\n",
      "Sensitivity :  0.7753164556962026\n",
      "Specificity :  0.9201680672268907\n",
      "Validation at Epoch 41 , AUROC: 0.9136768796389864 , AUPRC: 0.5944112938153195 , F1: 0.5772398722474366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 42 iteration 0 with loss 0.18325418\n",
      "Training at Epoch 42 iteration 100 with loss 0.25437135\n",
      "Training at Epoch 42 iteration 200 with loss 0.15383065\n",
      "Training at Epoch 42 iteration 300 with loss 0.07412135\n",
      "Training at Epoch 42 iteration 400 with loss 0.24412107\n",
      "Training at Epoch 42 iteration 500 with loss 0.30143356\n",
      "Training at Epoch 42 iteration 600 with loss 0.24081221\n",
      "Training at Epoch 42 iteration 700 with loss 0.27579886\n",
      "optimal threshold: 0.3722972571849823\n",
      "AUROC:0.9143298834565472\n",
      "AUPRC: 0.6041905512417161\n",
      "Confusion Matrix : \n",
      " [[8780 2596]\n",
      " [ 150 1754]]\n",
      "Recall :  0.9212184873949579\n",
      "Precision :  0.4032183908045977\n",
      "Accuracy :  0.793222891566265\n",
      "Sensitivity :  0.7718002812939522\n",
      "Specificity :  0.9212184873949579\n",
      "Validation at Epoch 42 , AUROC: 0.9143298834565472 , AUPRC: 0.6041905512417161 , F1: 0.5804686163844707\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 43 iteration 0 with loss 0.18983425\n",
      "Training at Epoch 43 iteration 100 with loss 0.21492381\n",
      "Training at Epoch 43 iteration 200 with loss 0.06634574\n",
      "Training at Epoch 43 iteration 300 with loss 0.23099649\n",
      "Training at Epoch 43 iteration 400 with loss 0.24737181\n",
      "Training at Epoch 43 iteration 500 with loss 0.15186825\n",
      "Training at Epoch 43 iteration 600 with loss 0.25750753\n",
      "Training at Epoch 43 iteration 700 with loss 0.2888925\n",
      "optimal threshold: 0.2929115295410156\n",
      "AUROC:0.9128518242911943\n",
      "AUPRC: 0.5890656463649934\n",
      "Confusion Matrix : \n",
      " [[8671 2704]\n",
      " [ 131 1774]]\n",
      "Recall :  0.9312335958005249\n",
      "Precision :  0.396158999553372\n",
      "Accuracy :  0.7865210843373494\n",
      "Sensitivity :  0.7622857142857142\n",
      "Specificity :  0.9312335958005249\n",
      "Validation at Epoch 43 , AUROC: 0.9128518242911943 , AUPRC: 0.5890656463649934 , F1: 0.5906862745098039\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 44 iteration 0 with loss 0.26957297\n",
      "Training at Epoch 44 iteration 100 with loss 0.09043445\n",
      "Training at Epoch 44 iteration 200 with loss 0.11769836\n",
      "Training at Epoch 44 iteration 300 with loss 0.19590393\n",
      "Training at Epoch 44 iteration 400 with loss 0.198528\n",
      "Training at Epoch 44 iteration 500 with loss 0.29244307\n",
      "Training at Epoch 44 iteration 600 with loss 0.09339993\n",
      "Training at Epoch 44 iteration 700 with loss 0.2211596\n",
      "optimal threshold: 0.36950165033340454\n",
      "AUROC:0.9129566086719499\n",
      "AUPRC: 0.5966977314660984\n",
      "Confusion Matrix : \n",
      " [[8747 2629]\n",
      " [ 153 1751]]\n",
      "Recall :  0.9196428571428571\n",
      "Precision :  0.3997716894977169\n",
      "Accuracy :  0.7905120481927711\n",
      "Sensitivity :  0.7688994374120957\n",
      "Specificity :  0.9196428571428571\n",
      "Validation at Epoch 44 , AUROC: 0.9129566086719499 , AUPRC: 0.5966977314660984 , F1: 0.5789563148053714\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 45 iteration 0 with loss 0.23409645\n",
      "Training at Epoch 45 iteration 100 with loss 0.18782994\n",
      "Training at Epoch 45 iteration 200 with loss 0.22795075\n",
      "Training at Epoch 45 iteration 300 with loss 0.10408983\n",
      "Training at Epoch 45 iteration 400 with loss 0.1828562\n",
      "Training at Epoch 45 iteration 500 with loss 0.18536589\n",
      "Training at Epoch 45 iteration 600 with loss 0.19458331\n",
      "Training at Epoch 45 iteration 700 with loss 0.55072665\n",
      "optimal threshold: 0.43174993991851807\n",
      "AUROC:0.9092854547027776\n",
      "AUPRC: 0.5916134419503746\n",
      "Confusion Matrix : \n",
      " [[8787 2588]\n",
      " [ 159 1746]]\n",
      "Recall :  0.9165354330708662\n",
      "Precision :  0.4028610982925704\n",
      "Accuracy :  0.7931475903614458\n",
      "Sensitivity :  0.7724835164835164\n",
      "Specificity :  0.9165354330708662\n",
      "Validation at Epoch 45 , AUROC: 0.9092854547027776 , AUPRC: 0.5916134419503746 , F1: 0.5688712083540527\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 46 iteration 0 with loss 0.123466864\n",
      "Training at Epoch 46 iteration 100 with loss 0.092747256\n",
      "Training at Epoch 46 iteration 200 with loss 0.14076228\n",
      "Training at Epoch 46 iteration 300 with loss 0.115617186\n",
      "Training at Epoch 46 iteration 400 with loss 0.40264568\n",
      "Training at Epoch 46 iteration 500 with loss 0.15432127\n",
      "Training at Epoch 46 iteration 600 with loss 0.18320692\n",
      "Training at Epoch 46 iteration 700 with loss 0.08309198\n",
      "optimal threshold: 0.41794851422309875\n",
      "AUROC:0.9113315973656618\n",
      "AUPRC: 0.5980930526568535\n",
      "Confusion Matrix : \n",
      " [[9015 2363]\n",
      " [ 172 1730]]\n",
      "Recall :  0.9095688748685594\n",
      "Precision :  0.42267285609577326\n",
      "Accuracy :  0.8091114457831325\n",
      "Sensitivity :  0.7923185094041132\n",
      "Specificity :  0.9095688748685594\n",
      "Validation at Epoch 46 , AUROC: 0.9113315973656618 , AUPRC: 0.5980930526568535 , F1: 0.5875588081547307\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 47 iteration 0 with loss 0.067153685\n",
      "Training at Epoch 47 iteration 100 with loss 0.20984688\n",
      "Training at Epoch 47 iteration 200 with loss 0.18631352\n",
      "Training at Epoch 47 iteration 300 with loss 0.10418084\n",
      "Training at Epoch 47 iteration 400 with loss 0.21360675\n",
      "Training at Epoch 47 iteration 500 with loss 0.046974234\n",
      "Training at Epoch 47 iteration 600 with loss 0.14390871\n",
      "Training at Epoch 47 iteration 700 with loss 0.34669095\n",
      "optimal threshold: 0.3813945949077606\n",
      "AUROC:0.915215082785188\n",
      "AUPRC: 0.5979344495795222\n",
      "Confusion Matrix : \n",
      " [[9055 2322]\n",
      " [ 170 1733]]\n",
      "Recall :  0.9106673673147662\n",
      "Precision :  0.4273736128236745\n",
      "Accuracy :  0.8123493975903614\n",
      "Sensitivity :  0.7959040168761536\n",
      "Specificity :  0.9106673673147662\n",
      "Validation at Epoch 47 , AUROC: 0.915215082785188 , AUPRC: 0.5979344495795222 , F1: 0.6009992862241256\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 48 iteration 0 with loss 0.19133948\n",
      "Training at Epoch 48 iteration 100 with loss 0.24119827\n",
      "Training at Epoch 48 iteration 200 with loss 0.08307915\n",
      "Training at Epoch 48 iteration 300 with loss 0.16204688\n",
      "Training at Epoch 48 iteration 400 with loss 0.1502855\n",
      "Training at Epoch 48 iteration 500 with loss 0.072999984\n",
      "Training at Epoch 48 iteration 600 with loss 0.18322676\n",
      "Training at Epoch 48 iteration 700 with loss 0.5355164\n",
      "optimal threshold: 0.37424883246421814\n",
      "AUROC:0.9130123225609812\n",
      "AUPRC: 0.5937065012648112\n",
      "Confusion Matrix : \n",
      " [[8980 2398]\n",
      " [ 172 1730]]\n",
      "Recall :  0.9095688748685594\n",
      "Precision :  0.41908914728682173\n",
      "Accuracy :  0.8064759036144579\n",
      "Sensitivity :  0.7892423976094217\n",
      "Specificity :  0.9095688748685594\n",
      "Validation at Epoch 48 , AUROC: 0.9130123225609812 , AUPRC: 0.5937065012648112 , F1: 0.5907322249734701\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 49 iteration 0 with loss 0.07250925\n",
      "Training at Epoch 49 iteration 100 with loss 0.1461625\n",
      "Training at Epoch 49 iteration 200 with loss 0.13970327\n",
      "Training at Epoch 49 iteration 300 with loss 0.11716861\n",
      "Training at Epoch 49 iteration 400 with loss 0.23560372\n",
      "Training at Epoch 49 iteration 500 with loss 0.028375529\n",
      "Training at Epoch 49 iteration 600 with loss 0.10837711\n",
      "Training at Epoch 49 iteration 700 with loss 0.51311153\n",
      "optimal threshold: 0.42201104760169983\n",
      "AUROC:0.9121285160558699\n",
      "AUPRC: 0.5850085440825392\n",
      "Confusion Matrix : \n",
      " [[8857 2520]\n",
      " [ 161 1742]]\n",
      "Recall :  0.9153967419863374\n",
      "Precision :  0.4087282965743782\n",
      "Accuracy :  0.798117469879518\n",
      "Sensitivity :  0.7785004834314846\n",
      "Specificity :  0.9153967419863374\n",
      "Validation at Epoch 49 , AUROC: 0.9121285160558699 , AUPRC: 0.5850085440825392 , F1: 0.5775905644481887\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 50 iteration 0 with loss 0.1802958\n",
      "Training at Epoch 50 iteration 100 with loss 0.103057116\n",
      "Training at Epoch 50 iteration 200 with loss 0.080528125\n",
      "Training at Epoch 50 iteration 300 with loss 0.28061414\n",
      "Training at Epoch 50 iteration 400 with loss 0.0994928\n",
      "Training at Epoch 50 iteration 500 with loss 0.1732974\n",
      "Training at Epoch 50 iteration 600 with loss 0.12155653\n",
      "Training at Epoch 50 iteration 700 with loss 0.28349686\n",
      "optimal threshold: 0.4289315640926361\n",
      "AUROC:0.9153101323872976\n",
      "AUPRC: 0.6132785395147138\n",
      "Confusion Matrix : \n",
      " [[9199 2176]\n",
      " [ 198 1707]]\n",
      "Recall :  0.8960629921259843\n",
      "Precision :  0.4396085500901365\n",
      "Accuracy :  0.8212349397590362\n",
      "Sensitivity :  0.8087032967032967\n",
      "Specificity :  0.8960629921259843\n",
      "Validation at Epoch 50 , AUROC: 0.9153101323872976 , AUPRC: 0.6132785395147138 , F1: 0.5954334641455583\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 51 iteration 0 with loss 0.17485061\n",
      "Training at Epoch 51 iteration 100 with loss 0.18615788\n",
      "Training at Epoch 51 iteration 200 with loss 0.10134514\n",
      "Training at Epoch 51 iteration 300 with loss 0.08519085\n",
      "Training at Epoch 51 iteration 400 with loss 0.12491052\n",
      "Training at Epoch 51 iteration 500 with loss 0.04977739\n",
      "Training at Epoch 51 iteration 600 with loss 0.2936322\n",
      "Training at Epoch 51 iteration 700 with loss 0.09835971\n",
      "optimal threshold: 0.40422412753105164\n",
      "AUROC:0.916602331201468\n",
      "AUPRC: 0.6223781986933977\n",
      "Confusion Matrix : \n",
      " [[8974 2402]\n",
      " [ 175 1729]]\n",
      "Recall :  0.9080882352941176\n",
      "Precision :  0.4185427257322682\n",
      "Accuracy :  0.8059487951807229\n",
      "Sensitivity :  0.7888537271448663\n",
      "Specificity :  0.9080882352941176\n",
      "Validation at Epoch 51 , AUROC: 0.916602331201468 , AUPRC: 0.6223781986933977 , F1: 0.5865883166263395\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 52 iteration 0 with loss 0.09331762\n",
      "Training at Epoch 52 iteration 100 with loss 0.12110766\n",
      "Training at Epoch 52 iteration 200 with loss 0.1290823\n",
      "Training at Epoch 52 iteration 300 with loss 0.10505031\n",
      "Training at Epoch 52 iteration 400 with loss 0.099622376\n",
      "Training at Epoch 52 iteration 500 with loss 0.046374045\n",
      "Training at Epoch 52 iteration 600 with loss 0.30389506\n",
      "Training at Epoch 52 iteration 700 with loss 0.09904933\n",
      "optimal threshold: 0.42877933382987976\n",
      "AUROC:0.9143774598446972\n",
      "AUPRC: 0.5742850591077125\n",
      "Confusion Matrix : \n",
      " [[9009 2367]\n",
      " [ 175 1729]]\n",
      "Recall :  0.9080882352941176\n",
      "Precision :  0.422119140625\n",
      "Accuracy :  0.8085843373493976\n",
      "Sensitivity :  0.7919303797468354\n",
      "Specificity :  0.9080882352941176\n",
      "Validation at Epoch 52 , AUROC: 0.9143774598446972 , AUPRC: 0.5742850591077125 , F1: 0.5856027753686036\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 53 iteration 0 with loss 0.3701138\n",
      "Training at Epoch 53 iteration 100 with loss 0.19154371\n",
      "Training at Epoch 53 iteration 200 with loss 0.1576529\n",
      "Training at Epoch 53 iteration 300 with loss 0.20858274\n",
      "Training at Epoch 53 iteration 400 with loss 0.11560878\n",
      "Training at Epoch 53 iteration 500 with loss 0.049105346\n",
      "Training at Epoch 53 iteration 600 with loss 0.050732296\n",
      "Training at Epoch 53 iteration 700 with loss 0.0466296\n",
      "optimal threshold: 0.37051668763160706\n",
      "AUROC:0.915411928880202\n",
      "AUPRC: 0.6083920965994212\n",
      "Confusion Matrix : \n",
      " [[9055 2321]\n",
      " [ 182 1722]]\n",
      "Recall :  0.9044117647058824\n",
      "Precision :  0.42592134553549343\n",
      "Accuracy :  0.8115210843373494\n",
      "Sensitivity :  0.7959739803094233\n",
      "Specificity :  0.9044117647058824\n",
      "Validation at Epoch 53 , AUROC: 0.915411928880202 , AUPRC: 0.6083920965994212 , F1: 0.5932776098168238\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 54 iteration 0 with loss 0.15472028\n",
      "Training at Epoch 54 iteration 100 with loss 0.16763501\n",
      "Training at Epoch 54 iteration 200 with loss 0.14926547\n",
      "Training at Epoch 54 iteration 300 with loss 0.07361464\n",
      "Training at Epoch 54 iteration 400 with loss 0.39740837\n",
      "Training at Epoch 54 iteration 500 with loss 0.22715084\n",
      "Training at Epoch 54 iteration 600 with loss 0.11286676\n",
      "Training at Epoch 54 iteration 700 with loss 0.31151676\n",
      "optimal threshold: 0.23147664964199066\n",
      "AUROC:0.9131868035980215\n",
      "AUPRC: 0.5909939250294628\n",
      "Confusion Matrix : \n",
      " [[8733 2643]\n",
      " [ 143 1761]]\n",
      "Recall :  0.9248949579831933\n",
      "Precision :  0.39986376021798364\n",
      "Accuracy :  0.790210843373494\n",
      "Sensitivity :  0.767668776371308\n",
      "Specificity :  0.9248949579831933\n",
      "Validation at Epoch 54 , AUROC: 0.9131868035980215 , AUPRC: 0.5909939250294628 , F1: 0.6004374772147284\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 55 iteration 0 with loss 0.29569513\n",
      "Training at Epoch 55 iteration 100 with loss 0.19725063\n",
      "Training at Epoch 55 iteration 200 with loss 0.23720658\n",
      "Training at Epoch 55 iteration 300 with loss 0.14140104\n",
      "Training at Epoch 55 iteration 400 with loss 0.09084603\n",
      "Training at Epoch 55 iteration 500 with loss 0.08100736\n",
      "Training at Epoch 55 iteration 600 with loss 0.049216263\n",
      "Training at Epoch 55 iteration 700 with loss 0.18524584\n",
      "optimal threshold: 0.3588006794452667\n",
      "AUROC:0.9149158509659138\n",
      "AUPRC: 0.5947952168267245\n",
      "Confusion Matrix : \n",
      " [[8896 2480]\n",
      " [ 166 1738]]\n",
      "Recall :  0.9128151260504201\n",
      "Precision :  0.4120436225699384\n",
      "Accuracy :  0.8007530120481928\n",
      "Sensitivity :  0.7819971870604782\n",
      "Specificity :  0.9128151260504201\n",
      "Validation at Epoch 55 , AUROC: 0.9149158509659138 , AUPRC: 0.5947952168267245 , F1: 0.5854678515420805\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 56 iteration 0 with loss 0.2675557\n",
      "Training at Epoch 56 iteration 100 with loss 0.2818756\n",
      "Training at Epoch 56 iteration 200 with loss 0.27903017\n",
      "Training at Epoch 56 iteration 300 with loss 0.036803193\n",
      "Training at Epoch 56 iteration 400 with loss 0.08071047\n",
      "Training at Epoch 56 iteration 500 with loss 0.25748122\n",
      "Training at Epoch 56 iteration 600 with loss 0.07865976\n",
      "Training at Epoch 56 iteration 700 with loss 0.06739135\n",
      "optimal threshold: 0.3695160150527954\n",
      "AUROC:0.9149747050849413\n",
      "AUPRC: 0.5869009580606384\n",
      "Confusion Matrix : \n",
      " [[9043 2332]\n",
      " [ 170 1735]]\n",
      "Recall :  0.910761154855643\n",
      "Precision :  0.4266043766904352\n",
      "Accuracy :  0.8115963855421687\n",
      "Sensitivity :  0.794989010989011\n",
      "Specificity :  0.910761154855643\n",
      "Validation at Epoch 56 , AUROC: 0.9149747050849413 , AUPRC: 0.5869009580606384 , F1: 0.5966772711205373\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 57 iteration 0 with loss 0.10816026\n",
      "Training at Epoch 57 iteration 100 with loss 0.07456422\n",
      "Training at Epoch 57 iteration 200 with loss 0.11948756\n",
      "Training at Epoch 57 iteration 300 with loss 0.12926261\n",
      "Training at Epoch 57 iteration 400 with loss 0.2896135\n",
      "Training at Epoch 57 iteration 500 with loss 0.11930359\n",
      "Training at Epoch 57 iteration 600 with loss 0.20346381\n",
      "Training at Epoch 57 iteration 700 with loss 0.11997069\n",
      "optimal threshold: 0.24839095771312714\n",
      "AUROC:0.9124019979094181\n",
      "AUPRC: 0.5778807136933722\n",
      "Confusion Matrix : \n",
      " [[8557 2820]\n",
      " [ 123 1780]]\n",
      "Recall :  0.9353652128218602\n",
      "Precision :  0.3869565217391304\n",
      "Accuracy :  0.7783885542168675\n",
      "Sensitivity :  0.7521314933638041\n",
      "Specificity :  0.9353652128218602\n",
      "Validation at Epoch 57 , AUROC: 0.9124019979094181 , AUPRC: 0.5778807136933722 , F1: 0.5824403733148981\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 58 iteration 0 with loss 0.142179\n",
      "Training at Epoch 58 iteration 100 with loss 0.14657699\n",
      "Training at Epoch 58 iteration 200 with loss 0.13254341\n",
      "Training at Epoch 58 iteration 300 with loss 0.22550926\n",
      "Training at Epoch 58 iteration 400 with loss 0.080046356\n",
      "Training at Epoch 58 iteration 500 with loss 0.07070409\n",
      "Training at Epoch 58 iteration 600 with loss 0.15735891\n",
      "Training at Epoch 58 iteration 700 with loss 0.044210162\n",
      "optimal threshold: 0.38025254011154175\n",
      "AUROC:0.9170705892532665\n",
      "AUPRC: 0.6085239080837868\n",
      "Confusion Matrix : \n",
      " [[9080 2295]\n",
      " [ 184 1721]]\n",
      "Recall :  0.9034120734908136\n",
      "Precision :  0.4285358565737052\n",
      "Accuracy :  0.813328313253012\n",
      "Sensitivity :  0.7982417582417582\n",
      "Specificity :  0.9034120734908136\n",
      "Validation at Epoch 58 , AUROC: 0.9170705892532665 , AUPRC: 0.6085239080837868 , F1: 0.5947955390334573\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 59 iteration 0 with loss 0.156956\n",
      "Training at Epoch 59 iteration 100 with loss 0.11946809\n",
      "Training at Epoch 59 iteration 200 with loss 0.083120905\n",
      "Training at Epoch 59 iteration 300 with loss 0.13318056\n",
      "Training at Epoch 59 iteration 400 with loss 0.3096341\n",
      "Training at Epoch 59 iteration 500 with loss 0.20912553\n",
      "Training at Epoch 59 iteration 600 with loss 0.11872391\n",
      "Training at Epoch 59 iteration 700 with loss 0.48538917\n",
      "optimal threshold: 0.34922072291374207\n",
      "AUROC:0.9140559917962958\n",
      "AUPRC: 0.6066181511074121\n",
      "Confusion Matrix : \n",
      " [[8901 2477]\n",
      " [ 176 1726]]\n",
      "Recall :  0.907465825446898\n",
      "Precision :  0.41065905305734\n",
      "Accuracy :  0.8002259036144578\n",
      "Sensitivity :  0.7822991738442608\n",
      "Specificity :  0.907465825446898\n",
      "Validation at Epoch 59 , AUROC: 0.9140559917962958 , AUPRC: 0.6066181511074121 , F1: 0.5828951980371538\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 60 iteration 0 with loss 0.08073101\n",
      "Training at Epoch 60 iteration 100 with loss 0.087035686\n",
      "Training at Epoch 60 iteration 200 with loss 0.22813526\n",
      "Training at Epoch 60 iteration 300 with loss 0.11259585\n",
      "Training at Epoch 60 iteration 400 with loss 0.0290258\n",
      "Training at Epoch 60 iteration 500 with loss 0.093504034\n",
      "Training at Epoch 60 iteration 600 with loss 0.030844532\n",
      "Training at Epoch 60 iteration 700 with loss 0.08409696\n",
      "optimal threshold: 0.3420063853263855\n",
      "AUROC:0.914883717859507\n",
      "AUPRC: 0.5969994915310236\n",
      "Confusion Matrix : \n",
      " [[8657 2719]\n",
      " [ 138 1766]]\n",
      "Recall :  0.9275210084033614\n",
      "Precision :  0.39375696767001117\n",
      "Accuracy :  0.7848644578313253\n",
      "Sensitivity :  0.7609880450070323\n",
      "Specificity :  0.9275210084033614\n",
      "Validation at Epoch 60 , AUROC: 0.914883717859507 , AUPRC: 0.5969994915310236 , F1: 0.5720976915356306\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 61 iteration 0 with loss 0.115327165\n",
      "Training at Epoch 61 iteration 100 with loss 0.14763609\n",
      "Training at Epoch 61 iteration 200 with loss 0.14609008\n",
      "Training at Epoch 61 iteration 300 with loss 0.14465968\n",
      "Training at Epoch 61 iteration 400 with loss 0.14742514\n",
      "Training at Epoch 61 iteration 500 with loss 0.19569716\n",
      "Training at Epoch 61 iteration 600 with loss 0.13266294\n",
      "Training at Epoch 61 iteration 700 with loss 0.042854022\n",
      "optimal threshold: 0.5104824304580688\n",
      "AUROC:0.91590272052914\n",
      "AUPRC: 0.6069523979970614\n",
      "Confusion Matrix : \n",
      " [[9014 2362]\n",
      " [ 172 1732]]\n",
      "Recall :  0.9096638655462185\n",
      "Precision :  0.4230581338544211\n",
      "Accuracy :  0.8091867469879518\n",
      "Sensitivity :  0.7923699015471167\n",
      "Specificity :  0.9096638655462185\n",
      "Validation at Epoch 61 , AUROC: 0.91590272052914 , AUPRC: 0.6069523979970614 , F1: 0.5752697095435685\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 62 iteration 0 with loss 0.27195168\n",
      "Training at Epoch 62 iteration 100 with loss 0.15493003\n",
      "Training at Epoch 62 iteration 200 with loss 0.10551827\n",
      "Training at Epoch 62 iteration 300 with loss 0.065673724\n",
      "Training at Epoch 62 iteration 400 with loss 0.13776456\n",
      "Training at Epoch 62 iteration 500 with loss 0.1821544\n",
      "Training at Epoch 62 iteration 600 with loss 0.3177464\n",
      "Training at Epoch 62 iteration 700 with loss 0.03495381\n",
      "optimal threshold: 0.40456390380859375\n",
      "AUROC:0.9150511188137899\n",
      "AUPRC: 0.5986608575258021\n",
      "Confusion Matrix : \n",
      " [[8972 2406]\n",
      " [ 188 1714]]\n",
      "Recall :  0.9011566771819137\n",
      "Precision :  0.4160194174757282\n",
      "Accuracy :  0.8046686746987952\n",
      "Sensitivity :  0.7885392863420636\n",
      "Specificity :  0.9011566771819137\n",
      "Validation at Epoch 62 , AUROC: 0.9150511188137899 , AUPRC: 0.5986608575258021 , F1: 0.5808019441069259\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 63 iteration 0 with loss 0.08314149\n",
      "Training at Epoch 63 iteration 100 with loss 0.18394662\n",
      "Training at Epoch 63 iteration 200 with loss 0.05970665\n",
      "Training at Epoch 63 iteration 300 with loss 0.1306719\n",
      "Training at Epoch 63 iteration 400 with loss 0.16791508\n",
      "Training at Epoch 63 iteration 500 with loss 0.17764677\n",
      "Training at Epoch 63 iteration 600 with loss 0.13537797\n",
      "Training at Epoch 63 iteration 700 with loss 0.14199895\n",
      "optimal threshold: 0.23032313585281372\n",
      "AUROC:0.9147610271592285\n",
      "AUPRC: 0.5980876242938657\n",
      "Confusion Matrix : \n",
      " [[8889 2488]\n",
      " [ 160 1743]]\n",
      "Recall :  0.9159222280609564\n",
      "Precision :  0.41195934767194514\n",
      "Accuracy :  0.8006024096385542\n",
      "Sensitivity :  0.7813131757053705\n",
      "Specificity :  0.9159222280609564\n",
      "Validation at Epoch 63 , AUROC: 0.9147610271592285 , AUPRC: 0.5980876242938657 , F1: 0.6056026538886841\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 64 iteration 0 with loss 0.03935249\n",
      "Training at Epoch 64 iteration 100 with loss 0.117124565\n",
      "Training at Epoch 64 iteration 200 with loss 0.08174132\n",
      "Training at Epoch 64 iteration 300 with loss 0.1227389\n",
      "Training at Epoch 64 iteration 400 with loss 0.06874637\n",
      "Training at Epoch 64 iteration 500 with loss 0.09608402\n",
      "Training at Epoch 64 iteration 600 with loss 0.14689836\n",
      "Training at Epoch 64 iteration 700 with loss 0.085859574\n",
      "optimal threshold: 0.42906978726387024\n",
      "AUROC:0.9115634695678807\n",
      "AUPRC: 0.5952535372593576\n",
      "Confusion Matrix : \n",
      " [[8522 2855]\n",
      " [ 136 1767]]\n",
      "Recall :  0.928533893851813\n",
      "Precision :  0.3823020337516227\n",
      "Accuracy :  0.7747740963855422\n",
      "Sensitivity :  0.7490551111892415\n",
      "Specificity :  0.928533893851813\n",
      "Validation at Epoch 64 , AUROC: 0.9115634695678807 , AUPRC: 0.5952535372593576 , F1: 0.548870993210169\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 65 iteration 0 with loss 0.17225255\n",
      "Training at Epoch 65 iteration 100 with loss 0.15498962\n",
      "Training at Epoch 65 iteration 200 with loss 0.0751078\n",
      "Training at Epoch 65 iteration 300 with loss 0.14203952\n",
      "Training at Epoch 65 iteration 400 with loss 0.033154123\n",
      "Training at Epoch 65 iteration 500 with loss 0.1394022\n",
      "Training at Epoch 65 iteration 600 with loss 0.20306039\n",
      "Training at Epoch 65 iteration 700 with loss 0.44357967\n",
      "optimal threshold: 0.44640210270881653\n",
      "AUROC:0.913079588128407\n",
      "AUPRC: 0.6012276856868545\n",
      "Confusion Matrix : \n",
      " [[9083 2292]\n",
      " [ 191 1714]]\n",
      "Recall :  0.899737532808399\n",
      "Precision :  0.4278582126809785\n",
      "Accuracy :  0.8130271084337349\n",
      "Sensitivity :  0.7985054945054945\n",
      "Specificity :  0.899737532808399\n",
      "Validation at Epoch 65 , AUROC: 0.913079588128407 , AUPRC: 0.6012276856868545 , F1: 0.5844133402453775\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 66 iteration 0 with loss 0.033188753\n",
      "Training at Epoch 66 iteration 100 with loss 0.07451683\n",
      "Training at Epoch 66 iteration 200 with loss 0.13047679\n",
      "Training at Epoch 66 iteration 300 with loss 0.18224105\n",
      "Training at Epoch 66 iteration 400 with loss 0.24311528\n",
      "Training at Epoch 66 iteration 500 with loss 0.21158358\n",
      "Training at Epoch 66 iteration 600 with loss 0.061107583\n",
      "Training at Epoch 66 iteration 700 with loss 0.10058914\n",
      "optimal threshold: 0.30804285407066345\n",
      "AUROC:0.9154617518963977\n",
      "AUPRC: 0.6052059439312528\n",
      "Confusion Matrix : \n",
      " [[8900 2475]\n",
      " [ 158 1747]]\n",
      "Recall :  0.9170603674540683\n",
      "Precision :  0.41378493604926575\n",
      "Accuracy :  0.8017319277108433\n",
      "Sensitivity :  0.7824175824175824\n",
      "Specificity :  0.9170603674540683\n",
      "Validation at Epoch 66 , AUROC: 0.9154617518963977 , AUPRC: 0.6052059439312528 , F1: 0.5913692061800746\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 67 iteration 0 with loss 0.22957216\n",
      "Training at Epoch 67 iteration 100 with loss 0.45841694\n",
      "Training at Epoch 67 iteration 200 with loss 0.037662555\n",
      "Training at Epoch 67 iteration 300 with loss 0.072008416\n",
      "Training at Epoch 67 iteration 400 with loss 0.1776635\n",
      "Training at Epoch 67 iteration 500 with loss 0.065154\n",
      "Training at Epoch 67 iteration 600 with loss 0.14962319\n",
      "Training at Epoch 67 iteration 700 with loss 0.18739508\n",
      "optimal threshold: 0.3609423041343689\n",
      "AUROC:0.9152485394210428\n",
      "AUPRC: 0.6141454406852983\n",
      "Confusion Matrix : \n",
      " [[9055 2321]\n",
      " [ 189 1715]]\n",
      "Recall :  0.9007352941176471\n",
      "Precision :  0.4249256689791873\n",
      "Accuracy :  0.8109939759036144\n",
      "Sensitivity :  0.7959739803094233\n",
      "Specificity :  0.9007352941176471\n",
      "Validation at Epoch 67 , AUROC: 0.9152485394210428 , AUPRC: 0.6141454406852983 , F1: 0.5899255054984037\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 68 iteration 0 with loss 0.07439102\n",
      "Training at Epoch 68 iteration 100 with loss 0.11158619\n",
      "Training at Epoch 68 iteration 200 with loss 0.16707742\n",
      "Training at Epoch 68 iteration 300 with loss 0.1938579\n",
      "Training at Epoch 68 iteration 400 with loss 0.17618422\n",
      "Training at Epoch 68 iteration 500 with loss 0.14534414\n",
      "Training at Epoch 68 iteration 600 with loss 0.05061315\n",
      "Training at Epoch 68 iteration 700 with loss 0.14154921\n",
      "optimal threshold: 0.4197181463241577\n",
      "AUROC:0.9095203557550786\n",
      "AUPRC: 0.6051907076103692\n",
      "Confusion Matrix : \n",
      " [[8842 2536]\n",
      " [ 171 1731]]\n",
      "Recall :  0.9100946372239748\n",
      "Precision :  0.4056714319193813\n",
      "Accuracy :  0.7961596385542169\n",
      "Sensitivity :  0.7771137282474951\n",
      "Specificity :  0.9100946372239748\n",
      "Validation at Epoch 68 , AUROC: 0.9095203557550786 , AUPRC: 0.6051907076103692 , F1: 0.5708075572646714\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 69 iteration 0 with loss 0.061243236\n",
      "Training at Epoch 69 iteration 100 with loss 0.13397813\n",
      "Training at Epoch 69 iteration 200 with loss 0.16671973\n",
      "Training at Epoch 69 iteration 300 with loss 0.07066485\n",
      "Training at Epoch 69 iteration 400 with loss 0.06414911\n",
      "Training at Epoch 69 iteration 500 with loss 0.04884406\n",
      "Training at Epoch 69 iteration 600 with loss 0.11298111\n",
      "Training at Epoch 69 iteration 700 with loss 0.02081564\n",
      "optimal threshold: 0.19986185431480408\n",
      "AUROC:0.9130548316372962\n",
      "AUPRC: 0.600328377857212\n",
      "Confusion Matrix : \n",
      " [[8780 2596]\n",
      " [ 151 1753]]\n",
      "Recall :  0.9206932773109243\n",
      "Precision :  0.40308116808461714\n",
      "Accuracy :  0.7931475903614458\n",
      "Sensitivity :  0.7718002812939522\n",
      "Specificity :  0.9206932773109243\n",
      "Validation at Epoch 69 , AUROC: 0.9130548316372962 , AUPRC: 0.600328377857212 , F1: 0.5970475669764899\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 70 iteration 0 with loss 0.12180461\n",
      "Training at Epoch 70 iteration 100 with loss 0.06618625\n",
      "Training at Epoch 70 iteration 200 with loss 0.23578498\n",
      "Training at Epoch 70 iteration 300 with loss 0.22001228\n",
      "Training at Epoch 70 iteration 400 with loss 0.09967093\n",
      "Training at Epoch 70 iteration 500 with loss 0.103382505\n",
      "Training at Epoch 70 iteration 600 with loss 0.15726417\n",
      "Training at Epoch 70 iteration 700 with loss 0.17835489\n",
      "optimal threshold: 0.558366596698761\n",
      "AUROC:0.9114547373885957\n",
      "AUPRC: 0.5930649805107445\n",
      "Confusion Matrix : \n",
      " [[8945 2430]\n",
      " [ 185 1720]]\n",
      "Recall :  0.9028871391076115\n",
      "Precision :  0.41445783132530123\n",
      "Accuracy :  0.8030873493975904\n",
      "Sensitivity :  0.7863736263736264\n",
      "Specificity :  0.9028871391076115\n",
      "Validation at Epoch 70 , AUROC: 0.9114547373885957 , AUPRC: 0.5930649805107445 , F1: 0.5567541458702302\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 71 iteration 0 with loss 0.067916065\n",
      "Training at Epoch 71 iteration 100 with loss 0.13761488\n",
      "Training at Epoch 71 iteration 200 with loss 0.18248516\n",
      "Training at Epoch 71 iteration 300 with loss 0.27854735\n",
      "Training at Epoch 71 iteration 400 with loss 0.059328318\n",
      "Training at Epoch 71 iteration 500 with loss 0.12284672\n",
      "Training at Epoch 71 iteration 600 with loss 0.061367027\n",
      "Training at Epoch 71 iteration 700 with loss 0.055621088\n",
      "optimal threshold: 0.19035768508911133\n",
      "AUROC:0.9161632479996218\n",
      "AUPRC: 0.6206826379972233\n",
      "Confusion Matrix : \n",
      " [[8786 2590]\n",
      " [ 150 1754]]\n",
      "Recall :  0.9212184873949579\n",
      "Precision :  0.4037753222836096\n",
      "Accuracy :  0.7936746987951807\n",
      "Sensitivity :  0.7723277074542897\n",
      "Specificity :  0.9212184873949579\n",
      "Validation at Epoch 71 , AUROC: 0.9161632479996218 , AUPRC: 0.6206826379972233 , F1: 0.6005841548010221\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 72 iteration 0 with loss 0.121905364\n",
      "Training at Epoch 72 iteration 100 with loss 0.15058234\n",
      "Training at Epoch 72 iteration 200 with loss 0.55379826\n",
      "Training at Epoch 72 iteration 300 with loss 0.038367584\n",
      "Training at Epoch 72 iteration 400 with loss 0.10431321\n",
      "Training at Epoch 72 iteration 500 with loss 0.31058276\n",
      "Training at Epoch 72 iteration 600 with loss 0.03543442\n",
      "Training at Epoch 72 iteration 700 with loss 0.064773366\n",
      "optimal threshold: 0.2995114028453827\n",
      "AUROC:0.9122598279530107\n",
      "AUPRC: 0.5902309491583672\n",
      "Confusion Matrix : \n",
      " [[8732 2648]\n",
      " [ 151 1749]]\n",
      "Recall :  0.9205263157894736\n",
      "Precision :  0.3977712076415738\n",
      "Accuracy :  0.7892319277108434\n",
      "Sensitivity :  0.767311072056239\n",
      "Specificity :  0.9205263157894736\n",
      "Validation at Epoch 72 , AUROC: 0.9122598279530107 , AUPRC: 0.5902309491583672 , F1: 0.5809442060085837\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 73 iteration 0 with loss 0.05031629\n",
      "Training at Epoch 73 iteration 100 with loss 0.096217275\n",
      "Training at Epoch 73 iteration 200 with loss 0.15410666\n",
      "Training at Epoch 73 iteration 300 with loss 0.18966727\n",
      "Training at Epoch 73 iteration 400 with loss 0.28471324\n",
      "Training at Epoch 73 iteration 500 with loss 0.08514854\n",
      "Training at Epoch 73 iteration 600 with loss 0.15750544\n",
      "Training at Epoch 73 iteration 700 with loss 0.059938703\n",
      "optimal threshold: 0.20032811164855957\n",
      "AUROC:0.9142379162899338\n",
      "AUPRC: 0.6008405373610448\n",
      "Confusion Matrix : \n",
      " [[8980 2396]\n",
      " [ 168 1736]]\n",
      "Recall :  0.9117647058823529\n",
      "Precision :  0.42013552758954503\n",
      "Accuracy :  0.8069277108433734\n",
      "Sensitivity :  0.789381153305204\n",
      "Specificity :  0.9117647058823529\n",
      "Validation at Epoch 73 , AUROC: 0.9142379162899338 , AUPRC: 0.6008405373610448 , F1: 0.6090704647676163\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 74 iteration 0 with loss 0.06351408\n",
      "Training at Epoch 74 iteration 100 with loss 0.11304522\n",
      "Training at Epoch 74 iteration 200 with loss 0.26257455\n",
      "Training at Epoch 74 iteration 300 with loss 0.3153764\n",
      "Training at Epoch 74 iteration 400 with loss 0.18574855\n",
      "Training at Epoch 74 iteration 500 with loss 0.23759873\n",
      "Training at Epoch 74 iteration 600 with loss 0.18941407\n",
      "Training at Epoch 74 iteration 700 with loss 0.088309705\n",
      "optimal threshold: 0.4462087154388428\n",
      "AUROC:0.9120378712091227\n",
      "AUPRC: 0.6060587857804636\n",
      "Confusion Matrix : \n",
      " [[8720 2657]\n",
      " [ 158 1745]]\n",
      "Recall :  0.9169732002101945\n",
      "Precision :  0.39641072239890957\n",
      "Accuracy :  0.7880271084337349\n",
      "Sensitivity :  0.7664586446339106\n",
      "Specificity :  0.9169732002101945\n",
      "Validation at Epoch 74 , AUROC: 0.9120378712091227 , AUPRC: 0.6060587857804636 , F1: 0.5596881598180932\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 75 iteration 0 with loss 0.06762381\n",
      "Training at Epoch 75 iteration 100 with loss 0.10901624\n",
      "Training at Epoch 75 iteration 200 with loss 0.0927767\n",
      "Training at Epoch 75 iteration 300 with loss 0.23717666\n",
      "Training at Epoch 75 iteration 400 with loss 0.15797111\n",
      "Training at Epoch 75 iteration 500 with loss 0.19603288\n",
      "Training at Epoch 75 iteration 600 with loss 0.10952252\n",
      "Training at Epoch 75 iteration 700 with loss 0.15822588\n",
      "optimal threshold: 0.33015140891075134\n",
      "AUROC:0.9130334326504863\n",
      "AUPRC: 0.5832935478494178\n",
      "Confusion Matrix : \n",
      " [[8885 2491]\n",
      " [ 159 1745]]\n",
      "Recall :  0.9164915966386554\n",
      "Precision :  0.4119452313503305\n",
      "Accuracy :  0.8004518072289156\n",
      "Sensitivity :  0.7810302390998594\n",
      "Specificity :  0.9164915966386554\n",
      "Validation at Epoch 75 , AUROC: 0.9130334326504863 , AUPRC: 0.5832935478494178 , F1: 0.5851452426508957\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 76 iteration 0 with loss 0.39469197\n",
      "Training at Epoch 76 iteration 100 with loss 0.29979047\n",
      "Training at Epoch 76 iteration 200 with loss 0.1430664\n",
      "Training at Epoch 76 iteration 300 with loss 0.02286754\n",
      "Training at Epoch 76 iteration 400 with loss 0.050276276\n",
      "Training at Epoch 76 iteration 500 with loss 0.27993506\n",
      "Training at Epoch 76 iteration 600 with loss 0.34214205\n",
      "Training at Epoch 76 iteration 700 with loss 0.10337691\n",
      "optimal threshold: 0.3681134581565857\n",
      "AUROC:0.9167047323940125\n",
      "AUPRC: 0.6144911956967156\n",
      "Confusion Matrix : \n",
      " [[8973 2403]\n",
      " [ 181 1723]]\n",
      "Recall :  0.904936974789916\n",
      "Precision :  0.4175957343674261\n",
      "Accuracy :  0.805421686746988\n",
      "Sensitivity :  0.7887658227848101\n",
      "Specificity :  0.904936974789916\n",
      "Validation at Epoch 76 , AUROC: 0.9167047323940125 , AUPRC: 0.6144911956967156 , F1: 0.5877009084556254\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 77 iteration 0 with loss 0.094115354\n",
      "Training at Epoch 77 iteration 100 with loss 0.017898109\n",
      "Training at Epoch 77 iteration 200 with loss 0.096551895\n",
      "Training at Epoch 77 iteration 300 with loss 0.06094011\n",
      "Training at Epoch 77 iteration 400 with loss 0.1422043\n",
      "Training at Epoch 77 iteration 500 with loss 0.0955078\n",
      "Training at Epoch 77 iteration 600 with loss 0.060049526\n",
      "Training at Epoch 77 iteration 700 with loss 0.26694113\n",
      "optimal threshold: 0.4736900329589844\n",
      "AUROC:0.9126127327442118\n",
      "AUPRC: 0.5873660918982377\n",
      "Confusion Matrix : \n",
      " [[9046 2331]\n",
      " [ 184 1719]]\n",
      "Recall :  0.9033105622700999\n",
      "Precision :  0.42444444444444446\n",
      "Accuracy :  0.8106174698795181\n",
      "Sensitivity :  0.7951129471741233\n",
      "Specificity :  0.9033105622700999\n",
      "Validation at Epoch 77 , AUROC: 0.9126127327442118 , AUPRC: 0.5873660918982377 , F1: 0.5803495672832174\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 78 iteration 0 with loss 0.1584715\n",
      "Training at Epoch 78 iteration 100 with loss 0.06520294\n",
      "Training at Epoch 78 iteration 200 with loss 0.11361044\n",
      "Training at Epoch 78 iteration 300 with loss 0.09606383\n",
      "Training at Epoch 78 iteration 400 with loss 0.16475376\n",
      "Training at Epoch 78 iteration 500 with loss 0.1118477\n",
      "Training at Epoch 78 iteration 600 with loss 0.08128419\n",
      "Training at Epoch 78 iteration 700 with loss 0.1257891\n",
      "optimal threshold: 0.4567438066005707\n",
      "AUROC:0.9142336473578746\n",
      "AUPRC: 0.596306707144133\n",
      "Confusion Matrix : \n",
      " [[9088 2289]\n",
      " [ 189 1714]]\n",
      "Recall :  0.9006831318970048\n",
      "Precision :  0.42817886585061204\n",
      "Accuracy :  0.8134036144578313\n",
      "Sensitivity :  0.7988046057835985\n",
      "Specificity :  0.9006831318970048\n",
      "Validation at Epoch 78 , AUROC: 0.9142336473578746 , AUPRC: 0.596306707144133 , F1: 0.5854414042333506\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 79 iteration 0 with loss 0.031610534\n",
      "Training at Epoch 79 iteration 100 with loss 0.15911958\n",
      "Training at Epoch 79 iteration 200 with loss 0.06482844\n",
      "Training at Epoch 79 iteration 300 with loss 0.108893484\n",
      "Training at Epoch 79 iteration 400 with loss 0.12281948\n",
      "Training at Epoch 79 iteration 500 with loss 0.07417762\n",
      "Training at Epoch 79 iteration 600 with loss 0.053982757\n",
      "Training at Epoch 79 iteration 700 with loss 0.085320055\n",
      "optimal threshold: 0.3323773443698883\n",
      "AUROC:0.913949872538678\n",
      "AUPRC: 0.6096424911877809\n",
      "Confusion Matrix : \n",
      " [[8801 2575]\n",
      " [ 152 1752]]\n",
      "Recall :  0.9201680672268907\n",
      "Precision :  0.40489946845389413\n",
      "Accuracy :  0.7946536144578313\n",
      "Sensitivity :  0.7736462728551337\n",
      "Specificity :  0.9201680672268907\n",
      "Validation at Epoch 79 , AUROC: 0.913949872538678 , AUPRC: 0.6096424911877809 , F1: 0.5798090040927694\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 80 iteration 0 with loss 0.13387343\n",
      "Training at Epoch 80 iteration 100 with loss 0.12408279\n",
      "Training at Epoch 80 iteration 200 with loss 0.074531525\n",
      "Training at Epoch 80 iteration 300 with loss 0.08391093\n",
      "Training at Epoch 80 iteration 400 with loss 0.043265752\n",
      "Training at Epoch 80 iteration 500 with loss 0.01608849\n",
      "Training at Epoch 80 iteration 600 with loss 0.13881224\n",
      "Training at Epoch 80 iteration 700 with loss 0.15981987\n",
      "optimal threshold: 0.41220128536224365\n",
      "AUROC:0.9087799050263565\n",
      "AUPRC: 0.5885609931463653\n",
      "Confusion Matrix : \n",
      " [[8752 2624]\n",
      " [ 163 1741]]\n",
      "Recall :  0.914390756302521\n",
      "Precision :  0.39885452462772053\n",
      "Accuracy :  0.7901355421686747\n",
      "Sensitivity :  0.7693389592123769\n",
      "Specificity :  0.914390756302521\n",
      "Validation at Epoch 80 , AUROC: 0.9087799050263565 , AUPRC: 0.5885609931463653 , F1: 0.5655697039854473\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 81 iteration 0 with loss 0.31090263\n",
      "Training at Epoch 81 iteration 100 with loss 0.14462969\n",
      "Training at Epoch 81 iteration 200 with loss 0.069909364\n",
      "Training at Epoch 81 iteration 300 with loss 0.2487352\n",
      "Training at Epoch 81 iteration 400 with loss 0.24057221\n",
      "Training at Epoch 81 iteration 500 with loss 0.14758243\n",
      "Training at Epoch 81 iteration 600 with loss 0.1310029\n",
      "Training at Epoch 81 iteration 700 with loss 0.40528166\n",
      "optimal threshold: 0.3483003079891205\n",
      "AUROC:0.9086892536550485\n",
      "AUPRC: 0.5766002548982767\n",
      "Confusion Matrix : \n",
      " [[8673 2703]\n",
      " [ 155 1749]]\n",
      "Recall :  0.9185924369747899\n",
      "Precision :  0.39285714285714285\n",
      "Accuracy :  0.7847891566265061\n",
      "Sensitivity :  0.7623945147679325\n",
      "Specificity :  0.9185924369747899\n",
      "Validation at Epoch 81 , AUROC: 0.9086892536550485 , AUPRC: 0.5766002548982767 , F1: 0.5725915875169606\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 82 iteration 0 with loss 0.10612264\n",
      "Training at Epoch 82 iteration 100 with loss 0.114743985\n",
      "Training at Epoch 82 iteration 200 with loss 0.16538092\n",
      "Training at Epoch 82 iteration 300 with loss 0.19623628\n",
      "Training at Epoch 82 iteration 400 with loss 0.13455054\n",
      "Training at Epoch 82 iteration 500 with loss 0.13278621\n",
      "Training at Epoch 82 iteration 600 with loss 0.054825082\n",
      "Training at Epoch 82 iteration 700 with loss 0.036469117\n",
      "optimal threshold: 0.3701268434524536\n",
      "AUROC:0.9109029909722824\n",
      "AUPRC: 0.570099713764146\n",
      "Confusion Matrix : \n",
      " [[8651 2724]\n",
      " [ 147 1758]]\n",
      "Recall :  0.9228346456692913\n",
      "Precision :  0.392235609103079\n",
      "Accuracy :  0.7838102409638554\n",
      "Sensitivity :  0.7605274725274725\n",
      "Specificity :  0.9228346456692913\n",
      "Validation at Epoch 82 , AUROC: 0.9109029909722824 , AUPRC: 0.570099713764146 , F1: 0.5677825654623798\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 83 iteration 0 with loss 0.1386539\n",
      "Training at Epoch 83 iteration 100 with loss 0.32913807\n",
      "Training at Epoch 83 iteration 200 with loss 0.10008864\n",
      "Training at Epoch 83 iteration 300 with loss 0.09194613\n",
      "Training at Epoch 83 iteration 400 with loss 0.26468492\n",
      "Training at Epoch 83 iteration 500 with loss 0.05894929\n",
      "Training at Epoch 83 iteration 600 with loss 0.1902137\n",
      "Training at Epoch 83 iteration 700 with loss 0.10045427\n",
      "optimal threshold: 0.383235365152359\n",
      "AUROC:0.9138145072111123\n",
      "AUPRC: 0.5977631999285191\n",
      "Confusion Matrix : \n",
      " [[8797 2579]\n",
      " [ 147 1757]]\n",
      "Recall :  0.9227941176470589\n",
      "Precision :  0.4052121771217712\n",
      "Accuracy :  0.7947289156626506\n",
      "Sensitivity :  0.7732946554149086\n",
      "Specificity :  0.9227941176470589\n",
      "Validation at Epoch 83 , AUROC: 0.9138145072111123 , AUPRC: 0.5977631999285191 , F1: 0.5773438815014308\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 84 iteration 0 with loss 0.098686144\n",
      "Training at Epoch 84 iteration 100 with loss 0.039923385\n",
      "Training at Epoch 84 iteration 200 with loss 0.1785228\n",
      "Training at Epoch 84 iteration 300 with loss 0.16247126\n",
      "Training at Epoch 84 iteration 400 with loss 0.10929007\n",
      "Training at Epoch 84 iteration 500 with loss 0.0961311\n",
      "Training at Epoch 84 iteration 600 with loss 0.06834765\n",
      "Training at Epoch 84 iteration 700 with loss 0.13623112\n",
      "optimal threshold: 0.4897163510322571\n",
      "AUROC:0.9138882148323464\n",
      "AUPRC: 0.5976098623129666\n",
      "Confusion Matrix : \n",
      " [[9021 2355]\n",
      " [ 186 1718]]\n",
      "Recall :  0.9023109243697479\n",
      "Precision :  0.4218021114657501\n",
      "Accuracy :  0.8086596385542169\n",
      "Sensitivity :  0.7929852320675106\n",
      "Specificity :  0.9023109243697479\n",
      "Validation at Epoch 84 , AUROC: 0.9138882148323464 , AUPRC: 0.5976098623129666 , F1: 0.5758951084215834\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 85 iteration 0 with loss 0.020919163\n",
      "Training at Epoch 85 iteration 100 with loss 0.08094153\n",
      "Training at Epoch 85 iteration 200 with loss 0.08547461\n",
      "Training at Epoch 85 iteration 300 with loss 0.04717966\n",
      "Training at Epoch 85 iteration 400 with loss 0.23505689\n",
      "Training at Epoch 85 iteration 500 with loss 0.07532826\n",
      "Training at Epoch 85 iteration 600 with loss 0.09728244\n",
      "Training at Epoch 85 iteration 700 with loss 0.21078077\n",
      "optimal threshold: 0.3014114201068878\n",
      "AUROC:0.9114675208163433\n",
      "AUPRC: 0.5892033398266846\n",
      "Confusion Matrix : \n",
      " [[8909 2467]\n",
      " [ 173 1731]]\n",
      "Recall :  0.9091386554621849\n",
      "Precision :  0.41233920914721295\n",
      "Accuracy :  0.8012048192771084\n",
      "Sensitivity :  0.7831399437412095\n",
      "Specificity :  0.9091386554621849\n",
      "Validation at Epoch 85 , AUROC: 0.9114675208163433 , AUPRC: 0.5892033398266846 , F1: 0.5923826395039858\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 86 iteration 0 with loss 0.03830954\n",
      "Training at Epoch 86 iteration 100 with loss 0.10182004\n",
      "Training at Epoch 86 iteration 200 with loss 0.105958246\n",
      "Training at Epoch 86 iteration 300 with loss 0.05478439\n",
      "Training at Epoch 86 iteration 400 with loss 0.17288642\n",
      "Training at Epoch 86 iteration 500 with loss 0.08597794\n",
      "Training at Epoch 86 iteration 600 with loss 0.018350556\n",
      "Training at Epoch 86 iteration 700 with loss 0.016623568\n",
      "optimal threshold: 0.2756744921207428\n",
      "AUROC:0.911626085692716\n",
      "AUPRC: 0.5802918274955073\n",
      "Confusion Matrix : \n",
      " [[8721 2655]\n",
      " [ 143 1761]]\n",
      "Recall :  0.9248949579831933\n",
      "Precision :  0.39877717391304346\n",
      "Accuracy :  0.7893072289156626\n",
      "Sensitivity :  0.7666139240506329\n",
      "Specificity :  0.9248949579831933\n",
      "Validation at Epoch 86 , AUROC: 0.911626085692716 , AUPRC: 0.5802918274955073 , F1: 0.5846795205836373\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 87 iteration 0 with loss 0.0407194\n",
      "Training at Epoch 87 iteration 100 with loss 0.049906157\n",
      "Training at Epoch 87 iteration 200 with loss 0.32432422\n",
      "Training at Epoch 87 iteration 300 with loss 0.3203194\n",
      "Training at Epoch 87 iteration 400 with loss 0.09924634\n",
      "Training at Epoch 87 iteration 500 with loss 0.07799994\n",
      "Training at Epoch 87 iteration 600 with loss 0.519963\n",
      "Training at Epoch 87 iteration 700 with loss 0.0685285\n",
      "optimal threshold: 0.36518412828445435\n",
      "AUROC:0.9139288890661752\n",
      "AUPRC: 0.5897269703209658\n",
      "Confusion Matrix : \n",
      " [[8959 2417]\n",
      " [ 173 1731]]\n",
      "Recall :  0.9091386554621849\n",
      "Precision :  0.4173095467695275\n",
      "Accuracy :  0.8049698795180723\n",
      "Sensitivity :  0.7875351617440225\n",
      "Specificity :  0.9091386554621849\n",
      "Validation at Epoch 87 , AUROC: 0.9139288890661752 , AUPRC: 0.5897269703209658 , F1: 0.585315049470578\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 88 iteration 0 with loss 0.039637588\n",
      "Training at Epoch 88 iteration 100 with loss 0.08167785\n",
      "Training at Epoch 88 iteration 200 with loss 0.107620716\n",
      "Training at Epoch 88 iteration 300 with loss 0.13467559\n",
      "Training at Epoch 88 iteration 400 with loss 0.084227115\n",
      "Training at Epoch 88 iteration 500 with loss 0.24342473\n",
      "Training at Epoch 88 iteration 600 with loss 0.098460406\n",
      "Training at Epoch 88 iteration 700 with loss 0.05445502\n",
      "optimal threshold: 0.19548918306827545\n",
      "AUROC:0.9140472671685271\n",
      "AUPRC: 0.5887037981369144\n",
      "Confusion Matrix : \n",
      " [[8794 2581]\n",
      " [ 142 1763]]\n",
      "Recall :  0.9254593175853019\n",
      "Precision :  0.40584714548802947\n",
      "Accuracy :  0.7949548192771084\n",
      "Sensitivity :  0.7730989010989011\n",
      "Specificity :  0.9254593175853019\n",
      "Validation at Epoch 88 , AUROC: 0.9140472671685271 , AUPRC: 0.5887037981369144 , F1: 0.5985663082437276\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 89 iteration 0 with loss 0.042839713\n",
      "Training at Epoch 89 iteration 100 with loss 0.0326724\n",
      "Training at Epoch 89 iteration 200 with loss 0.043964896\n",
      "Training at Epoch 89 iteration 300 with loss 0.071178235\n",
      "Training at Epoch 89 iteration 400 with loss 0.1317038\n",
      "Training at Epoch 89 iteration 500 with loss 0.048666723\n",
      "Training at Epoch 89 iteration 600 with loss 0.11960103\n",
      "Training at Epoch 89 iteration 700 with loss 0.047313757\n",
      "optimal threshold: 0.40822380781173706\n",
      "AUROC:0.9129658896859835\n",
      "AUPRC: 0.5775475817830525\n",
      "Confusion Matrix : \n",
      " [[9159 2218]\n",
      " [ 197 1706]]\n",
      "Recall :  0.8964792433000526\n",
      "Precision :  0.4347604485219164\n",
      "Accuracy :  0.8181475903614458\n",
      "Sensitivity :  0.8050452667662829\n",
      "Specificity :  0.8964792433000526\n",
      "Validation at Epoch 89 , AUROC: 0.9129658896859835 , AUPRC: 0.5775475817830525 , F1: 0.5951618641053006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 90 iteration 0 with loss 0.11906107\n",
      "Training at Epoch 90 iteration 100 with loss 0.0837397\n",
      "Training at Epoch 90 iteration 200 with loss 0.13322939\n",
      "Training at Epoch 90 iteration 300 with loss 0.12771505\n",
      "Training at Epoch 90 iteration 400 with loss 0.2976825\n",
      "Training at Epoch 90 iteration 500 with loss 0.19183029\n",
      "Training at Epoch 90 iteration 600 with loss 0.37324786\n",
      "Training at Epoch 90 iteration 700 with loss 0.11748476\n",
      "optimal threshold: 0.24404245615005493\n",
      "AUROC:0.9128850176489431\n",
      "AUPRC: 0.5736408264349155\n",
      "Confusion Matrix : \n",
      " [[8908 2470]\n",
      " [ 153 1749]]\n",
      "Recall :  0.919558359621451\n",
      "Precision :  0.4145532116615312\n",
      "Accuracy :  0.8024849397590361\n",
      "Sensitivity :  0.7829143962031991\n",
      "Specificity :  0.919558359621451\n",
      "Validation at Epoch 90 , AUROC: 0.9128850176489431 , AUPRC: 0.5736408264349155 , F1: 0.6007522837184309\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 91 iteration 0 with loss 0.16136289\n",
      "Training at Epoch 91 iteration 100 with loss 0.33984107\n",
      "Training at Epoch 91 iteration 200 with loss 0.07397864\n",
      "Training at Epoch 91 iteration 300 with loss 0.09870277\n",
      "Training at Epoch 91 iteration 400 with loss 0.35148454\n",
      "Training at Epoch 91 iteration 500 with loss 0.20699817\n",
      "Training at Epoch 91 iteration 600 with loss 0.11269607\n",
      "Training at Epoch 91 iteration 700 with loss 0.21680652\n",
      "optimal threshold: 0.22604042291641235\n",
      "AUROC:0.9110285146367841\n",
      "AUPRC: 0.5616739930339665\n",
      "Confusion Matrix : \n",
      " [[8704 2674]\n",
      " [ 137 1765]]\n",
      "Recall :  0.9279705573080967\n",
      "Precision :  0.39761207479161975\n",
      "Accuracy :  0.788328313253012\n",
      "Sensitivity :  0.7649850588855687\n",
      "Specificity :  0.9279705573080967\n",
      "Validation at Epoch 91 , AUROC: 0.9110285146367841 , AUPRC: 0.5616739930339665 , F1: 0.5896497647673812\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 92 iteration 0 with loss 0.07312938\n",
      "Training at Epoch 92 iteration 100 with loss 0.11692149\n",
      "Training at Epoch 92 iteration 200 with loss 0.105061375\n",
      "Training at Epoch 92 iteration 300 with loss 0.48677787\n",
      "Training at Epoch 92 iteration 400 with loss 0.26457885\n",
      "Training at Epoch 92 iteration 500 with loss 0.05180157\n",
      "Training at Epoch 92 iteration 600 with loss 0.019644622\n",
      "Training at Epoch 92 iteration 700 with loss 0.031925693\n",
      "optimal threshold: 0.3963451087474823\n",
      "AUROC:0.9132835030109089\n",
      "AUPRC: 0.5857631705964451\n",
      "Confusion Matrix : \n",
      " [[9080 2296]\n",
      " [ 191 1713]]\n",
      "Recall :  0.8996848739495799\n",
      "Precision :  0.4272886006485408\n",
      "Accuracy :  0.8127259036144578\n",
      "Sensitivity :  0.7981715893108298\n",
      "Specificity :  0.8996848739495799\n",
      "Validation at Epoch 92 , AUROC: 0.9132835030109089 , AUPRC: 0.5857631705964451 , F1: 0.591668131481807\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 93 iteration 0 with loss 0.027131371\n",
      "Training at Epoch 93 iteration 100 with loss 0.047320873\n",
      "Training at Epoch 93 iteration 200 with loss 0.12636119\n",
      "Training at Epoch 93 iteration 300 with loss 0.0392338\n",
      "Training at Epoch 93 iteration 400 with loss 0.09499032\n",
      "Training at Epoch 93 iteration 500 with loss 0.11409326\n",
      "Training at Epoch 93 iteration 600 with loss 0.07348686\n",
      "Training at Epoch 93 iteration 700 with loss 0.042604834\n",
      "optimal threshold: 0.3617077171802521\n",
      "AUROC:0.9130247568743552\n",
      "AUPRC: 0.5846236299558409\n",
      "Confusion Matrix : \n",
      " [[8967 2410]\n",
      " [ 174 1729]]\n",
      "Recall :  0.9085654230162901\n",
      "Precision :  0.4177337521140372\n",
      "Accuracy :  0.805421686746988\n",
      "Sensitivity :  0.7881691131229673\n",
      "Specificity :  0.9085654230162901\n",
      "Validation at Epoch 93 , AUROC: 0.9130247568743552 , AUPRC: 0.5846236299558409 , F1: 0.5863620629975771\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 94 iteration 0 with loss 0.014817904\n",
      "Training at Epoch 94 iteration 100 with loss 0.24724871\n",
      "Training at Epoch 94 iteration 200 with loss 0.13723211\n",
      "Training at Epoch 94 iteration 300 with loss 0.3822951\n",
      "Training at Epoch 94 iteration 400 with loss 0.1881293\n",
      "Training at Epoch 94 iteration 500 with loss 0.037568647\n",
      "Training at Epoch 94 iteration 600 with loss 0.19619499\n",
      "Training at Epoch 94 iteration 700 with loss 0.15148725\n",
      "optimal threshold: 0.3692009150981903\n",
      "AUROC:0.9111016373849119\n",
      "AUPRC: 0.5777588994330941\n",
      "Confusion Matrix : \n",
      " [[9125 2251]\n",
      " [ 184 1720]]\n",
      "Recall :  0.9033613445378151\n",
      "Precision :  0.4331402669352808\n",
      "Accuracy :  0.8166415662650602\n",
      "Sensitivity :  0.8021272855133614\n",
      "Specificity :  0.9033613445378151\n",
      "Validation at Epoch 94 , AUROC: 0.9111016373849119 , AUPRC: 0.5777588994330941 , F1: 0.5977175463623395\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 95 iteration 0 with loss 0.04775028\n",
      "Training at Epoch 95 iteration 100 with loss 0.14785916\n",
      "Training at Epoch 95 iteration 200 with loss 0.044488773\n",
      "Training at Epoch 95 iteration 300 with loss 0.21379152\n",
      "Training at Epoch 95 iteration 400 with loss 0.044545017\n",
      "Training at Epoch 95 iteration 500 with loss 0.08891416\n",
      "Training at Epoch 95 iteration 600 with loss 0.06723289\n",
      "Training at Epoch 95 iteration 700 with loss 0.05452648\n",
      "optimal threshold: 0.49941959977149963\n",
      "AUROC:0.9119213547760876\n",
      "AUPRC: 0.6008284202442461\n",
      "Confusion Matrix : \n",
      " [[8876 2500]\n",
      " [ 173 1731]]\n",
      "Recall :  0.9091386554621849\n",
      "Precision :  0.409123138737887\n",
      "Accuracy :  0.7987198795180723\n",
      "Sensitivity :  0.780239099859353\n",
      "Specificity :  0.9091386554621849\n",
      "Validation at Epoch 95 , AUROC: 0.9119213547760876 , AUPRC: 0.6008284202442461 , F1: 0.5639269406392694\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 96 iteration 0 with loss 0.042435557\n",
      "Training at Epoch 96 iteration 100 with loss 0.14598499\n",
      "Training at Epoch 96 iteration 200 with loss 0.10537744\n",
      "Training at Epoch 96 iteration 300 with loss 0.05691289\n",
      "Training at Epoch 96 iteration 400 with loss 0.03459255\n",
      "Training at Epoch 96 iteration 500 with loss 0.11193464\n",
      "Training at Epoch 96 iteration 600 with loss 0.05014459\n",
      "Training at Epoch 96 iteration 700 with loss 0.18428466\n",
      "optimal threshold: 0.4474298357963562\n",
      "AUROC:0.9124501428999869\n",
      "AUPRC: 0.5914573036686414\n",
      "Confusion Matrix : \n",
      " [[8964 2412]\n",
      " [ 183 1721]]\n",
      "Recall :  0.9038865546218487\n",
      "Precision :  0.4164045487539318\n",
      "Accuracy :  0.8045933734939759\n",
      "Sensitivity :  0.7879746835443038\n",
      "Specificity :  0.9038865546218487\n",
      "Validation at Epoch 96 , AUROC: 0.9124501428999869 , AUPRC: 0.5914573036686414 , F1: 0.5742141536392671\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 97 iteration 0 with loss 0.11048741\n",
      "Training at Epoch 97 iteration 100 with loss 0.13199726\n",
      "Training at Epoch 97 iteration 200 with loss 0.14312673\n",
      "Training at Epoch 97 iteration 300 with loss 0.059720334\n",
      "Training at Epoch 97 iteration 400 with loss 0.06774995\n",
      "Training at Epoch 97 iteration 500 with loss 0.023764495\n",
      "Training at Epoch 97 iteration 600 with loss 0.031624377\n",
      "Training at Epoch 97 iteration 700 with loss 0.060714684\n",
      "optimal threshold: 0.2444189190864563\n",
      "AUROC:0.9111499212186583\n",
      "AUPRC: 0.5688643580686554\n",
      "Confusion Matrix : \n",
      " [[8872 2505]\n",
      " [ 160 1743]]\n",
      "Recall :  0.9159222280609564\n",
      "Precision :  0.4103107344632768\n",
      "Accuracy :  0.7993222891566265\n",
      "Sensitivity :  0.7798189329348686\n",
      "Specificity :  0.9159222280609564\n",
      "Validation at Epoch 97 , AUROC: 0.9111499212186583 , AUPRC: 0.5688643580686554 , F1: 0.5938385269121812\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 98 iteration 0 with loss 0.15909943\n",
      "Training at Epoch 98 iteration 100 with loss 0.034291603\n",
      "Training at Epoch 98 iteration 200 with loss 0.07930222\n",
      "Training at Epoch 98 iteration 300 with loss 0.18144621\n",
      "Training at Epoch 98 iteration 400 with loss 0.021558028\n",
      "Training at Epoch 98 iteration 500 with loss 0.034096695\n",
      "Training at Epoch 98 iteration 600 with loss 0.04041305\n",
      "Training at Epoch 98 iteration 700 with loss 0.18675864\n",
      "optimal threshold: 0.5142779350280762\n",
      "AUROC:0.9132877770424062\n",
      "AUPRC: 0.5880539354484019\n",
      "Confusion Matrix : \n",
      " [[9095 2282]\n",
      " [ 179 1724]]\n",
      "Recall :  0.905937992643195\n",
      "Precision :  0.4303544682975537\n",
      "Accuracy :  0.814683734939759\n",
      "Sensitivity :  0.799419882218511\n",
      "Specificity :  0.905937992643195\n",
      "Validation at Epoch 98 , AUROC: 0.9132877770424062 , AUPRC: 0.5880539354484019 , F1: 0.5820266396897656\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 99 iteration 0 with loss 0.16593997\n",
      "Training at Epoch 99 iteration 100 with loss 0.12224085\n",
      "Training at Epoch 99 iteration 200 with loss 0.27123767\n",
      "Training at Epoch 99 iteration 300 with loss 0.12746307\n",
      "Training at Epoch 99 iteration 400 with loss 0.14039731\n",
      "Training at Epoch 99 iteration 500 with loss 0.030096978\n",
      "Training at Epoch 99 iteration 600 with loss 0.06909244\n",
      "Training at Epoch 99 iteration 700 with loss 0.27383608\n",
      "optimal threshold: 0.36322927474975586\n",
      "AUROC:0.9166881537026368\n",
      "AUPRC: 0.6129667395829396\n",
      "Confusion Matrix : \n",
      " [[9080 2298]\n",
      " [ 182 1720]]\n",
      "Recall :  0.9043112513144059\n",
      "Precision :  0.42807366849178696\n",
      "Accuracy :  0.8132530120481928\n",
      "Sensitivity :  0.7980312884513975\n",
      "Specificity :  0.9043112513144059\n",
      "Validation at Epoch 99 , AUROC: 0.9166881537026368 , AUPRC: 0.6129667395829396 , F1: 0.5923578094734989\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training at Epoch 100 iteration 0 with loss 0.07481482\n",
      "Training at Epoch 100 iteration 100 with loss 0.02345876\n",
      "Training at Epoch 100 iteration 200 with loss 0.036040112\n",
      "Training at Epoch 100 iteration 300 with loss 0.08567858\n",
      "Training at Epoch 100 iteration 400 with loss 0.043914407\n",
      "Training at Epoch 100 iteration 500 with loss 0.4043113\n",
      "Training at Epoch 100 iteration 600 with loss 0.048230037\n",
      "Training at Epoch 100 iteration 700 with loss 0.11866363\n",
      "optimal threshold: 0.4262527525424957\n",
      "AUROC:0.9136255636220733\n",
      "AUPRC: 0.5962479767537866\n",
      "Confusion Matrix : \n",
      " [[8625 2751]\n",
      " [ 134 1770]]\n",
      "Recall :  0.9296218487394958\n",
      "Precision :  0.391506303915063\n",
      "Accuracy :  0.7827560240963856\n",
      "Sensitivity :  0.758175105485232\n",
      "Specificity :  0.9296218487394958\n",
      "Validation at Epoch 100 , AUROC: 0.9136255636220733 , AUPRC: 0.5962479767537866 , F1: 0.5593845167494791\n",
      "--- Go for Testing ---\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n",
      "/home/kh278/jupyter-env/lib/python3.7/site-packages/ipykernel_launcher.py:31: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "optimal threshold: 0.37065815925598145\n",
      "AUROC:0.9168355109457471\n",
      "AUPRC: 0.609037863268461\n",
      "Confusion Matrix : \n",
      " [[9048 2327]\n",
      " [ 181 1724]]\n",
      "Recall :  0.90498687664042\n",
      "Precision :  0.42557393236237967\n",
      "Accuracy :  0.811144578313253\n",
      "Sensitivity :  0.7954285714285714\n",
      "Specificity :  0.90498687664042\n",
      "Testing AUROC: 0.9168355109457471 , AUPRC: 0.609037863268461 , F1: 0.5946808510638298 , Test loss: 0.7090408205986023\n",
      "51379.49452614784\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f715c67e8d0>]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXhU1fnA8e8hJGyyE5Q9qLigqGiKWDe0uIFLbV2w2r211q1qa39aK+5Wxdq64IJr3UCsFRFQFERkJ2HfDIQQSFgDCQkkZD+/P+YmTCaz3Jm5M3fJ+3keHmbu3Ln3zSzvPXNWpbVGCCGEt7SyOwAhhBDWk+QuhBAeJMldCCE8SJK7EEJ4kCR3IYTwoNZ2nbhHjx46IyPDrtMLIYQrLVu2bK/WOj3SfrYl94yMDLKzs+06vRBCuJJSaquZ/aRaRgghPEiSuxBCeJAkdyGE8CBJ7kII4UERk7tS6i2l1B6l1NoQjyul1AtKqVyl1Gql1OnWhymEECIaZkru7wCXhnn8MmCQ8e9m4JX4wxJCCBGPiMlda/0dUBxml6uAd7XPYqCLUqqXVQEKIYSInhV17n2AAr/7hca2ZpRSNyulspVS2UVFRRacWjhRbV09k7MKqKuX6aSFsEtSG1S11hO01pla68z09IgDrIRLvbMwn79+spoPl26zOxQhWiwrkvt2oJ/f/b7GNtFClVRUA1Bq/C9EOFNWbOdQdZ3dYXiOFcl9KvALo9fMcKBUa73TguMKQVVtHcOfnM3sDbvtDkUkwNItxdz10UoenbbO7lA8J+LcMkqpicAIoIdSqhB4CEgF0Fq/CswARgG5QAXw60QFK1qenfsr2VVWyaPT1vOjE4+0OxxhsQOVNQDsLquyORLviZjctdY3RHhcA7dZFpEQQoi4yQhVIYTwIEnuQgjhQZLcRcJo6eYuIpDPSOJIcheWUyi7QxAuI58Y60lyF0IID5LkLoQQHiTJXQghPEiSuxBCeJAkdwt8t7GI/TKPihBRk84yiSPJPU4HKmv4xVtL+d1/su0OxdOky5y3KQ90l7npjSXc/7/VdofRSJJ7nGrqfFlnc9FBmyNxHivysRe+9KJlmJ+7l4lLCyLvmCSS3IXlJCHbY9/BKrT8xBEGSe5CeED+3nLOeHwWb87fYncowiEkuVukpKKGyhpZcEDYY1txBQBzN8rylcJHkruFfvrKQrtDEMJVpBopcSS5x8n/w7luR5mNkQjhZtJQYzVJ7sIVtPSIFiIqktxFwljxi1tmmBQiNp5L7rvLKjnzyVnkSb/zuJVV1jDq+Xls2n0gqudJOhbCfp5L7tNW72R3WRXvLd5qdyiu993GItbvLOPfszbZHYowSdonRQPPJXchWiK3DhyTa1HiSHKPk3w4k0NKpN7m1ouTk0lyF44mX3ohYuO55C4TeLlTVn6xDGixgNu6jMpbnjieS+4fLtlmdwjCYDbRzFq/m2tfXcS7i6QRPFZu7zLq7uidyXPJXThAlHUphSW+eVGk+2pLJEX3RJHkbrEnZ2wgK7845ucXFFeQcd90Plu53cKoWq7KmjoOVtXaHUbSuLWaQ9pWrCfJPU6BX6YJ3+Vx7auLeOiztZSUR7/0Xs4u34ChP01aSYEx059d3JooGizI3csJD37JyQ/NjLjvngOVbNtn7+sdD0mOIpAk9wT5z6KtPPXF93EdY3VhqUXRxMmliePGN5aY3nfYE7M5b9ycBEYj3OTaVxfy9JfxfX/t1mKTe1VtHe8t3kp9feKKp3Vhir5rCkvZsrc8YecWQsQuK7+EV77dbHcYcWmxyf30R7/mwSlrmWKibnt/RTXvWzydwRUvzeeCZ7+19Jhed/64ObLSkBAmmUruSqlLlVI5SqlcpdR9QR7vr5Sao5RaoZRarZQaZX2o1iqv9q2aVHaoJuK+90xexd+nrGXdDodUk7RQW/dV8Ni09XaHISyUzHadVQX7ybhvOtlxdHhwk4jJXSmVAowHLgMGAzcopQYH7PZ3YLLWeigwBnjZ6kD9aa35OLuA8iT1gthnNIxW19Y3e6z0UPSNpuG4vA2ziVBf3F2llXKhTJDKmjque3URa7e76/VNRj/974wlCL/NaRlLEZopuQ8DcrXWeVrramAScFXAPhroZNzuDOywLsTmsvJLuPe/q3lo6rqQ+ySrRPCbd7ITfo7BY7/kN+9kJfw8gWJ9CSN9TYf/YzajX5gfXSwuuuotztvHb97JSmh7TiirC0tZml/MI5+H/m6IlqG1iX36AAV+9wuBMwP2eRj4Sil1B9ABGBnsQEqpm4GbAfr37x9trI3Kq30l9qIDVTEfwyrbktBdsaK6jm++35Pw84Ti0s4yEZWUV5PauhVHtDHzNTDvD+8to/RQDWWVNXRpn2bpsUNpeI9cdA0UCWZVg+oNwDta677AKOA9pVSzY2utJ2itM7XWmenp6XGfNNkfZC9/cZZuKW42L48T5npJZP/toY99zQivdH/06hVYxMxMct8O9PO739fY5u+3wGQArfUioC3Qw4oAgzHzOX5nYX5Sz5dMGfdN546JKyw95nWvLeJH/5wb9DHl4REyew9a22YSjfcWb2XZ1hLbzi+8zUxyzwIGKaUGKqXS8DWYTg3YZxvwIwCl1In4knvCWy027CxL9CmSzr+0HC6nfr4qoc0aIgkenLKWn76y0O4wbGX/b0Pvipjctda1wO3ATGADvl4x65RSjyqlrjR2+zPwe6XUKmAi8CudhN/0RQeqTM/jUlxezZgJi9hzoDLBUbVMtXX1DHpgBh9lyaycInpmfxz+5eNVZNw3PbHBeISpOnet9Qyt9XFa62O01k8Y28Zqracat9drrc/WWp+qtT5Na/1VIoP2rya49tVFjbcDe5QcMvqyA0xcuo3FecW8vSDf9HkOVddx78er2F8R2093rX2LTFvVa6KkvNoRg3gKiivIDxhdW15VR02d5onpGxq3mf2rvVZ6c0JbhVf9d1mh3SG4hutGqBYUV3D7B8ubbJu3yVcDFNij5LXvDg8fHjczJ+pzfbysgI+XFZJvTCgV7Xe26GAVpzz8FS/NyY363MEMfexrRwziOfeZOYwIM7o21ip6r9Xt2zHHulxYRAPXJfexn63lQMDgpZ+/uZQ5Oc27CtbWxfdBj/d7sqfMVwU0c92umJ5fHMOskqJlariQSGqPz/nj5nDLe8tM7//l2l18usKZvyZcl9xDfXhfm9t8kp/8fe6emOvvU9Y2q/4IZtnWEl7/Ls/0cevqNde9togFuXvjCa9Fqa2rT1qpuKK6Nu62IbeszJTMHxpmTrV1XwVfRlEYu+X9Zdz90arYg0og1yX3aKqvp63e2Wzboeo6bv3A/JU52QL/vIKS8IOkCoor+OkrC3lixoaw+/nbd7CKpVuKueujlTFEGLu35m9x5Rq3u0orOfaBL/hwaXIai68ev5BhT8xOyrmcIpk1cok4lxMbeV2X3Ovqm8/vEk5tXT1L8vY13v9s5XZmrDl8ZY6mnveJ6espr6pl38EqcvdETlJWlEp+/ubSkI+VVtRw7jPuGIRTU1fPo9PW85OXY+v6F2+p+VB1HV+ubX6xN6PhF+BnK811P433bc/ZfSDOIzjPvoNV1NaF/u7OWLOrsRpTWMN1yX1B7r6g2zftDp5sn5+9iesnLLbk3Mu37eekh2ZyxuOzGPncXBaarNaIpqQQTaHiQFXkGS2tYGVBJ9rJ3qxqZH1o6lpueX85qwr2W3I8U2yoGXFie2pFdS1nPD6LhyPMd/OUyxfHcBrXJfdQ9oVofAyV9ENZnLePyVkFkXcENkeoD9d+Zbjt+w9FFYcnJCjTFEaoqgpm6RbfeIhI66m6tbeJ0zoa1dXrxvai8ipfl+T3F2+T2UCTyDPJ3SpjJizmr5+stvSYa7eXcfZT3/DN97stPa7dVhcGLwUnujHvnKejr4rKN7k+qtcX066urWdh7l4u/Oe3VFQn7m/996yNjHj222arjcW79KQwz/PJPbDlO5nlssBC4NrtkadLiCa+WJJck3MZJ7tncvCG1UiF2D++v7zZNjeWex/6bG3jbbf2tTcb9bnPfMPP3lhCXlE53+9KXN3+kjzfL6VI9ejalZ8Yd/B8co9HxJ/oMf6EL7G7/3pAJvjf8vBLDZrKdyazS60Nc5xH8p9F1i6hGKuq2rqElqYBdpfZP032gcoa3py/xbVVYG7R4pL7/oqmjZDl1bXsTnIrfU2UPX6cSmvNF2t2Bl2hKtbjOZr2tclEivNApS9BX/Kv76I6/KX/nsfgsTNjDq+JGH6AXPXSfM55+htrzh/GQ1PX8di09czP3evIBmCvaHHJPdAzX+Zw5pOzqUnAIJVNJrpLutmO0kr++MFynvt6Y1zHcUtFyNL8YsZMWBx0/EQwu6IsNATWTyfbqsJSCksS3/BfahSwqmq8Uchxqhaf3BsMeuALBt4/I6rnFCThi2Ans/WhO0u9/ToEXnyC9Xyqqavn/cVbqQtR7fTszBxWbEvi3O1GGAs37+XvU9Yk77xBmG7G0PDI5+vI3eO9fv52kOQehwlRDPkPZavfFAnPz9rEH6KY1yKY8qpaxs/JDZlkYpHokrWZSJ3+633Cd3n8fcpaPgrRjfalOblcHeMArnj87PUlvL84+dMwx9JQWlhyiLcX5PPLt5K/XrAXSXK3QWX14Z+j/heIf82Kr3oD4Nmvchg3M8fkYh7WpMxQIzf9j26qTTZIES+aC4vWOuKcLImq422YFvpgkgaWuUWk3kfB3o5EdVhyQv1+Xb1uMhV5IklyD0FrzcvfNp+MLL5j+ib5Oi+B63Y2jACtqg39AUp0P3S7ehO+vSCfYU/MNjU1hNckugvn/5YXJn0MwLuL8pv8srVKPK/U1n3lcfV2+9OkFZw49ss4IjBPknsIS7YUs+eA9d3Gng4YYp2opQL/75M1PPq5/XO/W8VMqathXv9txaETQqQc6JbG3WRavq2Eeyav4sEpayPvbJHKmjrGfraOa/wW43GC88d9y/lxFM7MNsZbQZJ7CGMsmo/Gn1KHh8E3WL7NN8rTqu6E/t5aEH7Vpr0Hq9kU5SRV4y1aeCSSEx/8kj+8lx0yGR99//SwsSSyW6UTLwCJLLhXGNMH7DlQybc5exj+5Gwqa+KvWjDzDpUdcl41V1mlO0YxS3J3iMASfbJcFEVf7G37KiKvaGVRTj1UU8fMdaGna6jX4VfXmmRyfiAv2LT7AF+saToSe6mJtYVjuR48Nm09u8oqY5rfB3y/wNw4CLi8qtaR0/qG09ruAIRPngvmOc/bGzpGp31htxXHlnzcKNQF2vaR0IZQH41dZZV0bpealBhGvzCPdTtirwKNdsyCE0jJPYlCDfbJ3XMgqkVIvO7uJC8iEolb55sZ+tjXtp07XK1Yw2MPTlmbtFHJ8SR2t5KSuwOMfC66YeqhvDh7k6n9zOaqhu/d0i3FMXXf0trXMJaaErwMUVevSWnVPJhPV4Sf6ybZoklAW/Y2/cVQU1cf8u9vCVx6XfSElvup86B/Bvll8NxXEerITdhRWsnfPjU/ytH/+3zCg19y56QVQfd7bFrT3jw1YVbqcZLAhDXkoZm8Ps/XeD0xYCm+Wz9oPnNmtF6bu5mM+6bbNveOzNzoTpLcPe6Fb2Lv3eKfS3JMTg97IEhf6Omrdwat2pi2uungpw+WxDaSsr5eUx9QrxVqrvlECPY3N/h6ffNG4cqauqhGEP/DIXOgu2XRbeEj1TIipD9/HNuq7huNQUSrLEyw4Rq0rn5lIasK9nPB8emN20Itx+iEUYonPPglV5zaO+nnnRmwtkEoWusmF2MrXzOzvwIS+j7FUFfkxsualNw9ZtaGPRH32ZzgEZwN9fPzNplbYzZeSV0X1SLmpoeIzGzChuZVRqEUFAefCC7W+vPAX21RrSnsxqzqR2vNNpOrgFlNkrvHFIfp/lZSXs2UFdvDLhhedKCKH49fkPQ57pPl+11lTA2TWDcXHWROTuQLpFPEO9FcMOESakOBeldp+NHb8RS8C0sOJWRQX4Norxd19bG3Orw5fwvnjZvD2u3JXztWknsLcvN72dwVoZvhR1nbWFmwnzOfnJ2kqJLr8ekbuHNi8AZegCtfWsCv3/bNSpiVX8yo5+dRHdDQ65W650+WF3LFi/Mj7pdlDIjSGvKKfFM73PZh+IbihsbfqF4pv52fmOGMqTMqqms55m8zTPdEC9Tw2sU66CseUufeQqzfUUZWfuT5xJ/9Knhf/PU7yxLW39sJ9eDBPDhlLd/vOtCY0LzG7FTALwZplDc7/UCsH5mtNlVlBCo75GssnxJi5tNQvs3ZQ5Xfr488GxZikZJ7C/Di7E2MemFeQs/hnwB3lVm3eIcTS8lurwe2U0l5DdV1wa/mgfMuudmv3s5qUmX2zJfxd0mOluuSe7BBLyK8YP3frfbj8Qsab6/dbnY0oLyXABn3TecfMzYk/DyrC+Or9w13UXtjXp6pqof1O8v4bmNRxP1KjQnDnPqrzg1MJXel1KVKqRylVK5S6r4Q+1ynlFqvlFqnlPrQ2jAP+925AxN1aJFAdq8PGq1yv77ryej185oFq3pFEq6xPVr+OXfPgUoen76BX7y1NOi+4S7hoZL39ybHVYjQIta5K6VSgPHARUAhkKWUmqq1Xu+3zyDgfuBsrXWJUqpnogJOa8FDud3sV28H/+JHYlcVyEkPzWy8PWtD6Nkpk6lew6LcvZwzqIfdoTRRb1Qtlyd5MY9wSsqrqdOaHke0ScjxA8cCBD4WTKyNsrEykymHAbla6zytdTUwCbgqYJ/fA+O11iUAWmv39CUTptXVx949rTZoPatm70HrF0RJtFATwCXa2wu2cNObS6Lq224Xq2pTYr24D33sazIfn2VRFM2FG08ycWnw6aaTUT3qz0xy7wP4R1tobPN3HHCcUmqBUmqxUurSYAdSSt2slMpWSmUXFUWudxPOUlVjbd/jvQeryXx8Fgs3Bx9NCvBCkks7ZlRU1zWb7iAZGnqQ7Cq1ZwyCf/WU+Zwbfk+3Vdc1CLdWr/8vPTvbDKyq42gNDAJGADcAryulugTupLWeoLXO1FpnpqenBz4sWqhwAzwizTdjx+AQsKe6yEkTeFkVySELVnSK1sLNexsbbL3MTHLfDvTzu9/X2OavEJiqta7RWm8BNuJL9sJDnJNaDrvcxCAcrzDbLz1aV700v0kVhqkLl/bN4jk5q4B6B3VpKSiuiDjcPyu/hD+8l52kiOxjJrlnAYOUUgOVUmnAGGBqwD5T8JXaUUr1wFdNk/jmf+Ea4RKG9Bu316rC0qjbPqrr6hn0wBf89ZPVTFkZ+/z78U5jvLP0EI98vq5xls1zn5nDeSYWsF6cV2xqPeADle4t4UdM7lrrWuB2YCawAZistV6nlHpUKXWlsdtMYJ9Saj0wB7hXax26IjUOJxzVKRGHFSZ8m6A5V3ZaVIc8J8fXjhO88TYxnkxg//RQVU6xJMTSiuiSVDRz6zeM4gwl7Fw1cb5V9368mrcX5LNkS/TpZtzMHMoiJO/t+6MfkGfXvPuBTNW5a61naK2P01ofo7V+wtg2Vms91bittdb3aK0Ha62HaK0nJSrgHwzsmqhDiwjiHQSTLJHmz7FCw/d3QgL7p0eqcjI7BQDAy3PNz+v/6fLtDHrgC7bui72x06r8pvXhZHnP5Obva20cPbga1NdrdpZaN6raKVzXadyJw9GFs1TEsCSgG53w4Jem931trvmL0PQ1OwHYuNvc1NANF4FgCT3eKreq2noG3j8DgP8tP1z9U15Vyx/fX8aeA/F3pX3tuzzO+sc3bLZokXqnzIvjuonDpH7WPgcdNEhFOMcXa2Pvdx9rAf/zVTviOq+/hZt9XTy3l5gvvYf7ZeI/SZidFTQuLLkLuxRG8eEP5LX3bWuxfaWzRCeMWC/isRS8zFbfjJvpjKUG3cR9yV2K7q4UOCe6213w7Ld2h5Aw8VzEE2X8nM0R9/nKBSN3k8l91TJ2ByBisvegdZNWtXTf5hTRtX2a3WGYFq6dLNxIz2ht2Om8ycbs7DjjuuTeSkruwiHMLgXnP52vFdMWzN1YxFwT0+bGKxnftGmrdybhLKFFSr5urilwXbVMamv3vtgismQkLcDUnOKR3PhG6LVo/flP51sXZVHub5+uiWp/Ox2oPFxX74ye3tEJFnOobPPOwvwERmIN1yX39mmu+7EhovD2gi1JOU+oucejYWbZwnh9GGFuHSdp0gU1ljVU43Aoyd1fc/dY020ykVyX3IUQyZG9Nf6LV7KqNf6zaKujJlZzAknuQoigtux1bul0Xm7T1bF2xTjCNJ5rj7kuo/ZdcKSOQziK11tU7pqU+KkRrBLraPC8ooPsLvMbOZqAN3V6QEPsxt0HOSbd+t4y4VJzpNkn7SbJXYgkahja7xaxzIp44T/nJiCSyEKNWP3vskJTz4+2J5PTZ4yUahnhKG7ueuY1BSUVvLtoq91hxO0vH68ytd+OKKt2rp9grreUXaTkLhxFUrtzrNtRxrodZVE9J3CmSqe/n1qHL1A4Pf5wpOQuhLCMm9oUzHDzD0lJ7kIIy3wp87s0MWtDYha4MUOSu3CUJVuK7Q5BWMwtpd8HPl1rdwiWcmWde0or1bhmovAWmTPeW7YVV9A6xdnZ3dnRxc6VJfeXbzzd7hCEECbcMXEFo18Iv1xgIm3c7byZIpPFlcn9kpOOsjsEIYQLXPyv7+wOwTauTO5CCGGVcG0Cbl6zWZK7EEKE4ObJyCS5CyFECD9/M/6poe3i2uTev1t7u0MQQgjHcm1yH9yrk90hCCGEY7k2uQshhAjNtcndLaPehBDO5tVU4trkLoQQVtixv9LuEBJCkrsQouXSkOPRUayuTe5SLSOEEKG5NrkLIYQIzVRyV0pdqpTKUUrlKqXuC7PfT5VSWimVaV2IIc7l2WYQIUTSeDiNREzuSqkUYDxwGTAYuEEpNTjIfh2BPwFLrA5SCCG8pDwJU1ubKbkPA3K11nla62pgEnBVkP0eA54GktL0fP+oExh54pHJOJUQwqPyig7act6s/MQvSmMmufcBCvzuFxrbGimlTgf6aa2nhzuQUupmpVS2Uiq7qKgo6mD99e3anjd+mclfLj4uruMIIVquq19eaHcICRN3g6pSqhXwHPDnSPtqrSdorTO11pnp6enxnhqA2y8cZMlxhBDCS8wk9+1AP7/7fY1tDToCJwPfKqXygeHA1GQ0qgohhAjOTHLPAgYppQYqpdKAMcDUhge11qVa6x5a6wytdQawGLhSa52dkIiFEMLlVBIG6kRM7lrrWuB2YCawAZistV6nlHpUKXVlogMUQggRvdZmdtJazwBmBGwbG2LfEfGHFZ1rzujLecelc+fEFck+tRBCRC0Z3es9MUL12WtP5cpTe9sdhhBCOIYnkrsQQrhJMubGkuQuhBAeJMldCCGSbP6mvQk/hyR3IYRIstfn5SX8HJLchRDCgzyV3Pt2bWd3CEII4QieSu63X3AsAN06pNkciRBChOaIEapuMmZYf/KfGs3jPz7Z7lCEECIkGcQUo1FDevH6L2TeMiFEy+XJ5A7woxN62h2CEEIEJYOY4tCqlYcXRxRCuFoy1oD2bHIXQgjHkpJ7/Lq2T7U7BCGEaKK6tj7h5/B0ct/w6KVMue1su8MQQoik83Ryb5eWQmqKp/9EIYQIyvOZr7XRsHpir042RyKEEMnj+eTes1NbnvrJEN759Q/sDkUIIZLG88kdfCNXj+zUlrOP7W53KEIIkRQtIrk3+OB3w+0OQQghkqJFJXeA0UN62R2CEEIkXItL7uNvPN3uEIQQIuFaXHIXQoiWoEUn95P7SPdIIYQ3tdjknjmgK9PuOJcfn9bb7lCEEMJyre0OwA6rxl5M2zTfda1taorN0QghhPVaZMm9c/tU2rT2JfWzj+0BwEm9pYpGCOEdLbLk7u+KU3tz3qB0OrdPJeO+6XaHI4QQlmiRJfdAnWVaYCGEx0hyF0IID5LkLoQQHmQquSulLlVK5SilcpVS9wV5/B6l1Hql1Gql1Gyl1ADrQ02ejO7t7Q5BCCHiEjG5K6VSgPHAZcBg4Aal1OCA3VYAmVrrU4D/As9YHWgyrHroYn59dgb/+MkpANw98jg6tmnxbc5CCBcyU3IfBuRqrfO01tXAJOAq/x201nO01hXG3cVAX2vDTI7O7VJ56IqTOOuY7nx993ncceGxrHnkErvDEkKIqJlJ7n2AAr/7hca2UH4LfBHsAaXUzUqpbKVUdlFRkfkobTDoyI60apWEJcqFECIBLG1QVUrdBGQC44I9rrWeoLXO1FpnpqenW3nqpMh6YCQXDT6S1BRJ+kIIZzOT3LcD/fzu9zW2NaGUGgk8AFypta6yJjxnSe/Yhtd/kclxR3a0OxQhhAjLTHLPAgYppQYqpdKAMcBU/x2UUkOB1/Al9j3Wh+ksWtsdgRBChBcxuWuta4HbgZnABmCy1nqdUupRpdSVxm7jgCOAj5VSK5VSU0MczhMktwshnM5UPz+t9QxgRsC2sX63R1ocl6Ms+/tI6v0yupaiuxDC4aQTtwndj2jT5H6ndjIXjRDC2WT6gRgck36E3SEIIURYktxjMLhX094y+U+NtikSIYQITpJ7DG4aPoDpd55jdxhCCBGSJPcYKKU4qXfnJttuGNYvxN5w18hBiQ5JCCGakORukSevHkLek6OYdsc5jB7Si2l3HC7Z33GhJHchRHJJbxmLKKVQCk7u05nxN55OTV09AJkDupIic9QIIZJMknscZt1zHtW1wfu8p6a04tNbf8gxPaVnjRAi+aRaJg7H9uzI4N6dQj4+tH9XOrX19Yl/fsxpjds7tW1Nm9by0gshEkcyTJJcdVoffnP2QADe/e2Z5Dx+GX8ccYzNUQkhvEqSexKNvWIw+U+N5rR+XQCalN5vGt6fDmkpdoUmhPAYSe4OcOePBvH4j4dw6wXH2h2KEMIjJLnbqHfndgD06twWgAuO79nk8dUPX8yoIUclPS4hhPtJbxkbXZvZlx4d0xqT+uDenVj/6CUs37ofpaBT21TapkpVjRAielJyt5FSigtPOBKlDveDb5/WmnMG9eDsY3sA8PPhA5o9r50kfCFEBFJyd7ih/buS/9Ro1iZnfkoAAA32SURBVO0o5WBlLWce3R2A/L3lbNpzkGN7HsET09cza4PnF8ASQkRBkrtLBM5lk9GjAxk9OgBwXWY/Se5CiCakWsYDLj7pKD783ZkAnDuoR8j9Mgd0TVZIQgibSXL3iB8e24P8p0Zz549CT1L28S1ncebAbmH38Xd9ZuiZLoUQzibJ3WN+kNGNGXeey8gTfT1w5vxlRONjSik++sNZ/PacgUGf26dLu8bbHdJS6NJBlhMUwq0kuXvQ4N6deO7603j5xtMZ2KMDJ/ZqOv9N53aprBx7EbP/fH5jov/hMd2Z/efzmXTzcK7L7Mv8/7sw5PEX3X8hPQLWlRVCOIskd4/q1DaVUUN6AfD57Wez8fHLmjzepX0ax6QfwS/O8nW1vGxIL9qmpjD86O48c82pdO2Q1tiI29sYZNWgV+d29O/WjnCuy+zbeDtcO4AQIjGkt0wL0Dol9DV8QPcOrHn4Yo5o0/yjcMUpvUhLUVw0+ChSWiky7pve+NirPz+DYU/MDnncs47pzrmD0jnuyI689t3mxu0/yOhKVn5JjH+JEMIsKbkLOrZNbTKQqoFSiktP7tW42EhG9/aNj/Xs2LbJNMZjLx/c5LlXD+3LFaf25vijOqI4fOyjezSf3/61n5/BO7/+QeP9k8JMoyyEMEdK7sK0T/74Q/L3lTfev+q0Pvxp0koATuvfpXF74OLhfxt1AisKSnj22lM54aiOnD6gC+3SWnPnxBUAXHKSb/6crAdG0rqVomuHtCa/EoQQ0ZPkLkzrfkQbugc0pN583tHMWr+b0/t35Wdn9ufcY3s0G3DV/Yg2fPPnEY33r/9BfwAe/Xw9ew9WNW5P7yiNtEJYRaplRFz+NupEvjG6Wz559RAuMxpxzZhx5zl8dPPwsPs89ZMhXHVa72bb5/31Au695PiI5whsSBaipZCSu7BNz05t6dmpbdDH8p8a3Xj7x0P7cM0Zfdmx/xCZGd3YVVpJv27tue2CYxl+dDd++sqiJs89tW9nVhWWApDWuhW/+mEG7yzMjxjPyBN7yjQOwjMkuQvHa5uawrmD0hvvH5N+uFF2aL/DUyr8fPgAHhh9Im1TU3h3UT4LcvcCENhWfPfI4/jXrI2c3KcTp/btwgdLtvHdvRfQv3t7hjw0kwNVtRzZqQ0pSrGjtDKhf5to6syB3ViypdjuMDxBaa1tOXFmZqbOzs625dzCW5ZtLaFv13YcGeJXwPb9h7jlvWW8+atM1u8oI71jG0a/MJ9hGd344PdnNv4SaFBdW0/rVopWRi+hPQcqmbJiO//8aiNz772AP7yXzd6D1ew9WMWNZw6glYK/Xz6Yypo6Roz7ll1lhy8INwzrx8SlBYl9AVxu1j3nM2npNt5ZmM/Gxy/j6L/NsDukpPD/dRoNpdQyrXVmxP0kuYuWRmvN+Dm5XJvZL+QFIVa1dfUc+8AXAOQ+cRk7Sys595k5jDzxSGZt2A3Ap7f+kF2llfzxg+VNnntavy6sLNgPQJf2qeyvqAF87QvnPjOncb9pd5zDHRNX0LNjG0Yc35OBPTpwy/vLgsYzbGA3lgaUhAf36sT6nWXW/MEWaPjV1GDPgcqwYyi8whHJXSl1KfA8kAK8obV+KuDxNsC7wBnAPuB6rXV+uGNKchctRVVtHWkprRg3M4dlW0v46A9nATBt9Q527q9kZcF+pq/Zydx7RzCge4fG5932wXLapLbiuetO459f5fDiN7l88+fzOTq9+ViBguIKZq7bxb++3sjMu8/jihfnc+WpvXnkqpP5fNUOzhuUTpvUVo0re42b+T3j52xm1JCjuOSko6it0yzdUsxH2Yd/ZVx+Si+mrd7ZeP/Vm07nlveXNzv3Sz8byuWn9Gbkc3PJ3XPQ1GtyRJvWHKyq5frMfjx9zSnNHl+7vZTLX5wPwJZ/jOLVuXm8Pi+PAd3bs2Lb/rDH7pCWQnl1nak47HLtGX0Zd+2pMT3XsuSulEoBNgIXAYVAFnCD1nq93z63AqdorW9RSo0BrtZaXx/uuJLchfApr6olK7+YEQFr6Pqrr9fsLKtsMrlbPLTW1NZrUoOMXt5VWsna7aVceEJPnpyxgZuGD2hcOwBgTWEph2rq2LqvnGv9Zg6tqq1jWX4Jfbq2o03rFHaUHuInLy9kym1n85+F+Xy6YjtPXj2E9mkpXHlq78Zqr1B2l1XSs2ObZgPsDlXXMWbCIu6+6Dh+/2424645laz8Yvp2bc9Fg3tybM+OjX/je4u3MvazdQD86UeDGNC9PfdMXtXsXOcfl87cjUVB4xg9pBfT1xy+yK0aezFXjZ9P/r4K7ho5iNw9B1lZsJ/CkkOkpihq6nw5dVhGN5bmN/3V1KV9KoN7deL9354Z8e8PxcrkfhbwsNb6EuP+/QBa63/47TPT2GeRUqo1sAtI12EOLsldiJZld1ml5dVgsaqpq+eb7/ewp6ySq4b2oVPbVFYX7mdlwX7G/KA/aa1bUXqohtq6+saxHYeq66isqaNrhzTq6zUaGkdv+ztQWcP+ihr6dGnHHRNXcMEJPbnmjL7N9ouV2eRuprdMH8C/RagQODPUPlrrWqVUKdAd2BsQ1M3AzQD9+/c3cWohhFc4JbEDpKa0ahwZ3eCUvl04pe/hkdad2zWd8rpdWgrt0nzVWuFK3R3bptKxre+542883aqQo5bUQUxa6wla60ytdWZ6enrkJwghhIiJmeS+HfBfkqevsS3oPka1TGd8DatCCCFsYCa5ZwGDlFIDlVJpwBhgasA+U4FfGrevAb4JV98uhBAisSLWuRt16LcDM/F1hXxLa71OKfUokK21ngq8CbynlMoFivFdAIQQQtjE1PQDWusZwIyAbWP9blcC11obmhBCiFjJrJBCCOFBktyFEMKDJLkLIYQH2TZxmFKqCNga49N7EDBAykGcGptT4wLnxubUuMC5sTk1LnBubNHGNUBrHXGgkG3JPR5KqWwzw2/t4NTYnBoXODc2p8YFzo3NqXGBc2NLVFxSLSOEEB4kyV0IITzIrcl9gt0BhOHU2JwaFzg3NqfGBc6NzalxgXNjS0hcrqxzF0IIEZ5bS+5CCCHCkOQuhBBepLV21T/gUiAHyAXuS9A53gL2AGv9tnUDvgY2Gf93NbYr4AUjntXA6X7P+aWx/ybgl37bzwDWGM95AaN6zERc/YA5wHpgHfAnB8XWFlgKrDJie8TYPhBYYhzvIyDN2N7GuJ9rPJ7hd6z7je05wCVWvPf4Jr1bAUxzWFz5xuu9Et9EfE55P7sA/wW+BzYAZzkkruON16rhXxlwl0NiuxvfZ38tMBHfd8K2z1lCEnCi/uH7gm4GjgbS8CWSwQk4z3nA6TRN7s80vKDAfcDTxu1RwBfGh2g4sMTvC5pn/N/VuN3wgVtq7KuM515mMq5eDR9OoCO+tW0HOyQ2BRxh3E41PrDDgcnAGGP7q8Afjdu3Aq8at8cAHxm3Bxvvaxvji7HZeN/jeu+Be4APOZzcnRJXPtAjYJsT3s//AL8zbqfhS/a2xxUkH+wCBtgdG77V6LYA7fw+X7+y83Nme8KO8s08C5jpd/9+4P4EnSuDpsk9B+hl3O4F5Bi3X8O3YHiT/YAbgNf8tr9mbOsFfO+3vcl+Ucb4Gb6Fyx0VG9AeWI5vOca9QOvA9w/fFNJnGbdbG/upwPe0Yb943nt8C8zMBi4EphnnsT0uY/98mid3W99PfIvtbCGgxGp3XEHivBhY4ITYOLzUaDfjczMNuMTOz5nb6tyDrefaJ0nnPlJr3bAE+i7gyAgxhdteGGR7VJRSGcBQfCVkR8SmlEpRSq3EV6X1Nb6Sxn6tdW2Q4zVZdxdoWHc32pjN+DfwV6DeuN/dIXEBaOArpdQyY41hsP/9HAgUAW8rpVYopd5QSnVwQFyBxuCr/sDu2LTW24FngW3ATnyfm2XY+DlzW3J3BO27dGq7zq+UOgL4BLhLa13m/5idsWmt67TWp+ErKQ8DTrAjDn9KqcuBPVrrZXbHEsI5WuvTgcuA25RS5/k/aNP72RpfteQrWuuhQDm+qg6742pkrAp3JfBx4GN2xKaU6gpche/C2BvogK+O3DZuS+5m1nNNlN1KqV4Axv97IsQUbnvfINtNUUql4kvsH2it/+ek2Bporffja/g9C+hirKsbeLxQ6+5GG3MkZwNXKqXygUn4qmaed0BcQGOJD631HuBTfBdFu9/PQqBQa73EuP9ffMne7rj8XQYs11rvNu7bHdtIYIvWukhrXQP8D99nz77PWbT1XHb+w1eiyMN3dWxoVDgpQefKoGmd+ziaNtg8Y9weTdMGm6XG9m746i27Gv+2AN2MxwIbbEaZjEkB7wL/DtjuhNjSgS7G7XbAPOByfCUr/walW43bt9G0QWmycfskmjYo5eFrTIr7vQdGcLhB1fa48JXuOvrdXoivtOeE93MecLxx+2EjJtvj8otvEvBrp3wH8LUvrcPX3qTwNUjfYefnzPaEHe0/fK3fG/HV5z6QoHNMxFdvVoOvFPNbfPVhs/F1m5rl90FQwHgjnjVApt9xfoOv21JuwAcxE193qc3AS5jvanUOvp+bqzncFWyUQ2I7BV9Xw9XG88ca2482viy5xge9jbG9rXE/13j8aL9jPWCcPwe/ngrxvvc0Te62x2XEsIrD3UcfMLY74f08Dcg23s8p+BKg7XEZz+2Ar5Tb2W+b7bEBj+DrOroWeA9fgrbtcybTDwghhAe5rc5dCCGECZLchRDCgyS5CyGEB0lyF0IID5LkLoQQHiTJXQghPEiSuxBCeND/A3w+W5BUKlL+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fold 1\n",
    "#biosnap interaction times 1e-6, flat, batch size 64, len 205, channel 3, epoch 50\n",
    "s = time()\n",
    "model_max, loss_history = main(1, 5e-6)\n",
    "e = time()\n",
    "print(e-s)\n",
    "lh = list(filter(lambda x: x < 1, loss_history))\n",
    "plt.plot(lh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}