5. Feedforward Neural Networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A. 1 Hidden Layer Feedforward Neural Network (Sigmoid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Steps:\n",
    "1. Load dataset\n",
    "2. Make dataset Iterable\n",
    "3. Create model class\n",
    "4. Instantiate model class\n",
    "5. Instantiate loss class\n",
    "6. Instantiate optimizer class\n",
    "7. Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as dsets\n",
    "from torch.autograd import Variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Step 1: Load dataset\n",
    "\n",
    "train_dataset = dsets.MNIST(root='./mnist', \n",
    "                            train=True, \n",
    "                            transform=transforms.ToTensor(),\n",
    "                            download=True)\n",
    "\n",
    "test_dataset = dsets.MNIST(root='./mnist', \n",
    "                           train=False, \n",
    "                           transform=transforms.ToTensor())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Step 2: Make dataset iterable\n",
    "batch_size = 100\n",
    "n_iters = 3000\n",
    "num_epochs = n_iters / (len(train_dataset) / batch_size)\n",
    "num_epochs = int(num_epochs)\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, \n",
    "                                           batch_size=batch_size, \n",
    "                                           shuffle=True)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, \n",
    "                                          batch_size=batch_size, \n",
    "                                          shuffle=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 3: Create model class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class FeedforwardNeuralNetModel(nn.Module):\n",
    "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
    "        super(FeedforwardNeuralNetModel, self).__init__()\n",
    "        # Linear function: 784 --> 100\n",
    "        self.fc1 = nn.Linear(input_dim, hidden_dim) \n",
    "        \n",
    "        # Non-linearity \n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "        \n",
    "        # Linear function (readout):\n",
    "        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n",
    "    \n",
    "    def forward(self, x):\n",
    "        # Linear function 1\n",
    "        out = self.fc1(x)\n",
    "        # Non-linearity 1\n",
    "        out = self.sigmoid(out)\n",
    "        # Linear function 4 (readout)\n",
    "        out = self.fc2(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 4: Instantiate model class\n",
    "- Input dimension: 784\n",
    "    - size of image\n",
    "    - 28 x 28 = 784\n",
    "- Output dimension: 10\n",
    "    - 0,1,2,3,4,5,6,7,8,9\n",
    "- Hidden dimension: 100\n",
    "    - can be any number\n",
    "    - also called: number of neurons, num of non-linear activation function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_dim = 28*28\n",
    "hidden_dim = 100\n",
    "output_dim = 10\n",
    "\n",
    "model = FeedforwardNeuralNetModel(input_dim, hidden_dim, output_dim)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 5: Instantiate Loss Class\n",
    "- Feedforward neural network: Cross Entropy Loss\n",
    "    - Logistic regression: Cross Entropy Loss\n",
    "    - Linear Regression: MSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 6: Instantiate optimizer class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.1\n",
    "\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "torch.Size([100, 784])\n",
      "torch.Size([100])\n",
      "torch.Size([10, 100])\n",
      "torch.Size([10])\n"
     ]
    }
   ],
   "source": [
    "#Params\n",
    "print(len(list(model.parameters())))\n",
    "\n",
    "#Hidden layer params\n",
    "print(list(model.parameters())[0].size())\n",
    "\n",
    "#Bias params\n",
    "print(list(model.parameters())[1].size())\n",
    "\n",
    "#FC Param\n",
    "print(list(model.parameters())[2].size())\n",
    "#FC Bias\n",
    "print(list(model.parameters())[3].size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 7: Train model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 500. Loss: 0.36283618211746216. Accuracy: 89.48\n",
      "Iteration: 1000. Loss: 0.5215944051742554. Accuracy: 90.48\n",
      "Iteration: 1500. Loss: 0.4186626374721527. Accuracy: 91.31\n",
      "Iteration: 2000. Loss: 0.3877609968185425. Accuracy: 91.61\n",
      "Iteration: 2500. Loss: 0.2371458113193512. Accuracy: 92.1\n",
      "Iteration: 3000. Loss: 0.2324649691581726. Accuracy: 92.31\n"
     ]
    }
   ],
   "source": [
    "iter = 0\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "\n",
    "        images = Variable(images.view(-1, 28*28))\n",
    "        labels = Variable(labels)\n",
    "        \n",
    "        # Clear gradients w.r.t. parameters\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Forward pass to get output/logits\n",
    "        outputs = model(images)\n",
    "        \n",
    "        # Calculate Loss: softmax --> cross entropy loss\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        # Getting gradients w.r.t. parameters\n",
    "        loss.backward()\n",
    "        \n",
    "        # Updating parameters\n",
    "        optimizer.step()\n",
    "        \n",
    "        iter += 1\n",
    "        \n",
    "        if iter % 500 == 0:\n",
    "            # Calculate Accuracy         \n",
    "            correct = 0\n",
    "            total = 0\n",
    "            # Iterate through test dataset\n",
    "            for images, labels in test_loader:\n",
    "                images = Variable(images.view(-1, 28*28))\n",
    "                \n",
    "                # Forward pass only to get logits/output\n",
    "                outputs = model(images)\n",
    "                \n",
    "                # Get predictions from the maximum value\n",
    "                _, predicted = torch.max(outputs.data, 1)\n",
    "                \n",
    "                # Total number of labels\n",
    "                total += labels.size(0)\n",
    "                \n",
    "                # Total correct predictions\n",
    "                correct += (predicted == labels).sum()\n",
    "            \n",
    "            accuracy = 100 * correct / total\n",
    "            \n",
    "            # Print Loss\n",
    "            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## B. 1 Hidden Layer Feedforward Neural Network (ReLU)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class FeedforwardNeuralNetModel(nn.Module):\n",
    "    def __init__(self, input_dim, hidden_dim, output_dim):\n",
    "        super(FeedforwardNeuralNetModel, self).__init__()\n",
    "        # Linear function: 784 --> 100\n",
    "        self.fc1 = nn.Linear(input_dim, hidden_dim) \n",
    "        \n",
    "        # Non-linearity \n",
    "        self.relu = nn.ReLU()\n",
    "        \n",
    "        # Linear function (readout):\n",
    "        self.fc2 = nn.Linear(hidden_dim, output_dim)  \n",
    "    \n",
    "    def forward(self, x):\n",
    "        # Linear function 1\n",
    "        out = self.fc1(x)\n",
    "        # Non-linearity 1\n",
    "        out = self.relu(out)\n",
    "        # Linear function (readout)\n",
    "        out = self.fc2(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 500. Loss: 0.29238009452819824. Accuracy: 92.51\n",
      "Iteration: 1000. Loss: 0.305687814950943. Accuracy: 92.88\n",
      "Iteration: 1500. Loss: 0.20709852874279022. Accuracy: 93.11\n",
      "Iteration: 2000. Loss: 0.18495804071426392. Accuracy: 93.3\n",
      "Iteration: 2500. Loss: 0.19227074086666107. Accuracy: 93.3\n",
      "Iteration: 3000. Loss: 0.22882875800132751. Accuracy: 93.63\n"
     ]
    }
   ],
   "source": [
    "#Train model(same as sigmoid)\n",
    "\n",
    "iter = 0\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "\n",
    "        images = Variable(images.view(-1, 28*28))\n",
    "        labels = Variable(labels)\n",
    "        \n",
    "        # Clear gradients w.r.t. parameters\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Forward pass to get output/logits\n",
    "        outputs = model(images)\n",
    "        \n",
    "        # Calculate Loss: softmax --> cross entropy loss\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        # Getting gradients w.r.t. parameters\n",
    "        loss.backward()\n",
    "        \n",
    "        # Updating parameters\n",
    "        optimizer.step()\n",
    "        \n",
    "        iter += 1\n",
    "        \n",
    "        if iter % 500 == 0:\n",
    "            # Calculate Accuracy         \n",
    "            correct = 0\n",
    "            total = 0\n",
    "            # Iterate through test dataset\n",
    "            for images, labels in test_loader:\n",
    "                images = Variable(images.view(-1, 28*28))\n",
    "                \n",
    "                # Forward pass only to get logits/output\n",
    "                outputs = model(images)\n",
    "                \n",
    "                # Get predictions from the maximum value\n",
    "                _, predicted = torch.max(outputs.data, 1)\n",
    "                \n",
    "                # Total number of labels\n",
    "                total += labels.size(0)\n",
    "                \n",
    "                # Total correct predictions\n",
    "                correct += (predicted == labels).sum()\n",
    "            \n",
    "            accuracy = 100 * correct / total\n",
    "            \n",
    "            # Print Loss\n",
    "            print('Iteration: {}. Loss: {}. Accuracy: {}'.format(iter, loss.data[0], accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}