notebooks/rnn_mnist_simple.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sequence classification with LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Packages imported\n",
      "Extracting data/train-images-idx3-ubyte.gz\n",
      "Extracting data/train-labels-idx1-ubyte.gz\n",
      "Extracting data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/t10k-labels-idx1-ubyte.gz\n",
      "MNIST loaded\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow.examples.tutorials.mnist.input_data as input_data\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline \n",
    "print (\"Packages imported\")\n",
    "\n",
    "mnist = input_data.read_data_sets(\"data/\", one_hot=True)\n",
    "trainimgs, trainlabels, testimgs, testlabels \\\n",
    " = mnist.train.images, mnist.train.labels, mnist.test.images, mnist.test.labels \n",
    "ntrain, ntest, dim, nclasses \\\n",
    " = trainimgs.shape[0], testimgs.shape[0], trainimgs.shape[1], trainlabels.shape[1]\n",
    "print (\"MNIST loaded\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# We will treat the MNIST image $\\in \\mathcal{R}^{28 \\times 28}$ as $28$ sequences of a vector $\\mathbf{x} \\in \\mathcal{R}^{28}$. \n",
    "# Our simple RNN consists of  \n",
    "1. One input layer which converts a $28$ dimensional input to an $128$ dimensional hidden layer, \n",
    "2. One intermediate recurrent neural network (LSTM) \n",
    "3. One output layer which converts an $128$ dimensional output of the LSTM to $10$ dimensional output indicating a class label. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"images/etc/rnn_input3.jpg\" width=\"700\" height=\"400\" >"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Contruct a Recurrent Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "diminput  = 28\n",
    "dimhidden = 128\n",
    "dimoutput = nclasses\n",
    "nsteps    = 28\n",
    "weights = {\n",
    "    'hidden': tf.Variable(tf.random_normal([diminput, dimhidden])), \n",
    "    'out': tf.Variable(tf.random_normal([dimhidden, dimoutput]))\n",
    "}\n",
    "biases = {\n",
    "    'hidden': tf.Variable(tf.random_normal([dimhidden])),\n",
    "    'out': tf.Variable(tf.random_normal([dimoutput]))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network ready\n"
     ]
    }
   ],
   "source": [
    "def _RNN(_X, _istate, _W, _b, _nsteps, _name):\n",
    "    # 1. Permute input from [batchsize, nsteps, diminput] \n",
    "    #   => [nsteps, batchsize, diminput]\n",
    "    _X = tf.transpose(_X, [1, 0, 2])\n",
    "    # 2. Reshape input to [nsteps*batchsize, diminput] \n",
    "    _X = tf.reshape(_X, [-1, diminput])\n",
    "    # 3. Input layer => Hidden layer\n",
    "    _H = tf.matmul(_X, _W['hidden']) + _b['hidden']\n",
    "    # 4. Splite data to 'nsteps' chunks. An i-th chunck indicates i-th batch data \n",
    "    _Hsplit = tf.split(0, _nsteps, _H) \n",
    "    # 5. Get LSTM's final output (_LSTM_O) and state (_LSTM_S)\n",
    "    #    Both _LSTM_O and _LSTM_S consist of 'batchsize' elements\n",
    "    #    Only _LSTM_O will be used to predict the output. \n",
    "    with tf.variable_scope(_name):\n",
    "        lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(dimhidden, forget_bias=1.0)\n",
    "        _LSTM_O, _LSTM_S = tf.nn.rnn(lstm_cell, _Hsplit, initial_state=_istate)\n",
    "    # 6. Output\n",
    "    _O = tf.matmul(_LSTM_O[-1], _W['out']) + _b['out']    \n",
    "    # Return! \n",
    "    return {\n",
    "        'X': _X, 'H': _H, 'Hsplit': _Hsplit,\n",
    "        'LSTM_O': _LSTM_O, 'LSTM_S': _LSTM_S, 'O': _O \n",
    "    }\n",
    "print (\"Network ready\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Out Network looks like this"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"images/etc/rnn_mnist_look.jpg\" width=\"700\" height=\"400\" >"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Network Ready!\n"
     ]
    }
   ],
   "source": [
    "learning_rate = 0.001\n",
    "x      = tf.placeholder(\"float\", [None, nsteps, diminput])\n",
    "istate = tf.placeholder(\"float\", [None, 2*dimhidden]) \n",
    "    # state & cell => 2x n_hidden\n",
    "y      = tf.placeholder(\"float\", [None, dimoutput])\n",
    "myrnn  = _RNN(x, istate, weights, biases, nsteps, 'basic')\n",
    "pred   = myrnn['O']\n",
    "cost   = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y)) \n",
    "optm   = tf.train.AdamOptimizer(learning_rate).minimize(cost) # Adam Optimizer\n",
    "accr   = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(pred,1), tf.argmax(y,1)), tf.float32))\n",
    "init   = tf.initialize_all_variables()\n",
    "print (\"Network Ready!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start optimization\n",
      "Epoch: 000/005 cost: 0.479400075\n",
      " Training accuracy: 0.992\n",
      " Test accuracy: 0.922\n",
      "Epoch: 001/005 cost: 0.136942688\n",
      " Training accuracy: 0.969\n",
      " Test accuracy: 0.959\n",
      "Epoch: 002/005 cost: 0.081425477\n",
      " Training accuracy: 0.984\n",
      " Test accuracy: 0.951\n",
      "Epoch: 003/005 cost: 0.061170839\n",
      " Training accuracy: 0.969\n",
      " Test accuracy: 0.973\n",
      "Epoch: 004/005 cost: 0.047727333\n",
      " Training accuracy: 0.992\n",
      " Test accuracy: 0.973\n",
      "Optimization Finished.\n"
     ]
    }
   ],
   "source": [
    "training_epochs = 5\n",
    "batch_size      = 128\n",
    "display_step    = 1\n",
    "sess = tf.Session()\n",
    "sess.run(init)\n",
    "summary_writer = tf.train.SummaryWriter('/tmp/tensorflow_logs', graph=sess.graph)\n",
    "print (\"Start optimization\")\n",
    "for epoch in range(training_epochs):\n",
    "    avg_cost = 0.\n",
    "    total_batch = int(mnist.train.num_examples/batch_size)\n",
    "    # Loop over all batches\n",
    "    for i in range(total_batch):\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        batch_xs = batch_xs.reshape((batch_size, nsteps, diminput))\n",
    "        # Fit training using batch data\n",
    "        feeds = {x: batch_xs, y: batch_ys, istate: np.zeros((batch_size, 2*dimhidden))}\n",
    "        sess.run(optm, feed_dict=feeds)\n",
    "        # Compute average loss\n",
    "        avg_cost += sess.run(cost, feed_dict=feeds)/total_batch\n",
    "    # Display logs per epoch step\n",
    "    if epoch % display_step == 0: \n",
    "        print (\"Epoch: %03d/%03d cost: %.9f\" % (epoch, training_epochs, avg_cost))\n",
    "        feeds = {x: batch_xs, y: batch_ys, istate: np.zeros((batch_size, 2*dimhidden))}\n",
    "        train_acc = sess.run(accr, feed_dict=feeds)\n",
    "        print (\" Training accuracy: %.3f\" % (train_acc))\n",
    "        testimgs = testimgs.reshape((ntest, nsteps, diminput))\n",
    "        feeds = {x: testimgs, y: testlabels, istate: np.zeros((ntest, 2*dimhidden))}\n",
    "        test_acc = sess.run(accr, feed_dict=feeds)\n",
    "        print (\" Test accuracy: %.3f\" % (test_acc))\n",
    "print (\"Optimization Finished.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## What we have done so far is to feed 28 sequences of vectors $ \\mathbf{x} \\in \\mathcal{R}^{28}$. \n",
    "\n",
    "## What will happen if we feed first 25 sequences of $\\mathbf{x}$?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " If we use 25 seqs, test accuracy becomes 0.766\n"
     ]
    }
   ],
   "source": [
    "# How may sequences will we use?\n",
    "nsteps2     = 25\n",
    "\n",
    "# Test with truncated inputs\n",
    "testimgs = testimgs.reshape((ntest, nsteps, diminput))\n",
    "testimgs_trucated = np.zeros(testimgs.shape)\n",
    "testimgs_trucated[:, 28-nsteps2:] = testimgs[:, :nsteps2, :]\n",
    "feeds = {x: testimgs_trucated, y: testlabels, istate: np.zeros((ntest, 2*dimhidden))}\n",
    "test_acc = sess.run(accr, feed_dict=feeds)\n",
    "print (\" If we use %d seqs, test accuracy becomes %.3f\" % (nsteps2, test_acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# What's going on inside the RNN?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inputs to the RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of 'xtest' is (5, 784)\n"
     ]
    }
   ],
   "source": [
    "batch_size = 5\n",
    "xtest, _ = mnist.test.next_batch(batch_size)\n",
    "print (\"Shape of 'xtest' is %s\" % (xtest.shape,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reshaped inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of 'xtest1' is (5, 28, 28)\n"
     ]
    }
   ],
   "source": [
    "# Reshape (this will go into the network)\n",
    "xtest1 = xtest.reshape((batch_size, nsteps, diminput))\n",
    "print (\"Shape of 'xtest1' is %s\" % (xtest1.shape,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feeds: inputs and initial states"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feeds = {x: xtest1, istate: np.zeros((batch_size, 2*dimhidden))}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Each indivisual input to the LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of 'rnnout_X' is (140, 28)\n"
     ]
    }
   ],
   "source": [
    "rnnout_X = sess.run(myrnn['X'], feed_dict=feeds)\n",
    "print (\"Shape of 'rnnout_X' is %s\" % (rnnout_X.shape,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Each indivisual intermediate state "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of 'rnnout_H' is (140, 128)\n"
     ]
    }
   ],
   "source": [
    "rnnout_H = sess.run(myrnn['H'], feed_dict=feeds)\n",
    "print (\"Shape of 'rnnout_H' is %s\" % (rnnout_H.shape,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Actual input to the LSTM (List)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type of 'rnnout_Hsplit' is <type 'list'>\n",
      "Length of 'rnnout_Hsplit' is 28 and the shape of each item is (5, 128)\n"
     ]
    }
   ],
   "source": [
    "rnnout_Hsplit = sess.run(myrnn['Hsplit'], feed_dict=feeds)\n",
    "print (\"Type of 'rnnout_Hsplit' is %s\" % (type(rnnout_Hsplit)))\n",
    "print (\"Length of 'rnnout_Hsplit' is %s and the shape of each item is %s\" \n",
    "       % (len(rnnout_Hsplit), rnnout_Hsplit[0].shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Output from the LSTM (List)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type of 'rnnout_LSTM_O' is <type 'list'>\n",
      "Length of 'rnnout_LSTM_O' is 28 and the shape of each item is (5, 128)\n"
     ]
    }
   ],
   "source": [
    "rnnout_LSTM_O = sess.run(myrnn['LSTM_O'], feed_dict=feeds)\n",
    "print (\"Type of 'rnnout_LSTM_O' is %s\" % (type(rnnout_LSTM_O)))\n",
    "print (\"Length of 'rnnout_LSTM_O' is %s and the shape of each item is %s\" \n",
    "       % (len(rnnout_LSTM_O), rnnout_LSTM_O[0].shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of 'rnnout_O' is (5, 10)\n"
     ]
    }
   ],
   "source": [
    "rnnout_O = sess.run(myrnn['O'], feed_dict=feeds)\n",
    "print (\"Shape of 'rnnout_O' is %s\" % (rnnout_O.shape,))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}