CIFAR10 image classification using a MLP model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' \\nAuthor - Tejas Krishna Reddy\\nDate - 2nd Feb 2019\\n\\nA Multi-Layer perceptron (Fully Connected Layer Network) to classify the objects from CIFAR image dataset\\n\\n Cifar-10 Classification using Keras Tutorial. The CIFAR-10 data set consists of 60000 32×32x3 color images in 10 classes,\\n with 6000 images per class. There are 50000 training images and 10000 test images.\\n \\n'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''' \n",
    "Author - Tejas Krishna Reddy\n",
    "Date - 2nd Feb 2019\n",
    "\n",
    "A Multi-Layer perceptron (Fully Connected Layer Network) to classify the objects from CIFAR image dataset\n",
    "\n",
    " Cifar-10 Classification using Keras Tutorial. The CIFAR-10 data set consists of 60000 32×32x3 color images in 10 classes,\n",
    " with 6000 images per class. There are 50000 training images and 10000 test images.\n",
    " \n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Tejas\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "#import keras as k\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_inputs = 32*32*3       # flattening each image to provide as input.  #  trying to form a model in the shape of a tunnel\n",
    "n_hidden1 = 400        # Num of nodes in hidden layer 1.\n",
    "n_hidden2 = 250        \n",
    "n_hidden3 = 64\n",
    "n_outputs = 10        # Number of final outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now declare the input and output placeholders. \n",
    "# Placeholders let us use different userdefined inputs in the first and last layers \n",
    "\n",
    "X = tf.placeholder(tf.float32, shape = (None, n_inputs), name = \"X\")  #The None values can be taken as i/p's from user later.\n",
    "y = tf.placeholder(tf.int32, shape=(None), name = \"y\")\n",
    "training = tf.placeholder_with_default(False, shape=(), name='training')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "### This function is not used in the code below - It is just for personal reference.\n",
    "def neural_layer(X, n_neurons, name, activation=None):\n",
    "    with tf.name_scope(name):\n",
    "        n_inputs = int(X.get_shape()[1])\n",
    "        std_dev = 2/np.sqrt(n_inputs)   # Inorder to not let the SGD algo break or explode, it is essential to initialize the \n",
    "                                        # variables. Initializing randomly using a truncated normal distribution with a \n",
    "                                        # std_dev of 2/ sqrt(n_inputs) can drastically improve the performance of the nn.\n",
    "        init = tf.truncated_normal((n_inputs, n_neurons), stddev = std_dev)\n",
    "        W = tf.Variable(init, name = \"Kernel\")\n",
    "        b = tf.Variable(tf.zeros([n_neurons]), name = \"bias\")\n",
    "        Z = tf.matmul(X, W) + b;\n",
    "        if activation is not None:\n",
    "            return activation(Z)\n",
    "        else:\n",
    "            return Z\n",
    "\n",
    "##Instead of this function we can use tensorflow inbuilt function - tf.layers.dense() to achieve the same purpose."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial  # The batch normalization can be automated using \"partial\" - but i havent done that below\n",
    "\n",
    "X = tf.placeholder(tf.float32, shape=(None, n_inputs), name=\"X\")\n",
    "training = tf.placeholder_with_default(False, shape=(), name='training')\n",
    "\n",
    "batch_norm_momentum = 0.9\n",
    "with tf.name_scope(\"dnn\"):\n",
    "    \n",
    "    #### Different initializers\n",
    "    \n",
    "    #he_init = tf.variance_scaling_initializer()  - around 45% acc (SGD)\n",
    "    #he_init = tf.random_uniform_initializer()    - around 46.5% acc (SGD)\n",
    "    he_init = tf.contrib.layers.xavier_initializer()  # - 48% accuracy (SGD) , upto 50% using Adam\n",
    "    \n",
    "    #### 3 Hidden layers, each using relu activation function\n",
    "\n",
    "    hidden1 = tf.layers.dense(X, n_hidden1, name=\"hidden1\")\n",
    "    bn1 = tf.layers.batch_normalization(hidden1, training=training, momentum=0.9) \n",
    "    # Momentum of 0.9 is best known for optimization.\n",
    "    bn1Activation = tf.nn.relu(bn1)\n",
    "    \n",
    "    hidden2 = tf.layers.dense(bn1Activation, n_hidden2, name = \"hidden2\")\n",
    "    bn2 = tf.layers.batch_normalization(hidden2, training=training, momentum=0.9)\n",
    "    bn2Activation = tf.nn.relu(bn2)\n",
    "\n",
    "    hidden3 = tf.layers.dense(bn2Activation, n_hidden3, name = \"hidden3\")\n",
    "    bn3 = tf.layers.batch_normalization(hidden3, training = training, momentum = 0.9)\n",
    "    bn3Activation = tf.nn.relu(bn3)\n",
    "    logits_before_bn = tf.layers.dense(bn3Activation, n_outputs, name = \"outputs\") \n",
    "    \n",
    "    # NOTE:: logits are the nn outputs before classifying/ or activation layer\n",
    "    #logits = tf.nn.softmax(my_batch_norm_layer(logits_before_bn))\n",
    "\n",
    "with tf.name_scope(\"loss\"):\n",
    "    xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels = y, logits =logits_before_bn)\n",
    "    # \"labels\": vector must provide a single specific index for the true class\n",
    "    \n",
    "    # WARNING: This op expects unscaled logits, since it performs a softmax on logits internally for efficiency.\n",
    "    # Do not call this op with the output of softmax, as it will produce incorrect results.\n",
    "    loss = tf.reduce_mean(xentropy, name=\"loss\")\n",
    "    \n",
    "\n",
    "with tf.name_scope(\"train\"):\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=0.05, epsilon=1e-06, use_locking=True,name='Adam')\n",
    "    training_op = optimizer.minimize(loss)\n",
    "\n",
    "with tf.name_scope(\"eval\"):\n",
    "    correct = tf.nn.in_top_k(logits_before_bn, y, 1)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()      #initializes the tf.variable/constant/placeholders declared above\n",
    "#he_init = tf.contrib.layers.xavier_initializer()\n",
    "saver = tf.train.Saver()                      # helps to save the model and reuse later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "### Now load the data\n",
    "from keras.datasets import cifar10\n",
    "from keras.utils.np_utils import to_categorical\n",
    "\n",
    "(X_train, y_train), (X_test, y_test) = cifar10.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of training data:\n",
      "(50000, 32, 32, 3)\n",
      "(50000, 1)\n",
      "Shape of test data:\n",
      "(10000, 32, 32, 3)\n",
      "(10000, 1)\n"
     ]
    }
   ],
   "source": [
    "print(\"Shape of training data:\")\n",
    "print(X_train.shape)\n",
    "print(y_train.shape)\n",
    "print(\"Shape of test data:\")\n",
    "print(X_test.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Transforming label indices to one-hot encoded vectors \n",
    "\n",
    "## No need to convert to one hot now, we have done later in the code. \n",
    "## in tn.softmax_with_crossentropy_with_logits, the labels argument asks for the index of actual class. Therefore, we halt one\n",
    "## hot encoding as of now.\n",
    "\n",
    "#y_train = to_categorical(y_train, num_classes=10)\n",
    "#y_test = to_categorical(y_test, num_classes=10)\n",
    "\n",
    "# Transforming images from (32,32,3) to a single vector of size (32*32*3) {32*32*3 = 3072}\n",
    "\n",
    "X_train = np.reshape(X_train,(50000,3072))\n",
    "X_test = np.reshape(X_test,(10000,3072))\n",
    "X_train = X_train.astype('float32')\n",
    "X_test = X_test.astype('float32')\n",
    "\n",
    "# Normalization of pixel values (to [0-1] range) # converting uint8 to double format\n",
    "\n",
    "X_train /= 255\n",
    "X_test /= 255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now divide the test and train datasets into 3 parts, training set, validation set, and testing set\n",
    "# Considering the validation set to have 5000 images\n",
    "X_valid, X_train = X_train[:5000], X_train[5000:]\n",
    "y_valid, y_train = y_train[:5000], y_train[5000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(45000, 3072) (45000, 1)\n",
      "(5000, 3072) (5000, 1)\n",
      "(10000, 3072) (10000, 1)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape, y_train.shape)     #Just to have a clear idea of the training/ validation/testing shapes of tensors\n",
    "print(X_valid.shape, y_valid.shape)\n",
    "print(X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 25\n",
    "batch_size = 6000 \n",
    "# If batch size is less like 100 or so, acc will fluctuate like 44% 20% 55% etc erratically due to underfitting nature\n",
    "\n",
    "# A function to shuffle the whole data set and select different batches randomly with replacement.\n",
    "def shuffle_batch(X, y, batch_size):\n",
    "    rnd_idx = np.random.permutation(len(X))\n",
    "    n_batches = len(X) // batch_size\n",
    "    for batch_idx in np.array_split(rnd_idx, n_batches):\n",
    "        X_batch, y_batch = X[batch_idx], y[batch_idx]\n",
    "        yield X_batch, y_batch                         # returns the first batch that was formed.\n",
    "        \n",
    "# Yeild is similar to return, the fun or loop runs untill it hits yeild, then what ever the first value of that variable is,\n",
    "# at that moment, it gets returned, and the loop continues."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Validation accuracy: 0.1368\n",
      "1 Validation accuracy: 0.2174\n",
      "2 Validation accuracy: 0.2638\n",
      "3 Validation accuracy: 0.317\n",
      "4 Validation accuracy: 0.3466\n",
      "5 Validation accuracy: 0.3646\n",
      "6 Validation accuracy: 0.384\n",
      "7 Validation accuracy: 0.3862\n",
      "8 Validation accuracy: 0.406\n",
      "9 Validation accuracy: 0.4106\n",
      "10 Validation accuracy: 0.4258\n",
      "11 Validation accuracy: 0.4382\n",
      "12 Validation accuracy: 0.4474\n",
      "13 Validation accuracy: 0.4488\n",
      "14 Validation accuracy: 0.462\n",
      "15 Validation accuracy: 0.4142\n",
      "16 Validation accuracy: 0.4462\n",
      "17 Validation accuracy: 0.4586\n",
      "18 Validation accuracy: 0.435\n",
      "19 Validation accuracy: 0.4918\n",
      "20 Validation accuracy: 0.4858\n",
      "21 Validation accuracy: 0.4794\n",
      "22 Validation accuracy: 0.473\n",
      "23 Validation accuracy: 0.4954\n",
      "24 Validation accuracy: 0.4884\n"
     ]
    }
   ],
   "source": [
    "# Now lets test the model with the built model on Validation Set\n",
    "\n",
    "extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n",
    "# tf.get_collection(), which can find some tensor that you want,\n",
    "# UPDATE_OPS is a collection of ops (operations performed when the graph runs, like multiplication, ReLU, etc.), not variables. \n",
    "#y = tf.placeholder(tf.int32, shape=(None,10), name=\"y\")\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run([training_op, extra_update_ops],\n",
    "                     feed_dict={training: True, X: X_batch, y: y_batch.reshape([-1])})\n",
    "        accuracy_val = accuracy.eval(feed_dict={X: X_valid, y: y_valid.reshape([-1])})\n",
    "        print(epoch, \"Validation accuracy:\", accuracy_val)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 TestSet accuracy: 0.1972\n",
      "2 TestSet accuracy: 0.3236\n",
      "4 TestSet accuracy: 0.3675\n",
      "6 TestSet accuracy: 0.3853\n",
      "8 TestSet accuracy: 0.4102\n",
      "10 TestSet accuracy: 0.4277\n",
      "12 TestSet accuracy: 0.4583\n",
      "14 TestSet accuracy: 0.4373\n",
      "16 TestSet accuracy: 0.4698\n",
      "18 TestSet accuracy: 0.476\n",
      "20 TestSet accuracy: 0.4591\n",
      "22 TestSet accuracy: 0.4715\n",
      "24 TestSet accuracy: 0.4955\n"
     ]
    }
   ],
   "source": [
    "# Once accuracy over validation set is satisfactory, test the results on test set\n",
    "\n",
    "n_epochs = 25\n",
    "batch_size = 5000\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init.run()\n",
    "\n",
    "    for epoch in range(n_epochs):\n",
    "        for X_batch, y_batch in shuffle_batch(X_train, y_train, batch_size):\n",
    "            sess.run([training_op, extra_update_ops],\n",
    "                     feed_dict={training: True, X: X_batch, y: y_batch.reshape([-1])})\n",
    "        if epoch % 2 == 0:\n",
    "            accuracy_val = accuracy.eval(feed_dict={X: X_test, y: y_test.reshape([-1])})\n",
    "            print(epoch, \"TestSet accuracy:\", accuracy_val)\n",
    "\n",
    "    save_path = saver.save(sess, \"./my_cifar10_model_final.ckpt\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}