NMT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import unicodedata\n",
    "import re\n",
    "import numpy as np\n",
    "import os\n",
    "import io\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000 <start> can we make this quick   roxanne korrine and andrew barrett are having an incredibly horrendous public break up on the quad  again  <end>\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "with open('features.pickle', 'rb') as handle:\n",
    "    input_sent = pickle.load(handle)\n",
    "\n",
    "\n",
    "with open('labels.pickle', 'rb') as handle:\n",
    "    output_sent = pickle.load(handle)\n",
    "    \n",
    "input_sent=input_sent[:40000]\n",
    "output_sent=output_sent[:40000]\n",
    "print(len(input_sent),input_sent[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(lang):\n",
    "    lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(\n",
    "      filters='')\n",
    "    lang_tokenizer.fit_on_texts(lang)\n",
    "    \n",
    "    tensor = lang_tokenizer.texts_to_sequences(lang)\n",
    "\n",
    "    \n",
    "    tensor = tf.keras.preprocessing.sequence.pad_sequences(tensor,\n",
    "                                                         padding='post')\n",
    "    return tensor, lang_tokenizer\n",
    "\n",
    "def dataset():\n",
    "    input_tensor, inp_lang_tokenizer = tokenize(input_sent)\n",
    "    target_tensor, targ_lang_tokenizer = tokenize(output_sent)\n",
    "    return input_tensor, target_tensor, inp_lang_tokenizer, targ_lang_tokenizer\n",
    "\n",
    "\n",
    "input_tensor, output_tensor, inp_lang, out_lang = dataset()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[   1   35   19  114   23  912 8163 8164   16 4499 2858   30  393   87\n",
      " 4500 8165 1103  509   58   37    6 5754  197    2    0    0    0    0\n",
      "    0    0    0    0    0    0    0    0] <keras_preprocessing.text.Tokenizer object at 0x000001A1DB63BE88>\n",
      "40000\n",
      "36 35\n"
     ]
    }
   ],
   "source": [
    "print(input_tensor[0],inp_lang)\n",
    "print(len(output_tensor))\n",
    "\n",
    "def max_len(tensor):\n",
    "    return max(len(t) for t in tensor)\n",
    "\n",
    "max_len_inp = max_len(input_tensor)\n",
    "max_len_out = max_len(output_tensor)\n",
    "    \n",
    "print(max_len_inp,max_len_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32000 32000 8000 8000\n"
     ]
    }
   ],
   "source": [
    "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, output_tensor, test_size=0.2)\n",
    "\n",
    "print(len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Language; index to word mapping\n",
      "1 ----> <start>\n",
      "4 ----> you\n",
      "27 ----> have\n",
      "7 ----> to\n",
      "1203 ----> quickly\n",
      "2 ----> <end>\n",
      "\n",
      "Target Language; index to word mapping\n",
      "1 ----> <start>\n",
      "176 ----> .\n",
      "857 ----> imagine\n",
      "10 ----> a\n",
      "144 ----> better\n",
      "7065 ----> grape\n",
      "25 ----> for\n",
      "6 ----> the\n",
      "5453 ----> region\n",
      "2 ----> <end>\n"
     ]
    }
   ],
   "source": [
    "def convert(lang, tensor):\n",
    "    for t in tensor:\n",
    "        if t!=0:\n",
    "            print (\"%d ----> %s\" % (t, lang.index_word[t]))\n",
    "\n",
    "\n",
    "print (\"Input Language; index to word mapping\")\n",
    "convert(inp_lang, input_tensor_train[0])\n",
    "print ()\n",
    "print (\"Target Language; index to word mapping\")\n",
    "convert(out_lang, target_tensor_train[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(<tf.Tensor: shape=(64, 36), dtype=int32, numpy=\n",
      "array([[  1,  10,  42, ...,   0,   0,   0],\n",
      "       [  1,  19,  21, ...,   0,   0,   0],\n",
      "       [  1,   5,  21, ...,   0,   0,   0],\n",
      "       ...,\n",
      "       [  1,  41, 104, ...,   0,   0,   0],\n",
      "       [  1,   4,  89, ...,   0,   0,   0],\n",
      "       [  1, 170,   2, ...,   0,   0,   0]])>, <tf.Tensor: shape=(64, 35), dtype=int32, numpy=\n",
      "array([[    1,     5,    35, ...,     0,     0,     0],\n",
      "       [    1,    63,     3, ...,     0,     0,     0],\n",
      "       [    1,     5,    23, ...,     0,     0,     0],\n",
      "       ...,\n",
      "       [    1,    56,     4, ...,     0,     0,     0],\n",
      "       [    1,    10, 14798, ...,     0,     0,     0],\n",
      "       [    1,    13,    23, ...,     0,     0,     0]])>)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "14848"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BUFFER_SIZE = len(input_tensor_train)\n",
    "BATCH_SIZE = 64\n",
    "steps_per_epoch = len(input_tensor_train)//BATCH_SIZE\n",
    "embedding_dim = 32\n",
    "units = 256\n",
    "vocab_inp_size=len(inp_lang.word_index)+1\n",
    "vocab_tar_size=len(out_lang.word_index)+1\n",
    "\n",
    "\n",
    "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n",
    "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n",
    "\n",
    "for element in dataset:\n",
    "    print(element)\n",
    "    break\n",
    "    \n",
    "vocab_inp_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(TensorShape([64, 36]),\n",
       " TensorShape([64, 35]),\n",
       " <tf.Tensor: shape=(64, 36), dtype=int32, numpy=\n",
       " array([[  1,  69,  74, ...,   0,   0,   0],\n",
       "        [  1,  10,   8, ...,   0,   0,   0],\n",
       "        [  1,  13, 190, ...,   0,   0,   0],\n",
       "        ...,\n",
       "        [  1,   4,  84, ...,   0,   0,   0],\n",
       "        [  1,  64,   3, ...,   0,   0,   0],\n",
       "        [  1, 971, 980, ...,   0,   0,   0]])>)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_input_batch, example_target_batch = next(iter(dataset))\n",
    "example_input_batch.shape, example_target_batch.shape,example_input_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.Model):\n",
    "    def __init__(self,vocab_size,embedding_dim,enc_units,batch_size):\n",
    "        super(Encoder,self).__init__()\n",
    "        self.batch_size=batch_size\n",
    "        self.enc_units=enc_units\n",
    "        self.embedding=tf.keras.layers.Embedding(vocab_size,embedding_dim)\n",
    "        self.gru=tf.keras.layers.GRU(self.enc_units,\n",
    "                                    return_sequences=True,\n",
    "                                    return_state=True,\n",
    "                                    recurrent_initializer='glorot_uniform')\n",
    "    def call(self,x,hidden):\n",
    "        x=self.embedding(x)\n",
    "        output,state=self.gru(x,initial_state=hidden)\n",
    "        return output,state\n",
    "    \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_size,self.enc_units))\n",
    "    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoder output shape: (batch size, sequence length, units) (64, 36, 256)\n",
      "Encoder Hidden state shape: (batch size, units) (64, 256)\n"
     ]
    }
   ],
   "source": [
    "encoder=Encoder(vocab_inp_size,embedding_dim, units, BATCH_SIZE)\n",
    "\n",
    "sample_hidden = encoder.initialize_hidden_state()\n",
    "sample_output, sample_hidden = encoder(example_input_batch, sample_hidden)\n",
    "print ('Encoder output shape: (batch size, sequence length, units) {}'.format(sample_output.shape))\n",
    "print ('Encoder Hidden state shape: (batch size, units) {}'.format(sample_hidden.shape))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BahdanauAttention(tf.keras.layers.Layer):\n",
    "    def __init__(self, units):\n",
    "        super(BahdanauAttention, self).__init__()\n",
    "        self.W1 = tf.keras.layers.Dense(units)\n",
    "        self.W2 = tf.keras.layers.Dense(units)\n",
    "        self.V = tf.keras.layers.Dense(1)\n",
    "    \n",
    "    def call(self, query, values):\n",
    "        # query hidden state shape == (batch_size, hidden size)\n",
    "        # query_with_time_axis shape == (batch_size, 1, hidden size)\n",
    "        # values shape == (batch_size, max_len, hidden size)\n",
    "        # we are doing this to broadcast addition along the time axis to calculate the score\n",
    "        query_with_time_axis = tf.expand_dims(query, 1)\n",
    "        \n",
    "        # score shape == (batch_size, max_length, 1)\n",
    "        # we get 1 at the last axis because we are applying score to self.V\n",
    "        # the shape of the tensor before applying self.V is (batch_size, max_length, units)\n",
    "        score = self.V(tf.nn.tanh(\n",
    "            self.W1(query_with_time_axis) + self.W2(values)))\n",
    "\n",
    "        # attention_weights shape == (batch_size, max_length, 1)\n",
    "        attention_weights = tf.nn.softmax(score, axis=1)\n",
    "\n",
    "        # context_vector shape after sum == (batch_size, hidden_size)\n",
    "        context_vector = attention_weights * values\n",
    "        context_vector = tf.reduce_sum(context_vector, axis=1)\n",
    "\n",
    "        return context_vector, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attention result shape: (batch size, units) (64, 256)\n",
      "Attention weights shape: (batch_size, sequence_length, 1) (64, 36, 1)\n"
     ]
    }
   ],
   "source": [
    "attention_layer = BahdanauAttention(10)\n",
    "attention_result, attention_weights = attention_layer(sample_hidden, sample_output)\n",
    "\n",
    "print(\"Attention result shape: (batch size, units) {}\".format(attention_result.shape))\n",
    "print(\"Attention weights shape: (batch_size, sequence_length, 1) {}\".format(attention_weights.shape))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.dec_units = dec_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = tf.keras.layers.GRU(self.dec_units,\n",
    "                                       return_sequences=True,\n",
    "                                       return_state=True,\n",
    "                                       recurrent_initializer='glorot_uniform')\n",
    "        self.fc = tf.keras.layers.Dense(vocab_size)\n",
    "\n",
    "        # used for attention\n",
    "        self.attention = BahdanauAttention(self.dec_units)\n",
    "    \n",
    "    def call(self, x, hidden, enc_output):\n",
    "        # enc_output shape == (batch_size, max_length, hidden_size)\n",
    "        context_vector, attention_weights = self.attention(hidden, enc_output)\n",
    "\n",
    "        # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
    "        x = self.embedding(x)\n",
    "\n",
    "        # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
    "        x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
    "\n",
    "        # passing the concatenated vector to the GRU\n",
    "        output, state = self.gru(x)\n",
    "\n",
    "        # output shape == (batch_size * 1, hidden_size)\n",
    "        output = tf.reshape(output, (-1, output.shape[2]))\n",
    "\n",
    "        # output shape == (batch_size, vocab)\n",
    "        x = self.fc(output)\n",
    "\n",
    "        return x, state, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decoder output shape: (batch_size, vocab size) (64, 14988)\n"
     ]
    }
   ],
   "source": [
    "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)\n",
    "\n",
    "sample_decoder_output, _, _ = decoder(tf.random.uniform((BATCH_SIZE, 1)),\n",
    "                                      sample_hidden, sample_output)\n",
    "\n",
    "print ('Decoder output shape: (batch_size, vocab size) {}'.format(sample_decoder_output.shape))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.keras.optimizers.Adam()\n",
    "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n",
    "    from_logits=True, reduction='none')\n",
    "\n",
    "def loss_function(real, pred):\n",
    "    mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
    "    loss_ = loss_object(real, pred)\n",
    "    \n",
    "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "    loss_ *= mask\n",
    "    \n",
    "    return tf.reduce_mean(loss_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_dir = './training_checkpoints2'\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
    "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
    "                                 encoder=encoder,\n",
    "                                 decoder=decoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def train_step(inp, targ, enc_hidden):\n",
    "    loss = 0\n",
    "    with tf.GradientTape() as tape:\n",
    "        enc_output, enc_hidden = encoder(inp, enc_hidden)\n",
    "\n",
    "        dec_hidden = enc_hidden\n",
    "\n",
    "        dec_input = tf.expand_dims([out_lang.word_index['<start>']] * BATCH_SIZE, 1)\n",
    "\n",
    "        # Teacher forcing - feeding the target as the next input\n",
    "        for t in range(1, targ.shape[1]):\n",
    "            # passing enc_output to the decoder\n",
    "            predictions, dec_hidden, _ = decoder(dec_input, \n",
    "                                                 dec_hidden, enc_output)\n",
    "            loss += loss_function(targ[:, t], predictions)\n",
    "            # using teacher forcing\n",
    "            dec_input = tf.expand_dims(targ[:, t], 1)\n",
    "        \n",
    "    batch_loss = (loss / int(targ.shape[1]))\n",
    "    variables = encoder.trainable_variables + decoder.trainable_variables\n",
    "        \n",
    "    gradients = tape.gradient(loss, variables)\n",
    "    \n",
    "    optimizer.apply_gradients(zip(gradients, variables))\n",
    "    \n",
    "    return batch_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 Batch 0 Loss 2.4510\n",
      "Epoch 1 Batch 100 Loss 1.7852\n",
      "Epoch 1 Batch 200 Loss 1.6879\n",
      "Epoch 1 Batch 300 Loss 1.5609\n",
      "Epoch 1 Batch 400 Loss 1.6729\n",
      "Epoch 1 Loss 1.6774\n",
      "Time taken for 1 epoch 116.02587938308716 sec\n",
      "\n",
      "Epoch 2 Batch 0 Loss 1.6240\n",
      "Epoch 2 Batch 100 Loss 1.5182\n",
      "Epoch 2 Batch 200 Loss 1.7699\n",
      "Epoch 2 Batch 300 Loss 1.3747\n",
      "Epoch 2 Batch 400 Loss 1.3332\n",
      "Epoch 2 Loss 1.5347\n",
      "Time taken for 1 epoch 89.34082388877869 sec\n",
      "\n",
      "Epoch 3 Batch 0 Loss 1.4305\n",
      "Epoch 3 Batch 100 Loss 1.5475\n",
      "Epoch 3 Batch 200 Loss 1.6970\n",
      "Epoch 3 Batch 300 Loss 1.6634\n",
      "Epoch 3 Batch 400 Loss 1.4628\n",
      "Epoch 3 Loss 1.4637\n",
      "Time taken for 1 epoch 87.19940662384033 sec\n",
      "\n",
      "Epoch 4 Batch 0 Loss 1.5956\n",
      "Epoch 4 Batch 100 Loss 1.5673\n",
      "Epoch 4 Batch 200 Loss 1.5086\n",
      "Epoch 4 Batch 300 Loss 1.5579\n",
      "Epoch 4 Batch 400 Loss 1.2083\n",
      "Epoch 4 Loss 1.4087\n",
      "Time taken for 1 epoch 87.71989631652832 sec\n",
      "\n",
      "Epoch 5 Batch 0 Loss 1.3770\n",
      "Epoch 5 Batch 100 Loss 1.2152\n",
      "Epoch 5 Batch 200 Loss 1.4408\n",
      "Epoch 5 Batch 300 Loss 1.3056\n",
      "Epoch 5 Batch 400 Loss 1.5956\n",
      "Epoch 5 Loss 1.3611\n",
      "Time taken for 1 epoch 89.73325562477112 sec\n",
      "\n",
      "Epoch 6 Batch 0 Loss 1.3615\n",
      "Epoch 6 Batch 100 Loss 1.4501\n",
      "Epoch 6 Batch 200 Loss 1.3475\n",
      "Epoch 6 Batch 300 Loss 1.1577\n",
      "Epoch 6 Batch 400 Loss 1.2006\n",
      "Epoch 6 Loss 1.3229\n",
      "Time taken for 1 epoch 92.26588201522827 sec\n",
      "\n",
      "Epoch 7 Batch 0 Loss 1.2775\n",
      "Epoch 7 Batch 100 Loss 1.4089\n",
      "Epoch 7 Batch 200 Loss 1.1201\n",
      "Epoch 7 Batch 300 Loss 1.4323\n",
      "Epoch 7 Batch 400 Loss 1.2582\n",
      "Epoch 7 Loss 1.2912\n",
      "Time taken for 1 epoch 90.99503302574158 sec\n",
      "\n",
      "Epoch 8 Batch 0 Loss 1.2093\n",
      "Epoch 8 Batch 100 Loss 1.2706\n",
      "Epoch 8 Batch 200 Loss 1.2743\n",
      "Epoch 8 Batch 300 Loss 1.2282\n",
      "Epoch 8 Batch 400 Loss 1.5540\n",
      "Epoch 8 Loss 1.2641\n",
      "Time taken for 1 epoch 91.21008539199829 sec\n",
      "\n",
      "Epoch 9 Batch 0 Loss 1.1828\n",
      "Epoch 9 Batch 100 Loss 1.3353\n",
      "Epoch 9 Batch 200 Loss 1.2086\n",
      "Epoch 9 Batch 300 Loss 1.3594\n",
      "Epoch 9 Batch 400 Loss 1.0640\n",
      "Epoch 9 Loss 1.2401\n",
      "Time taken for 1 epoch 91.43283867835999 sec\n",
      "\n",
      "Epoch 10 Batch 0 Loss 1.1060\n",
      "Epoch 10 Batch 100 Loss 1.2441\n",
      "Epoch 10 Batch 200 Loss 1.2408\n",
      "Epoch 10 Batch 300 Loss 1.1574\n",
      "Epoch 10 Batch 400 Loss 1.1831\n",
      "Epoch 10 Loss 1.2177\n",
      "Time taken for 1 epoch 92.46051573753357 sec\n",
      "\n",
      "Epoch 11 Batch 0 Loss 1.1933\n",
      "Epoch 11 Batch 100 Loss 1.3954\n",
      "Epoch 11 Batch 200 Loss 1.0591\n",
      "Epoch 11 Batch 300 Loss 1.3733\n",
      "Epoch 11 Batch 400 Loss 1.3549\n",
      "Epoch 11 Loss 1.1967\n",
      "Time taken for 1 epoch 91.02508997917175 sec\n",
      "\n",
      "Epoch 12 Batch 0 Loss 1.2396\n",
      "Epoch 12 Batch 100 Loss 1.0528\n",
      "Epoch 12 Batch 200 Loss 1.2932\n",
      "Epoch 12 Batch 300 Loss 1.0956\n",
      "Epoch 12 Batch 400 Loss 1.3739\n",
      "Epoch 12 Loss 1.1766\n",
      "Time taken for 1 epoch 90.06372094154358 sec\n",
      "\n",
      "Epoch 13 Batch 0 Loss 1.0647\n",
      "Epoch 13 Batch 100 Loss 1.1839\n",
      "Epoch 13 Batch 200 Loss 1.0570\n",
      "Epoch 13 Batch 300 Loss 1.3302\n",
      "Epoch 13 Batch 400 Loss 1.1261\n",
      "Epoch 13 Loss 1.1571\n",
      "Time taken for 1 epoch 88.92660593986511 sec\n",
      "\n",
      "Epoch 14 Batch 0 Loss 0.9910\n",
      "Epoch 14 Batch 100 Loss 0.9657\n",
      "Epoch 14 Batch 200 Loss 1.2368\n",
      "Epoch 14 Batch 300 Loss 1.0760\n",
      "Epoch 14 Batch 400 Loss 1.1080\n",
      "Epoch 14 Loss 1.1380\n",
      "Time taken for 1 epoch 90.3012912273407 sec\n",
      "\n",
      "Epoch 15 Batch 0 Loss 1.0000\n",
      "Epoch 15 Batch 100 Loss 1.1324\n",
      "Epoch 15 Batch 200 Loss 1.1651\n",
      "Epoch 15 Batch 300 Loss 1.0952\n",
      "Epoch 15 Batch 400 Loss 1.1376\n",
      "Epoch 15 Loss 1.1196\n",
      "Time taken for 1 epoch 88.99976301193237 sec\n",
      "\n",
      "Epoch 16 Batch 0 Loss 1.1238\n",
      "Epoch 16 Batch 100 Loss 1.0031\n",
      "Epoch 16 Batch 200 Loss 1.1475\n",
      "Epoch 16 Batch 300 Loss 1.1700\n",
      "Epoch 16 Batch 400 Loss 1.2169\n",
      "Epoch 16 Loss 1.1010\n",
      "Time taken for 1 epoch 91.35719513893127 sec\n",
      "\n",
      "Epoch 17 Batch 0 Loss 1.2208\n",
      "Epoch 17 Batch 100 Loss 1.0888\n",
      "Epoch 17 Batch 200 Loss 1.1518\n",
      "Epoch 17 Batch 300 Loss 1.1027\n",
      "Epoch 17 Batch 400 Loss 1.0662\n",
      "Epoch 17 Loss 1.0833\n",
      "Time taken for 1 epoch 88.2383553981781 sec\n",
      "\n",
      "Epoch 18 Batch 0 Loss 1.1370\n",
      "Epoch 18 Batch 100 Loss 0.8921\n",
      "Epoch 18 Batch 200 Loss 0.8963\n",
      "Epoch 18 Batch 300 Loss 1.0695\n",
      "Epoch 18 Batch 400 Loss 1.0348\n",
      "Epoch 18 Loss 1.0661\n",
      "Time taken for 1 epoch 91.29691529273987 sec\n",
      "\n",
      "Epoch 19 Batch 0 Loss 1.0966\n",
      "Epoch 19 Batch 100 Loss 1.0519\n",
      "Epoch 19 Batch 200 Loss 1.0671\n",
      "Epoch 19 Batch 300 Loss 0.9039\n",
      "Epoch 19 Batch 400 Loss 1.0999\n",
      "Epoch 19 Loss 1.0488\n",
      "Time taken for 1 epoch 88.74976253509521 sec\n",
      "\n",
      "Epoch 20 Batch 0 Loss 0.8820\n",
      "Epoch 20 Batch 100 Loss 0.9846\n",
      "Epoch 20 Batch 200 Loss 0.9423\n",
      "Epoch 20 Batch 300 Loss 1.0766\n",
      "Epoch 20 Batch 400 Loss 0.9355\n",
      "Epoch 20 Loss 1.0325\n",
      "Time taken for 1 epoch 87.32861185073853 sec\n",
      "\n",
      "Epoch 21 Batch 0 Loss 1.0347\n",
      "Epoch 21 Batch 100 Loss 1.0369\n",
      "Epoch 21 Batch 200 Loss 1.0648\n",
      "Epoch 21 Batch 300 Loss 0.7770\n",
      "Epoch 21 Batch 400 Loss 0.9947\n",
      "Epoch 21 Loss 1.0162\n",
      "Time taken for 1 epoch 88.34226536750793 sec\n",
      "\n",
      "Epoch 22 Batch 0 Loss 0.8447\n",
      "Epoch 22 Batch 100 Loss 1.0944\n",
      "Epoch 22 Batch 200 Loss 1.0679\n",
      "Epoch 22 Batch 300 Loss 0.9564\n",
      "Epoch 22 Batch 400 Loss 0.9772\n",
      "Epoch 22 Loss 1.0006\n",
      "Time taken for 1 epoch 89.75909996032715 sec\n",
      "\n",
      "Epoch 23 Batch 0 Loss 1.0492\n",
      "Epoch 23 Batch 100 Loss 0.9728\n",
      "Epoch 23 Batch 200 Loss 0.9323\n",
      "Epoch 23 Batch 300 Loss 1.0291\n",
      "Epoch 23 Batch 400 Loss 0.9887\n",
      "Epoch 23 Loss 0.9853\n",
      "Time taken for 1 epoch 88.1415627002716 sec\n",
      "\n",
      "Epoch 24 Batch 0 Loss 0.9323\n",
      "Epoch 24 Batch 100 Loss 0.9768\n",
      "Epoch 24 Batch 200 Loss 0.8050\n",
      "Epoch 24 Batch 300 Loss 0.8363\n",
      "Epoch 24 Batch 400 Loss 0.7338\n",
      "Epoch 24 Loss 0.9701\n",
      "Time taken for 1 epoch 89.40855574607849 sec\n",
      "\n",
      "Epoch 25 Batch 0 Loss 1.0443\n",
      "Epoch 25 Batch 100 Loss 0.8327\n",
      "Epoch 25 Batch 200 Loss 0.9803\n",
      "Epoch 25 Batch 300 Loss 0.9726\n",
      "Epoch 25 Batch 400 Loss 0.9825\n",
      "Epoch 25 Loss 0.9555\n",
      "Time taken for 1 epoch 91.56253051757812 sec\n",
      "\n",
      "Epoch 26 Batch 0 Loss 0.9286\n",
      "Epoch 26 Batch 100 Loss 1.0323\n",
      "Epoch 26 Batch 200 Loss 0.9769\n",
      "Epoch 26 Batch 300 Loss 0.9321\n",
      "Epoch 26 Batch 400 Loss 0.9740\n",
      "Epoch 26 Loss 0.9416\n",
      "Time taken for 1 epoch 92.59807968139648 sec\n",
      "\n",
      "Epoch 27 Batch 0 Loss 0.8683\n",
      "Epoch 27 Batch 100 Loss 0.9329\n",
      "Epoch 27 Batch 200 Loss 1.0644\n",
      "Epoch 27 Batch 300 Loss 0.7616\n",
      "Epoch 27 Batch 400 Loss 0.9470\n",
      "Epoch 27 Loss 0.9276\n",
      "Time taken for 1 epoch 92.07777404785156 sec\n",
      "\n",
      "Epoch 28 Batch 0 Loss 0.7913\n",
      "Epoch 28 Batch 100 Loss 0.9497\n",
      "Epoch 28 Batch 200 Loss 0.8406\n",
      "Epoch 28 Batch 300 Loss 0.9258\n",
      "Epoch 28 Batch 400 Loss 0.9718\n",
      "Epoch 28 Loss 0.9140\n",
      "Time taken for 1 epoch 91.9351921081543 sec\n",
      "\n",
      "Epoch 29 Batch 0 Loss 0.9717\n",
      "Epoch 29 Batch 100 Loss 0.8353\n",
      "Epoch 29 Batch 200 Loss 0.8586\n",
      "Epoch 29 Batch 300 Loss 1.0526\n",
      "Epoch 29 Batch 400 Loss 0.8713\n",
      "Epoch 29 Loss 0.9012\n",
      "Time taken for 1 epoch 91.3871099948883 sec\n",
      "\n",
      "Epoch 30 Batch 0 Loss 0.9386\n",
      "Epoch 30 Batch 100 Loss 0.7622\n",
      "Epoch 30 Batch 200 Loss 0.7994\n",
      "Epoch 30 Batch 300 Loss 0.8660\n",
      "Epoch 30 Batch 400 Loss 0.7733\n",
      "Epoch 30 Loss 0.8879\n",
      "Time taken for 1 epoch 91.06531739234924 sec\n",
      "\n",
      "Epoch 31 Batch 0 Loss 0.8097\n",
      "Epoch 31 Batch 100 Loss 0.7938\n",
      "Epoch 31 Batch 200 Loss 0.8371\n",
      "Epoch 31 Batch 300 Loss 0.9311\n",
      "Epoch 31 Batch 400 Loss 0.8078\n",
      "Epoch 31 Loss 0.8752\n",
      "Time taken for 1 epoch 90.14439725875854 sec\n",
      "\n",
      "Epoch 32 Batch 0 Loss 0.8239\n",
      "Epoch 32 Batch 100 Loss 0.9086\n",
      "Epoch 32 Batch 200 Loss 0.9267\n",
      "Epoch 32 Batch 300 Loss 0.7511\n",
      "Epoch 32 Batch 400 Loss 0.6984\n",
      "Epoch 32 Loss 0.8627\n",
      "Time taken for 1 epoch 90.68565440177917 sec\n",
      "\n",
      "Epoch 33 Batch 0 Loss 1.0665\n",
      "Epoch 33 Batch 100 Loss 0.8568\n",
      "Epoch 33 Batch 200 Loss 0.8465\n",
      "Epoch 33 Batch 300 Loss 0.8158\n",
      "Epoch 33 Batch 400 Loss 0.8157\n",
      "Epoch 33 Loss 0.8511\n",
      "Time taken for 1 epoch 89.96452069282532 sec\n",
      "\n",
      "Epoch 34 Batch 0 Loss 0.7347\n",
      "Epoch 34 Batch 100 Loss 0.8092\n",
      "Epoch 34 Batch 200 Loss 0.9063\n",
      "Epoch 34 Batch 300 Loss 0.8204\n",
      "Epoch 34 Batch 400 Loss 0.9144\n",
      "Epoch 34 Loss 0.8393\n",
      "Time taken for 1 epoch 90.52053570747375 sec\n",
      "\n",
      "Epoch 35 Batch 0 Loss 0.8264\n",
      "Epoch 35 Batch 100 Loss 0.7960\n",
      "Epoch 35 Batch 200 Loss 0.8027\n",
      "Epoch 35 Batch 300 Loss 0.8385\n",
      "Epoch 35 Batch 400 Loss 0.7913\n",
      "Epoch 35 Loss 0.8276\n",
      "Time taken for 1 epoch 89.4411129951477 sec\n",
      "\n",
      "Epoch 36 Batch 0 Loss 0.7872\n",
      "Epoch 36 Batch 100 Loss 0.7590\n",
      "Epoch 36 Batch 200 Loss 0.7412\n",
      "Epoch 36 Batch 300 Loss 0.8834\n",
      "Epoch 36 Batch 400 Loss 0.8000\n",
      "Epoch 36 Loss 0.8167\n",
      "Time taken for 1 epoch 91.49013352394104 sec\n",
      "\n",
      "Epoch 37 Batch 0 Loss 0.8237\n",
      "Epoch 37 Batch 100 Loss 0.7303\n",
      "Epoch 37 Batch 200 Loss 0.8475\n",
      "Epoch 37 Batch 300 Loss 1.0124\n",
      "Epoch 37 Batch 400 Loss 0.7249\n",
      "Epoch 37 Loss 0.8058\n",
      "Time taken for 1 epoch 92.27395534515381 sec\n",
      "\n",
      "Epoch 38 Batch 0 Loss 0.8145\n",
      "Epoch 38 Batch 100 Loss 0.7405\n",
      "Epoch 38 Batch 200 Loss 0.8306\n",
      "Epoch 38 Batch 300 Loss 0.7521\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 38 Batch 400 Loss 0.9101\n",
      "Epoch 38 Loss 0.7952\n",
      "Time taken for 1 epoch 92.69184494018555 sec\n",
      "\n",
      "Epoch 39 Batch 0 Loss 0.7493\n",
      "Epoch 39 Batch 100 Loss 0.8005\n",
      "Epoch 39 Batch 200 Loss 0.8133\n",
      "Epoch 39 Batch 300 Loss 0.7640\n",
      "Epoch 39 Batch 400 Loss 0.9139\n",
      "Epoch 39 Loss 0.7844\n",
      "Time taken for 1 epoch 89.68784165382385 sec\n",
      "\n",
      "Epoch 40 Batch 0 Loss 0.6822\n",
      "Epoch 40 Batch 100 Loss 0.7587\n",
      "Epoch 40 Batch 200 Loss 0.6765\n",
      "Epoch 40 Batch 300 Loss 0.7595\n",
      "Epoch 40 Batch 400 Loss 0.7652\n",
      "Epoch 40 Loss 0.7742\n",
      "Time taken for 1 epoch 90.88142848014832 sec\n",
      "\n",
      "Epoch 41 Batch 0 Loss 0.7202\n",
      "Epoch 41 Batch 100 Loss 0.7427\n",
      "Epoch 41 Batch 200 Loss 0.7088\n",
      "Epoch 41 Batch 300 Loss 0.8278\n",
      "Epoch 41 Batch 400 Loss 0.6635\n",
      "Epoch 41 Loss 0.7635\n",
      "Time taken for 1 epoch 92.21189403533936 sec\n",
      "\n",
      "Epoch 42 Batch 0 Loss 0.6391\n",
      "Epoch 42 Batch 100 Loss 0.7857\n",
      "Epoch 42 Batch 200 Loss 0.7312\n",
      "Epoch 42 Batch 300 Loss 0.7104\n",
      "Epoch 42 Batch 400 Loss 0.8752\n",
      "Epoch 42 Loss 0.7540\n",
      "Time taken for 1 epoch 91.06829953193665 sec\n",
      "\n",
      "Epoch 43 Batch 0 Loss 0.7616\n",
      "Epoch 43 Batch 100 Loss 0.7239\n",
      "Epoch 43 Batch 200 Loss 0.7167\n",
      "Epoch 43 Batch 300 Loss 0.6577\n",
      "Epoch 43 Batch 400 Loss 0.8100\n",
      "Epoch 43 Loss 0.7446\n",
      "Time taken for 1 epoch 90.96114993095398 sec\n",
      "\n",
      "Epoch 44 Batch 0 Loss 0.7413\n",
      "Epoch 44 Batch 100 Loss 0.7010\n",
      "Epoch 44 Batch 200 Loss 0.6373\n",
      "Epoch 44 Batch 300 Loss 0.7007\n",
      "Epoch 44 Batch 400 Loss 0.7214\n",
      "Epoch 44 Loss 0.7355\n",
      "Time taken for 1 epoch 89.71075868606567 sec\n",
      "\n",
      "Epoch 45 Batch 0 Loss 0.6851\n",
      "Epoch 45 Batch 100 Loss 0.7541\n",
      "Epoch 45 Batch 200 Loss 0.6396\n",
      "Epoch 45 Batch 300 Loss 0.8163\n",
      "Epoch 45 Batch 400 Loss 0.7358\n",
      "Epoch 45 Loss 0.7260\n",
      "Time taken for 1 epoch 89.1763060092926 sec\n",
      "\n",
      "Epoch 46 Batch 0 Loss 0.6611\n",
      "Epoch 46 Batch 100 Loss 0.6781\n",
      "Epoch 46 Batch 200 Loss 0.6660\n",
      "Epoch 46 Batch 300 Loss 0.6699\n",
      "Epoch 46 Batch 400 Loss 0.6403\n",
      "Epoch 46 Loss 0.7170\n",
      "Time taken for 1 epoch 90.34504342079163 sec\n",
      "\n",
      "Epoch 47 Batch 0 Loss 0.7517\n",
      "Epoch 47 Batch 100 Loss 0.8062\n",
      "Epoch 47 Batch 200 Loss 0.7613\n",
      "Epoch 47 Batch 300 Loss 0.7776\n",
      "Epoch 47 Batch 400 Loss 0.7534\n",
      "Epoch 47 Loss 0.7078\n",
      "Time taken for 1 epoch 88.94074726104736 sec\n",
      "\n",
      "Epoch 48 Batch 0 Loss 0.7683\n",
      "Epoch 48 Batch 100 Loss 0.7063\n",
      "Epoch 48 Batch 200 Loss 0.6855\n",
      "Epoch 48 Batch 300 Loss 0.7322\n",
      "Epoch 48 Batch 400 Loss 0.6730\n",
      "Epoch 48 Loss 0.6991\n",
      "Time taken for 1 epoch 90.14607954025269 sec\n",
      "\n",
      "Epoch 49 Batch 0 Loss 0.7467\n",
      "Epoch 49 Batch 100 Loss 0.7684\n",
      "Epoch 49 Batch 200 Loss 0.7696\n",
      "Epoch 49 Batch 300 Loss 0.5863\n",
      "Epoch 49 Batch 400 Loss 0.7590\n",
      "Epoch 49 Loss 0.6912\n",
      "Time taken for 1 epoch 89.1619610786438 sec\n",
      "\n",
      "Epoch 50 Batch 0 Loss 0.4997\n",
      "Epoch 50 Batch 100 Loss 0.6688\n",
      "Epoch 50 Batch 200 Loss 0.7363\n",
      "Epoch 50 Batch 300 Loss 0.6720\n",
      "Epoch 50 Batch 400 Loss 0.7144\n",
      "Epoch 50 Loss 0.6821\n",
      "Time taken for 1 epoch 92.58323168754578 sec\n",
      "\n",
      "Epoch 51 Batch 0 Loss 0.5933\n",
      "Epoch 51 Batch 100 Loss 0.7048\n",
      "Epoch 51 Batch 200 Loss 0.6441\n",
      "Epoch 51 Batch 300 Loss 0.7529\n",
      "Epoch 51 Batch 400 Loss 0.7003\n",
      "Epoch 51 Loss 0.6743\n",
      "Time taken for 1 epoch 91.28693628311157 sec\n",
      "\n",
      "Epoch 52 Batch 0 Loss 0.6406\n",
      "Epoch 52 Batch 100 Loss 0.7193\n",
      "Epoch 52 Batch 200 Loss 0.6018\n",
      "Epoch 52 Batch 300 Loss 0.6975\n",
      "Epoch 52 Batch 400 Loss 0.6399\n",
      "Epoch 52 Loss 0.6660\n",
      "Time taken for 1 epoch 92.90141224861145 sec\n",
      "\n",
      "Epoch 53 Batch 0 Loss 0.6044\n",
      "Epoch 53 Batch 100 Loss 0.6447\n",
      "Epoch 53 Batch 200 Loss 0.7373\n",
      "Epoch 53 Batch 300 Loss 0.6403\n",
      "Epoch 53 Batch 400 Loss 0.5300\n",
      "Epoch 53 Loss 0.6584\n",
      "Time taken for 1 epoch 89.14400386810303 sec\n",
      "\n",
      "Epoch 54 Batch 0 Loss 0.6255\n",
      "Epoch 54 Batch 100 Loss 0.6759\n",
      "Epoch 54 Batch 200 Loss 0.6079\n",
      "Epoch 54 Batch 300 Loss 0.7076\n",
      "Epoch 54 Batch 400 Loss 0.6008\n",
      "Epoch 54 Loss 0.6508\n",
      "Time taken for 1 epoch 91.71725702285767 sec\n",
      "\n",
      "Epoch 55 Batch 0 Loss 0.5163\n",
      "Epoch 55 Batch 100 Loss 0.6158\n",
      "Epoch 55 Batch 200 Loss 0.7105\n",
      "Epoch 55 Batch 300 Loss 0.4854\n",
      "Epoch 55 Batch 400 Loss 0.5903\n",
      "Epoch 55 Loss 0.6439\n",
      "Time taken for 1 epoch 89.6081235408783 sec\n",
      "\n",
      "Epoch 56 Batch 0 Loss 0.5849\n",
      "Epoch 56 Batch 100 Loss 0.5927\n",
      "Epoch 56 Batch 200 Loss 0.5236\n",
      "Epoch 56 Batch 300 Loss 0.7548\n",
      "Epoch 56 Batch 400 Loss 0.6828\n",
      "Epoch 56 Loss 0.6365\n",
      "Time taken for 1 epoch 90.35385823249817 sec\n",
      "\n",
      "Epoch 57 Batch 0 Loss 0.5247\n",
      "Epoch 57 Batch 100 Loss 0.6310\n",
      "Epoch 57 Batch 200 Loss 0.6371\n",
      "Epoch 57 Batch 300 Loss 0.6805\n",
      "Epoch 57 Batch 400 Loss 0.5849\n",
      "Epoch 57 Loss 0.6288\n",
      "Time taken for 1 epoch 89.80049705505371 sec\n",
      "\n",
      "Epoch 58 Batch 0 Loss 0.5578\n",
      "Epoch 58 Batch 100 Loss 0.6182\n",
      "Epoch 58 Batch 200 Loss 0.5711\n",
      "Epoch 58 Batch 300 Loss 0.6533\n",
      "Epoch 58 Batch 400 Loss 0.5992\n",
      "Epoch 58 Loss 0.6214\n",
      "Time taken for 1 epoch 93.1113953590393 sec\n",
      "\n",
      "Epoch 59 Batch 0 Loss 0.6060\n",
      "Epoch 59 Batch 100 Loss 0.6738\n",
      "Epoch 59 Batch 200 Loss 0.6950\n",
      "Epoch 59 Batch 300 Loss 0.5847\n",
      "Epoch 59 Batch 400 Loss 0.7621\n",
      "Epoch 59 Loss 0.6153\n",
      "Time taken for 1 epoch 92.37022376060486 sec\n",
      "\n",
      "Epoch 60 Batch 0 Loss 0.4749\n",
      "Epoch 60 Batch 100 Loss 0.6510\n",
      "Epoch 60 Batch 200 Loss 0.7352\n",
      "Epoch 60 Batch 300 Loss 0.6193\n",
      "Epoch 60 Batch 400 Loss 0.6129\n",
      "Epoch 60 Loss 0.6091\n",
      "Time taken for 1 epoch 93.42295956611633 sec\n",
      "\n",
      "Epoch 61 Batch 0 Loss 0.6392\n",
      "Epoch 61 Batch 100 Loss 0.5542\n",
      "Epoch 61 Batch 200 Loss 0.6909\n",
      "Epoch 61 Batch 300 Loss 0.5398\n",
      "Epoch 61 Batch 400 Loss 0.6562\n",
      "Epoch 61 Loss 0.6020\n",
      "Time taken for 1 epoch 93.95256233215332 sec\n",
      "\n",
      "Epoch 62 Batch 0 Loss 0.5826\n",
      "Epoch 62 Batch 100 Loss 0.5500\n",
      "Epoch 62 Batch 200 Loss 0.6956\n",
      "Epoch 62 Batch 300 Loss 0.6154\n",
      "Epoch 62 Batch 400 Loss 0.5199\n",
      "Epoch 62 Loss 0.5955\n",
      "Time taken for 1 epoch 92.52437019348145 sec\n",
      "\n",
      "Epoch 63 Batch 0 Loss 0.6055\n",
      "Epoch 63 Batch 100 Loss 0.5417\n",
      "Epoch 63 Batch 200 Loss 0.5681\n",
      "Epoch 63 Batch 300 Loss 0.6024\n",
      "Epoch 63 Batch 400 Loss 0.5726\n",
      "Epoch 63 Loss 0.5887\n",
      "Time taken for 1 epoch 91.17393946647644 sec\n",
      "\n",
      "Epoch 64 Batch 0 Loss 0.5925\n",
      "Epoch 64 Batch 100 Loss 0.6486\n",
      "Epoch 64 Batch 200 Loss 0.5500\n",
      "Epoch 64 Batch 300 Loss 0.5923\n",
      "Epoch 64 Batch 400 Loss 0.5582\n",
      "Epoch 64 Loss 0.5833\n",
      "Time taken for 1 epoch 92.22908616065979 sec\n",
      "\n",
      "Epoch 65 Batch 0 Loss 0.5988\n",
      "Epoch 65 Batch 100 Loss 0.5309\n",
      "Epoch 65 Batch 200 Loss 0.6220\n",
      "Epoch 65 Batch 300 Loss 0.5731\n",
      "Epoch 65 Batch 400 Loss 0.6094\n",
      "Epoch 65 Loss 0.5766\n",
      "Time taken for 1 epoch 92.3463888168335 sec\n",
      "\n",
      "Epoch 66 Batch 0 Loss 0.5066\n",
      "Epoch 66 Batch 100 Loss 0.5792\n",
      "Epoch 66 Batch 200 Loss 0.5961\n",
      "Epoch 66 Batch 300 Loss 0.4681\n",
      "Epoch 66 Batch 400 Loss 0.7120\n",
      "Epoch 66 Loss 0.5711\n",
      "Time taken for 1 epoch 90.32150292396545 sec\n",
      "\n",
      "Epoch 67 Batch 0 Loss 0.6081\n",
      "Epoch 67 Batch 100 Loss 0.5689\n",
      "Epoch 67 Batch 200 Loss 0.5744\n",
      "Epoch 67 Batch 300 Loss 0.5178\n",
      "Epoch 67 Batch 400 Loss 0.6942\n",
      "Epoch 67 Loss 0.5695\n",
      "Time taken for 1 epoch 91.49492645263672 sec\n",
      "\n",
      "Epoch 68 Batch 0 Loss 0.4900\n",
      "Epoch 68 Batch 100 Loss 0.6202\n",
      "Epoch 68 Batch 200 Loss 0.5603\n",
      "Epoch 68 Batch 300 Loss 0.5745\n",
      "Epoch 68 Batch 400 Loss 0.6149\n",
      "Epoch 68 Loss 0.5597\n",
      "Time taken for 1 epoch 91.28971290588379 sec\n",
      "\n",
      "Epoch 69 Batch 0 Loss 0.5659\n",
      "Epoch 69 Batch 100 Loss 0.5309\n",
      "Epoch 69 Batch 200 Loss 0.6098\n",
      "Epoch 69 Batch 300 Loss 0.6095\n",
      "Epoch 69 Batch 400 Loss 0.6015\n",
      "Epoch 69 Loss 0.5537\n",
      "Time taken for 1 epoch 92.90086030960083 sec\n",
      "\n",
      "Epoch 70 Batch 0 Loss 0.5577\n",
      "Epoch 70 Batch 100 Loss 0.5889\n",
      "Epoch 70 Batch 200 Loss 0.5361\n",
      "Epoch 70 Batch 300 Loss 0.6039\n",
      "Epoch 70 Batch 400 Loss 0.5006\n",
      "Epoch 70 Loss 0.5479\n",
      "Time taken for 1 epoch 94.21548628807068 sec\n",
      "\n",
      "Epoch 71 Batch 0 Loss 0.4492\n",
      "Epoch 71 Batch 100 Loss 0.5063\n",
      "Epoch 71 Batch 200 Loss 0.6349\n",
      "Epoch 71 Batch 300 Loss 0.5103\n",
      "Epoch 71 Batch 400 Loss 0.6403\n",
      "Epoch 71 Loss 0.5424\n",
      "Time taken for 1 epoch 88.82743453979492 sec\n",
      "\n",
      "Epoch 72 Batch 0 Loss 0.5210\n",
      "Epoch 72 Batch 100 Loss 0.5976\n",
      "Epoch 72 Batch 200 Loss 0.5520\n",
      "Epoch 72 Batch 300 Loss 0.4793\n",
      "Epoch 72 Batch 400 Loss 0.5445\n",
      "Epoch 72 Loss 0.5376\n",
      "Time taken for 1 epoch 93.18333077430725 sec\n",
      "\n",
      "Epoch 73 Batch 0 Loss 0.4961\n",
      "Epoch 73 Batch 100 Loss 0.4824\n",
      "Epoch 73 Batch 200 Loss 0.5840\n",
      "Epoch 73 Batch 300 Loss 0.5287\n",
      "Epoch 73 Batch 400 Loss 0.6039\n",
      "Epoch 73 Loss 0.5358\n",
      "Time taken for 1 epoch 90.53068113327026 sec\n",
      "\n",
      "Epoch 74 Batch 0 Loss 0.5311\n",
      "Epoch 74 Batch 100 Loss 0.5195\n",
      "Epoch 74 Batch 200 Loss 0.5005\n",
      "Epoch 74 Batch 300 Loss 0.4596\n",
      "Epoch 74 Batch 400 Loss 0.4797\n",
      "Epoch 74 Loss 0.5279\n",
      "Time taken for 1 epoch 91.94338250160217 sec\n",
      "\n",
      "Epoch 75 Batch 0 Loss 0.3994\n",
      "Epoch 75 Batch 100 Loss 0.4856\n",
      "Epoch 75 Batch 200 Loss 0.5754\n",
      "Epoch 75 Batch 300 Loss 0.5715\n",
      "Epoch 75 Batch 400 Loss 0.5533\n",
      "Epoch 75 Loss 0.5235\n",
      "Time taken for 1 epoch 90.54018926620483 sec\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 76 Batch 0 Loss 0.4234\n",
      "Epoch 76 Batch 100 Loss 0.5668\n",
      "Epoch 76 Batch 200 Loss 0.4980\n",
      "Epoch 76 Batch 300 Loss 0.5389\n",
      "Epoch 76 Batch 400 Loss 0.4809\n",
      "Epoch 76 Loss 0.5175\n",
      "Time taken for 1 epoch 91.792245388031 sec\n",
      "\n",
      "Epoch 77 Batch 0 Loss 0.5164\n",
      "Epoch 77 Batch 100 Loss 0.5051\n",
      "Epoch 77 Batch 200 Loss 0.4824\n",
      "Epoch 77 Batch 300 Loss 0.4684\n",
      "Epoch 77 Batch 400 Loss 0.4107\n",
      "Epoch 77 Loss 0.5160\n",
      "Time taken for 1 epoch 89.91190481185913 sec\n",
      "\n",
      "Epoch 78 Batch 0 Loss 0.4616\n",
      "Epoch 78 Batch 100 Loss 0.4108\n",
      "Epoch 78 Batch 200 Loss 0.4901\n",
      "Epoch 78 Batch 300 Loss 0.5315\n",
      "Epoch 78 Batch 400 Loss 0.5347\n",
      "Epoch 78 Loss 0.5102\n",
      "Time taken for 1 epoch 91.0253746509552 sec\n",
      "\n",
      "Epoch 79 Batch 0 Loss 0.4768\n",
      "Epoch 79 Batch 100 Loss 0.5867\n",
      "Epoch 79 Batch 200 Loss 0.4539\n",
      "Epoch 79 Batch 300 Loss 0.5167\n",
      "Epoch 79 Batch 400 Loss 0.4715\n",
      "Epoch 79 Loss 0.5029\n",
      "Time taken for 1 epoch 89.02837800979614 sec\n",
      "\n",
      "Epoch 80 Batch 0 Loss 0.4522\n",
      "Epoch 80 Batch 100 Loss 0.5198\n",
      "Epoch 80 Batch 200 Loss 0.5425\n",
      "Epoch 80 Batch 300 Loss 0.5372\n",
      "Epoch 80 Batch 400 Loss 0.5285\n",
      "Epoch 80 Loss 0.4994\n",
      "Time taken for 1 epoch 89.30011248588562 sec\n",
      "\n",
      "Epoch 81 Batch 0 Loss 0.5225\n",
      "Epoch 81 Batch 100 Loss 0.5236\n",
      "Epoch 81 Batch 200 Loss 0.4292\n",
      "Epoch 81 Batch 300 Loss 0.5277\n",
      "Epoch 81 Batch 400 Loss 0.4708\n",
      "Epoch 81 Loss 0.4949\n",
      "Time taken for 1 epoch 89.70644044876099 sec\n",
      "\n",
      "Epoch 82 Batch 0 Loss 0.4881\n",
      "Epoch 82 Batch 100 Loss 0.4674\n",
      "Epoch 82 Batch 200 Loss 0.5093\n",
      "Epoch 82 Batch 300 Loss 0.4386\n",
      "Epoch 82 Batch 400 Loss 0.5237\n",
      "Epoch 82 Loss 0.4911\n",
      "Time taken for 1 epoch 89.38204622268677 sec\n",
      "\n",
      "Epoch 83 Batch 0 Loss 0.5186\n",
      "Epoch 83 Batch 100 Loss 0.4365\n",
      "Epoch 83 Batch 200 Loss 0.5089\n",
      "Epoch 83 Batch 300 Loss 0.4923\n",
      "Epoch 83 Batch 400 Loss 0.5350\n",
      "Epoch 83 Loss 0.4871\n",
      "Time taken for 1 epoch 87.05247020721436 sec\n",
      "\n",
      "Epoch 84 Batch 0 Loss 0.4967\n",
      "Epoch 84 Batch 100 Loss 0.4793\n",
      "Epoch 84 Batch 200 Loss 0.5514\n",
      "Epoch 84 Batch 300 Loss 0.5168\n",
      "Epoch 84 Batch 400 Loss 0.5462\n",
      "Epoch 84 Loss 0.4823\n",
      "Time taken for 1 epoch 88.50480365753174 sec\n",
      "\n",
      "Epoch 85 Batch 0 Loss 0.4765\n",
      "Epoch 85 Batch 100 Loss 0.4795\n",
      "Epoch 85 Batch 200 Loss 0.4670\n",
      "Epoch 85 Batch 300 Loss 0.3821\n",
      "Epoch 85 Batch 400 Loss 0.4962\n",
      "Epoch 85 Loss 0.4784\n",
      "Time taken for 1 epoch 89.50660109519958 sec\n",
      "\n",
      "Epoch 86 Batch 0 Loss 0.4398\n",
      "Epoch 86 Batch 100 Loss 0.5203\n",
      "Epoch 86 Batch 200 Loss 0.4520\n",
      "Epoch 86 Batch 300 Loss 0.4523\n",
      "Epoch 86 Batch 400 Loss 0.4612\n",
      "Epoch 86 Loss 0.4752\n",
      "Time taken for 1 epoch 90.86906719207764 sec\n",
      "\n",
      "Epoch 87 Batch 0 Loss 0.5261\n",
      "Epoch 87 Batch 100 Loss 0.4037\n",
      "Epoch 87 Batch 200 Loss 0.4963\n",
      "Epoch 87 Batch 300 Loss 0.4993\n",
      "Epoch 87 Batch 400 Loss 0.5178\n",
      "Epoch 87 Loss 0.4709\n",
      "Time taken for 1 epoch 89.65723919868469 sec\n",
      "\n",
      "Epoch 88 Batch 0 Loss 0.4626\n",
      "Epoch 88 Batch 100 Loss 0.4269\n",
      "Epoch 88 Batch 200 Loss 0.4863\n",
      "Epoch 88 Batch 300 Loss 0.4899\n",
      "Epoch 88 Batch 400 Loss 0.4930\n",
      "Epoch 88 Loss 0.4680\n",
      "Time taken for 1 epoch 90.78694653511047 sec\n",
      "\n",
      "Epoch 89 Batch 0 Loss 0.3364\n",
      "Epoch 89 Batch 100 Loss 0.5079\n",
      "Epoch 89 Batch 200 Loss 0.3539\n",
      "Epoch 89 Batch 300 Loss 0.4732\n",
      "Epoch 89 Batch 400 Loss 0.3812\n",
      "Epoch 89 Loss 0.4637\n",
      "Time taken for 1 epoch 90.63450121879578 sec\n",
      "\n",
      "Epoch 90 Batch 0 Loss 0.4215\n",
      "Epoch 90 Batch 100 Loss 0.3153\n",
      "Epoch 90 Batch 200 Loss 0.5061\n",
      "Epoch 90 Batch 300 Loss 0.4583\n",
      "Epoch 90 Batch 400 Loss 0.4707\n",
      "Epoch 90 Loss 0.4598\n",
      "Time taken for 1 epoch 93.1877338886261 sec\n",
      "\n",
      "Epoch 91 Batch 0 Loss 0.5164\n",
      "Epoch 91 Batch 100 Loss 0.3980\n",
      "Epoch 91 Batch 200 Loss 0.5167\n",
      "Epoch 91 Batch 300 Loss 0.5197\n",
      "Epoch 91 Batch 400 Loss 0.4572\n",
      "Epoch 91 Loss 0.4562\n",
      "Time taken for 1 epoch 88.10193157196045 sec\n",
      "\n",
      "Epoch 92 Batch 0 Loss 0.5192\n",
      "Epoch 92 Batch 100 Loss 0.3888\n",
      "Epoch 92 Batch 200 Loss 0.4414\n",
      "Epoch 92 Batch 300 Loss 0.3892\n",
      "Epoch 92 Batch 400 Loss 0.4194\n",
      "Epoch 92 Loss 0.4532\n",
      "Time taken for 1 epoch 89.2917993068695 sec\n",
      "\n",
      "Epoch 93 Batch 0 Loss 0.3926\n",
      "Epoch 93 Batch 100 Loss 0.4884\n",
      "Epoch 93 Batch 200 Loss 0.5171\n",
      "Epoch 93 Batch 300 Loss 0.4772\n",
      "Epoch 93 Batch 400 Loss 0.3973\n",
      "Epoch 93 Loss 0.4496\n",
      "Time taken for 1 epoch 90.73948431015015 sec\n",
      "\n",
      "Epoch 94 Batch 0 Loss 0.4018\n",
      "Epoch 94 Batch 100 Loss 0.5134\n",
      "Epoch 94 Batch 200 Loss 0.4743\n",
      "Epoch 94 Batch 300 Loss 0.4515\n",
      "Epoch 94 Batch 400 Loss 0.3265\n",
      "Epoch 94 Loss 0.4458\n",
      "Time taken for 1 epoch 91.23586010932922 sec\n",
      "\n",
      "Epoch 95 Batch 0 Loss 0.4110\n",
      "Epoch 95 Batch 100 Loss 0.3694\n",
      "Epoch 95 Batch 200 Loss 0.4324\n",
      "Epoch 95 Batch 300 Loss 0.4584\n",
      "Epoch 95 Batch 400 Loss 0.4175\n",
      "Epoch 95 Loss 0.4421\n",
      "Time taken for 1 epoch 90.18427681922913 sec\n",
      "\n",
      "Epoch 96 Batch 0 Loss 0.4123\n",
      "Epoch 96 Batch 100 Loss 0.3314\n",
      "Epoch 96 Batch 200 Loss 0.4226\n",
      "Epoch 96 Batch 300 Loss 0.5536\n",
      "Epoch 96 Batch 400 Loss 0.4764\n",
      "Epoch 96 Loss 0.4398\n",
      "Time taken for 1 epoch 89.42188882827759 sec\n",
      "\n",
      "Epoch 97 Batch 0 Loss 0.4597\n",
      "Epoch 97 Batch 100 Loss 0.3379\n",
      "Epoch 97 Batch 200 Loss 0.4946\n",
      "Epoch 97 Batch 300 Loss 0.4900\n",
      "Epoch 97 Batch 400 Loss 0.4892\n",
      "Epoch 97 Loss 0.4353\n",
      "Time taken for 1 epoch 90.81108450889587 sec\n",
      "\n",
      "Epoch 98 Batch 0 Loss 0.4542\n",
      "Epoch 98 Batch 100 Loss 0.4577\n",
      "Epoch 98 Batch 200 Loss 0.4252\n",
      "Epoch 98 Batch 300 Loss 0.4302\n",
      "Epoch 98 Batch 400 Loss 0.5202\n",
      "Epoch 98 Loss 0.4342\n",
      "Time taken for 1 epoch 87.63887429237366 sec\n",
      "\n",
      "Epoch 99 Batch 0 Loss 0.4366\n",
      "Epoch 99 Batch 100 Loss 0.4680\n",
      "Epoch 99 Batch 200 Loss 0.3507\n",
      "Epoch 99 Batch 300 Loss 0.4367\n",
      "Epoch 99 Batch 400 Loss 0.4533\n",
      "Epoch 99 Loss 0.4289\n",
      "Time taken for 1 epoch 87.5113434791565 sec\n",
      "\n",
      "Epoch 100 Batch 0 Loss 0.4673\n",
      "Epoch 100 Batch 100 Loss 0.4748\n",
      "Epoch 100 Batch 200 Loss 0.3857\n",
      "Epoch 100 Batch 300 Loss 0.4323\n",
      "Epoch 100 Batch 400 Loss 0.4544\n",
      "Epoch 100 Loss 0.4252\n",
      "Time taken for 1 epoch 87.93658494949341 sec\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 100\n",
    "import gc\n",
    "gc.collect()\n",
    "for epoch in range(EPOCHS):\n",
    "    start = time.time()\n",
    "    \n",
    "    enc_hidden = encoder.initialize_hidden_state()\n",
    "    total_loss = 0\n",
    "    \n",
    "    for (batch, (inp, targ)) in enumerate(dataset.take(steps_per_epoch)):\n",
    "        batch_loss = train_step(inp, targ, enc_hidden)\n",
    "        total_loss += batch_loss\n",
    "        \n",
    "        if batch % 100 == 0:\n",
    "            print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                                         batch,\n",
    "                                                         batch_loss.numpy()))\n",
    "    # saving (checkpoint) the model every 2 epochs\n",
    "    \n",
    "    if (epoch + 1) % 2 == 0:\n",
    "        checkpoint.save(file_prefix = checkpoint_prefix)\n",
    "    \n",
    "    print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                      total_loss / steps_per_epoch))\n",
    "    \n",
    "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(sentence):\n",
    "    attention_plot = np.zeros((max_len_out, max_len_inp))\n",
    "    \n",
    "    \n",
    "    inputs = [inp_lang.word_index[i] for i in sentence.split(' ')]\n",
    "    \n",
    "    \n",
    "    inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],\n",
    "                                                         maxlen=max_len_inp,\n",
    "                                                         padding='post')\n",
    "    \n",
    "    inputs = tf.convert_to_tensor(inputs)\n",
    "    \n",
    "    result = ''\n",
    "    \n",
    "    hidden = [tf.zeros((1, units))]\n",
    "    \n",
    "    enc_out, enc_hidden = encoder(inputs, hidden)\n",
    "    \n",
    "    dec_hidden = enc_hidden\n",
    "    \n",
    "    dec_input = tf.expand_dims([out_lang.word_index['<start>']], 0)\n",
    "    \n",
    "    for t in range(max_len_out):\n",
    "        predictions, dec_hidden, attention_weights = decoder(dec_input,\n",
    "                                                         dec_hidden,\n",
    "                                                         enc_out)\n",
    "\n",
    "        # storing the attention weights to plot later on\n",
    "        attention_weights = tf.reshape(attention_weights, (-1, ))\n",
    "        attention_plot[t] = attention_weights.numpy()\n",
    "\n",
    "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
    "\n",
    "        result += out_lang.index_word[predicted_id] + ' '\n",
    "\n",
    "        if out_lang.index_word[predicted_id] == '<end>':\n",
    "            return result, sentence, attention_plot\n",
    "\n",
    "        # the predicted ID is fed back into the model\n",
    "        dec_input = tf.expand_dims([predicted_id], 0)\n",
    "\n",
    "    return result, sentence, attention_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_attention(attention, sentence, predicted_sentence):\n",
    "    fig = plt.figure(figsize=(10,10))\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.matshow(attention, cmap='viridis')\n",
    "    \n",
    "    fontdict = {'fontsize': 14}\n",
    "    \n",
    "    ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
    "    \n",
    "    ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
    "    \n",
    "    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "    \n",
    "    \n",
    "    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "    \n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate(sentence):\n",
    "    sentence=sentence.lower()\n",
    "    result, sentence, attention_plot = evaluate(sentence)\n",
    "    \n",
    "    print('Input: %s' % (sentence))\n",
    "    \n",
    "    print('Predicted translation: {}'.format(result))\n",
    "    \n",
    "    attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n",
    "    \n",
    "#     plot_attention(attention_plot, sentence.split(' '), result.split(' '))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x1a1da979ec8>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checkpoint.restore(tf.train.latest_checkpoint('./training_checkpoints1'))\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: <start> what are you doing <end>\n",
      "Predicted translation: i don t know i m a real berserk <end> \n"
     ]
    }
   ],
   "source": [
    "translate(u'<start> what are you doing <end>')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}