nwp_lstm_multi_word.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Currently Loaded Modules:\n",
      "  1) anaconda3/5.1.0-gcc/8.3.1     4) cudnn/8.0.0.180-11.0-linux-x64-gcc/7.5.0\n",
      "  2) anaconda3/2019.10-gcc/8.3.1   5) openjdk/1.8.0_222-b10-gcc/8.3.1\n",
      "  3) cuda/11.0.3-gcc/7.5.0         6) hadoop/3.2.1-gcc/8.3.1\n",
      "\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "!module list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.4.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Input, LSTM, Dense, Embedding, Flatten, TimeDistributed, Dropout, LSTMCell, RNN, Bidirectional, Concatenate, Layer\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "from tensorflow.python.keras.utils import tf_utils\n",
    "from tensorflow.keras import backend as K\n",
    "\n",
    "import unicodedata\n",
    "import re\n",
    "import os\n",
    "import time\n",
    "import shutil\n",
    "import requests\n",
    "import tarfile\n",
    "import glob\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string, os\n",
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = \"Project_CodeNet_LangClass.tar.gz\"\n",
    "data_url = f\"https://dax-cdn.cdn.appdomain.cloud/dax-project-codenet/1.0.0/{file_name}\"\n",
    "\n",
    "# Download tar archive to local disk\n",
    "with open(file_name, \"wb\") as f:\n",
    "    f.write(requests.get(data_url).content)\n",
    "    \n",
    "# Extract contents of archive to local disk\n",
    "if os.path.exists(\"data\"):\n",
    "    shutil.rmtree(\"data\")    \n",
    "with tarfile.open(file_name) as tfile:\n",
    "    tfile.extractall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data:\n",
      "test  train\n",
      "\n",
      "data/train:\n",
      "C  C#  C++  D  Haskell\tJava  JavaScript  PHP  Python  Rust\n"
     ]
    }
   ],
   "source": [
    "!ls data data/train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "langs = [\n",
    "  \"C\",\n",
    "  \"C#\",\n",
    "  \"C++\",\n",
    "  \"D\",\n",
    "  \"Haskell\",\n",
    "  \"Java\",\n",
    "  \"JavaScript\",\n",
    "  \"PHP\",\n",
    "  \"Python\",\n",
    "  \"Rust\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import argparse\n",
    "import re\n",
    "from tokenize import tokenize, untokenize, COMMENT, STRING, NEWLINE, ENCODING, ENDMARKER, NL, INDENT, NUMBER\n",
    "from io import BytesIO\n",
    "import json\n",
    "\n",
    "lits = json.load(open(\"literals.json\"))\n",
    "\n",
    "def process_string(token, special_chars={\" \": \"U+0020\", \",\": \"U+002C\"}):\n",
    "    str_quote_options = [\"'''\", '\"\"\"', \"'\", '\"']\n",
    "    start_quote = \"\"\n",
    "    end_quote = \"\"\n",
    "    qualifier_regex = r\"^[a-z]+\"\n",
    "    qualifier_match = re.search(qualifier_regex, token)\n",
    "    # string qualifiers like 'r' for regex, 'f' for formatted string, 'b' for bytes, 'u' for unicode, etc (or combination of them)\n",
    "    qualifier = \"\" if not qualifier_match else qualifier_match[0]\n",
    "    # token string without qualifiers\n",
    "    token_string = re.sub(qualifier_regex, \"\", token)\n",
    "    # string literal without quotes\n",
    "    str_lit = token_string\n",
    "    for q in str_quote_options:\n",
    "        if token_string.startswith(q):\n",
    "            start_quote = q\n",
    "            str_lit = str_lit[len(q) :]\n",
    "            if token_string.endswith(q):\n",
    "                end_quote = q\n",
    "                str_lit = str_lit[: -len(q)]\n",
    "            break\n",
    "    # if start_quote in str_quote_options[:2]:\n",
    "    #     return \"\"\n",
    "    for sc in special_chars:\n",
    "        str_lit = str_lit.replace(sc, special_chars[sc])\n",
    "    return (\n",
    "        f\"{qualifier}{start_quote}<STR_LIT:{str_lit}>{end_quote}\"\n",
    "        if str_lit in lits['str']\n",
    "        else f\"{qualifier}{start_quote}<STR_LIT>{end_quote}\"\n",
    "    )\n",
    "\n",
    "def py_tokenize(file_type):\n",
    "    file_paths = glob.glob(os.path.join(os.getcwd(),\"data/\"+file_type+\"/Python\",\"*.*\"))\n",
    "    wf = open(os.path.join(os.getcwd(), f\"{file_type}.txt\"), 'w')\n",
    "    local_corpus = []\n",
    "    for path in file_paths:\n",
    "        try:\n",
    "            code = open(path).read()\n",
    "            token_gen = tokenize(BytesIO(bytes(code, \"utf8\")).readline)\n",
    "            out_tokens = []\n",
    "            prev_eol = False\n",
    "            for toknum, tokval, _, _, _ in token_gen:\n",
    "                tokval = \" \".join(tokval.split())\n",
    "                if toknum == STRING:\n",
    "                    add_token = process_string(tokval)\n",
    "                    out_tokens.append(add_token)\n",
    "                    prev_eol = False\n",
    "                elif toknum == NUMBER:\n",
    "                    if tokval in lits['num']:\n",
    "                        out_tokens.append(f\"<NUM_LIT:{tokval}>\")\n",
    "                    else:\n",
    "                        out_tokens.append(f\"<NUM_LIT>\")\n",
    "                    prev_eol = False\n",
    "                elif toknum in [NEWLINE, NL]:\n",
    "                    if not prev_eol:\n",
    "                        out_tokens.append(\"<EOL>\")\n",
    "                        prev_eol = True\n",
    "                elif toknum in [COMMENT, INDENT, ENCODING, ENDMARKER] or len(tokval) == 0:\n",
    "                    continue\n",
    "                else:\n",
    "                    out_tokens.append(tokval)\n",
    "                    prev_eol = False\n",
    "            if out_tokens[0] == \"<EOL>\":\n",
    "                out_tokens = out_tokens[1:]\n",
    "            if out_tokens[-1] == \"<EOL>\":\n",
    "                out_tokens = out_tokens[:-1]\n",
    "        except Exception:\n",
    "            out_tokens = []\n",
    "        local_corpus.extend(out_tokens)\n",
    "        out_tokens = [\"<s>\"] + out_tokens + [\"</s>\"]\n",
    "        out = \" \".join(out_tokens)\n",
    "        wf.write(out+\"\\n\")\n",
    "    print(f\"{file_type}: are done\")\n",
    "    wf.close()\n",
    "    return local_corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test: are done\n",
      "train: are done\n"
     ]
    }
   ],
   "source": [
    "# file_list = glob.glob(os.path.join(os.getcwd(),\"data/train/Python\",\"*.*\"))\n",
    "# corpus = []\n",
    "\n",
    "# for file_path in file_list:\n",
    "#     with open(file_path) as f_input:\n",
    "#         corpus.extend([line for line in f_input])\n",
    "        \n",
    "# print(corpus[40:50])\n",
    "py_tokenize(\"test\")\n",
    "corpus = py_tokenize(\"train\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['from', 'math', 'import', '*', '<EOL>', 'def', 'g', '(', 'x', ')']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_dataset(): \n",
    "    output = []\n",
    "    for line in corpus:\n",
    "        token_list = line\n",
    "        for i in range(1, len(token_list)):\n",
    "            data = []\n",
    "            x_ngram = '<start> '+ token_list[:i+1] + ' <end>'\n",
    "            y_ngram = '<start> '+ token_list[i+1:] + ' <end>'\n",
    "            data.append(x_ngram)\n",
    "            data.append(y_ngram)\n",
    "            output.append(data)\n",
    "    print(\"Dataset prepared with prefix and suffixes for teacher forcing technique\")\n",
    "    dummy_df = pd.DataFrame(output, columns=['input','output'])\n",
    "    return output, dummy_df     \n",
    "\n",
    "class LanguageIndex():\n",
    "    def __init__(self, lang):\n",
    "        self.lang = lang\n",
    "        self.word2idx = {}\n",
    "        self.idx2word = {}\n",
    "        self.vocab = set()\n",
    "        self.create_index()\n",
    "    def create_index(self):\n",
    "        for phrase in self.lang:\n",
    "            self.vocab.update(phrase.split(' '))\n",
    "        self.vocab = sorted(self.vocab)\n",
    "        self.word2idx[\"<pad>\"] = 0\n",
    "        self.idx2word[0] = \"<pad>\"\n",
    "        for i,word in enumerate(self.vocab):\n",
    "            self.word2idx[word] = i + 1\n",
    "            self.idx2word[i+1] = word\n",
    "\n",
    "def max_length(t):\n",
    "    return max(len(i) for i in t)\n",
    "\n",
    "def load_dataset():\n",
    "    pairs,df = generate_dataset()\n",
    "    out_lang = LanguageIndex(sp for en, sp in pairs)\n",
    "    in_lang = LanguageIndex(en for en, sp in pairs)\n",
    "    input_data = [[in_lang.word2idx[s] for s in en.split(' ')] for en, sp in pairs]\n",
    "    output_data = [[out_lang.word2idx[s] for s in sp.split(' ')] for en, sp in pairs]\n",
    "\n",
    "    max_length_in, max_length_out = max_length(input_data), max_length(output_data)\n",
    "    input_data = tf.keras.preprocessing.sequence.pad_sequences(input_data, maxlen=max_length_in, padding=\"post\")\n",
    "    output_data = tf.keras.preprocessing.sequence.pad_sequences(output_data, maxlen=max_length_out, padding=\"post\")\n",
    "    return input_data, output_data, in_lang, out_lang, max_length_in, max_length_out, df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset prepared with prefix and suffixes for teacher forcing technique\n"
     ]
    }
   ],
   "source": [
    "input_data, teacher_data, input_lang, target_lang, len_input, len_target, df = load_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_data = [[teacher_data[n][i+1] for i in range(len(teacher_data[n])-1)] for n in range(len(teacher_data))]\n",
    "target_data = tf.keras.preprocessing.sequence.pad_sequences(target_data, maxlen=len_target, padding=\"post\")\n",
    "target_data = target_data.reshape((target_data.shape[0], target_data.shape[1], 1))\n",
    "\n",
    "# Shuffle all of the data in unison. This training set has the longest (e.g. most complicated) data at the end,\n",
    "# so a simple Keras validation split will be problematic if not shuffled.\n",
    "\n",
    "p = np.random.permutation(len(input_data))\n",
    "input_data = input_data[p]\n",
    "teacher_data = teacher_data[p]\n",
    "target_data = target_data[p]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>&lt;start&gt; fr &lt;end&gt;</td>\n",
       "      <td>&lt;start&gt; om &lt;end&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>&lt;start&gt; fro &lt;end&gt;</td>\n",
       "      <td>&lt;start&gt; m &lt;end&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>&lt;start&gt; from &lt;end&gt;</td>\n",
       "      <td>&lt;start&gt;  &lt;end&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>&lt;start&gt; ma &lt;end&gt;</td>\n",
       "      <td>&lt;start&gt; th &lt;end&gt;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>&lt;start&gt; mat &lt;end&gt;</td>\n",
       "      <td>&lt;start&gt; h &lt;end&gt;</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                input            output\n",
       "0  <start> fr <end>    <start> om <end>\n",
       "1  <start> fro <end>   <start> m <end> \n",
       "2  <start> from <end>  <start>  <end>  \n",
       "3  <start> ma <end>    <start> th <end>\n",
       "4  <start> mat <end>   <start> h <end> "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_colwidth', -1)\n",
    "BUFFER_SIZE = len(input_data)\n",
    "BATCH_SIZE = 128\n",
    "embedding_dim = 300\n",
    "units = 128\n",
    "vocab_in_size = len(input_lang.word2idx)\n",
    "vocab_out_size = len(target_lang.word2idx)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 3)]          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 3, 300)       728700      input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            [(None, None)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional (Bidirectional)   [(None, 3, 256), (No 439296      embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, None, 300)    556800      input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 256)          0           bidirectional[0][1]              \n",
      "                                                                 bidirectional[0][3]              \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 256)          0           bidirectional[0][3]              \n",
      "                                                                 bidirectional[0][4]              \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   [(None, None, 256),  570368      embedding_1[0][0]                \n",
      "                                                                 concatenate[0][0]                \n",
      "                                                                 concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, None, 256)    0           lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, None, 128)    32896       dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, None, 128)    0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, None, 1856)   239424      dropout[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 2,567,484\n",
      "Trainable params: 2,567,484\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Create the Encoder layers first.\n",
    "encoder_inputs = Input(shape=(len_input,))\n",
    "encoder_emb = Embedding(input_dim=vocab_in_size, output_dim=embedding_dim)\n",
    "\n",
    "# Use this if you dont need Bidirectional LSTM\n",
    "# encoder_lstm = CuDNNLSTM(units=units, return_sequences=True, return_state=True)\n",
    "# encoder_out, state_h, state_c = encoder_lstm(encoder_emb(encoder_inputs))\n",
    "\n",
    "encoder_lstm = Bidirectional(LSTM(units=units, return_sequences=True, return_state=True))\n",
    "encoder_out, fstate_h, fstate_c, bstate_h, bstate_c = encoder_lstm(encoder_emb(encoder_inputs))\n",
    "state_h = Concatenate()([fstate_h,bstate_h])\n",
    "state_c = Concatenate()([bstate_h,bstate_c])\n",
    "encoder_states = [state_h, state_c]\n",
    "\n",
    "\n",
    "# Now create the Decoder layers.\n",
    "decoder_inputs = Input(shape=(None,))\n",
    "decoder_emb = Embedding(input_dim=vocab_out_size, output_dim=embedding_dim)\n",
    "decoder_lstm = LSTM(units=units*2, return_sequences=True, return_state=True)\n",
    "decoder_lstm_out, _, _ = decoder_lstm(decoder_emb(decoder_inputs), initial_state=encoder_states)\n",
    "# Two dense layers added to this model to improve inference capabilities.\n",
    "decoder_d1 = Dense(units, activation=\"relu\")\n",
    "decoder_d2 = Dense(vocab_out_size, activation=\"softmax\")\n",
    "decoder_out = decoder_d2(Dropout(rate=.2)(decoder_d1(Dropout(rate=.2)(decoder_lstm_out))))\n",
    "\n",
    "\n",
    "# Finally, create a training model which combines the encoder and the decoder.\n",
    "# Note that this model has three inputs:\n",
    "model = Model(inputs = [encoder_inputs, decoder_inputs], outputs= decoder_out)\n",
    "\n",
    "# We'll use sparse_categorical_crossentropy so we don't have to expand decoder_out into a massive one-hot array.\n",
    "# Adam is used because it's, well, the best.\n",
    "\n",
    "model.compile(optimizer=tf.optimizers.Adam(), loss=\"sparse_categorical_crossentropy\", metrics=['sparse_categorical_accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "filepath = \"model.h5\"\n",
    "checkpoint = ModelCheckpoint(filepath, monitor='loss', verbose=1, save_best_only=True, mode='min')\n",
    "callbacks_list = [checkpoint]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "543/543 [==============================] - 11s 12ms/step - loss: 2.3180 - sparse_categorical_accuracy: 0.6318 - val_loss: 0.7685 - val_sparse_categorical_accuracy: 0.8468\n",
      "\n",
      "Epoch 00001: loss improved from inf to 1.43662, saving model to model.h5\n",
      "Epoch 2/10\n",
      "543/543 [==============================] - 5s 10ms/step - loss: 0.7033 - sparse_categorical_accuracy: 0.8502 - val_loss: 0.5477 - val_sparse_categorical_accuracy: 0.8742\n",
      "\n",
      "Epoch 00002: loss improved from 1.43662 to 0.64756, saving model to model.h5\n",
      "Epoch 3/10\n",
      "543/543 [==============================] - 5s 10ms/step - loss: 0.5239 - sparse_categorical_accuracy: 0.8743 - val_loss: 0.4643 - val_sparse_categorical_accuracy: 0.8852\n",
      "\n",
      "Epoch 00003: loss improved from 0.64756 to 0.50346, saving model to model.h5\n",
      "Epoch 4/10\n",
      "543/543 [==============================] - 5s 10ms/step - loss: 0.4460 - sparse_categorical_accuracy: 0.8845 - val_loss: 0.4255 - val_sparse_categorical_accuracy: 0.8914\n",
      "\n",
      "Epoch 00004: loss improved from 0.50346 to 0.43700, saving model to model.h5\n",
      "Epoch 5/10\n",
      "543/543 [==============================] - 5s 10ms/step - loss: 0.4050 - sparse_categorical_accuracy: 0.8893 - val_loss: 0.4059 - val_sparse_categorical_accuracy: 0.8939\n",
      "\n",
      "Epoch 00005: loss improved from 0.43700 to 0.39945, saving model to model.h5\n",
      "Epoch 6/10\n",
      "543/543 [==============================] - 5s 10ms/step - loss: 0.3749 - sparse_categorical_accuracy: 0.8935 - val_loss: 0.3880 - val_sparse_categorical_accuracy: 0.8972\n",
      "\n",
      "Epoch 00006: loss improved from 0.39945 to 0.37588, saving model to model.h5\n",
      "Epoch 7/10\n",
      "543/543 [==============================] - 5s 10ms/step - loss: 0.3592 - sparse_categorical_accuracy: 0.8952 - val_loss: 0.3819 - val_sparse_categorical_accuracy: 0.9000\n",
      "\n",
      "Epoch 00007: loss improved from 0.37588 to 0.35947, saving model to model.h5\n",
      "Epoch 8/10\n",
      "543/543 [==============================] - 5s 10ms/step - loss: 0.3448 - sparse_categorical_accuracy: 0.8961 - val_loss: 0.3750 - val_sparse_categorical_accuracy: 0.9017\n",
      "\n",
      "Epoch 00008: loss improved from 0.35947 to 0.34528, saving model to model.h5\n",
      "Epoch 9/10\n",
      "543/543 [==============================] - 5s 10ms/step - loss: 0.3307 - sparse_categorical_accuracy: 0.8995 - val_loss: 0.3706 - val_sparse_categorical_accuracy: 0.9016\n",
      "\n",
      "Epoch 00009: loss improved from 0.34528 to 0.33530, saving model to model.h5\n",
      "Epoch 10/10\n",
      "543/543 [==============================] - 5s 10ms/step - loss: 0.3259 - sparse_categorical_accuracy: 0.8992 - val_loss: 0.3659 - val_sparse_categorical_accuracy: 0.9025\n",
      "\n",
      "Epoch 00010: loss improved from 0.33530 to 0.32668, saving model to model.h5\n"
     ]
    }
   ],
   "source": [
    "epochs = 10\n",
    "history = model.fit([input_data, teacher_data], target_data,\n",
    "                 batch_size= BATCH_SIZE,\n",
    "                 epochs=epochs,\n",
    "                 validation_split=0.2,\n",
    "                 callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU9b3/8ddnlmSyJxPCkgTIoCiEECCGgBcVUOvPpWK1VkWpVatcl6qt13vl9nrr0vZ3bWtbirW11itdRCmVqtSN+/NxqbhUVtmRgrKFsITs+2SS7++PM4QkJGSSTJjMzOf5eMwjM+ec+Z5PRnmfk+98z/eIMQallFLhzxbqApRSSgWHBrpSSkUIDXSllIoQGuhKKRUhNNCVUipCOEK14yFDhpicnJxQ7V4ppcLShg0bjhtjMrpaF7JAz8nJYf369aHavVJKhSUR2d/dOu1yUUqpCKGBrpRSEUIDXSmlIkTI+tCVUmdWc3MzxcXFNDY2hroUFQCXy0V2djZOpzPg92igKxUliouLSUpKIicnBxEJdTnqNIwxlJWVUVxcjMfjCfh92uWiVJRobGwkPT1dwzwMiAjp6em9/mtKA12pKKJhHj768t8q7AJ915EafvjWDhq8LaEuRSmlBpWwC/Tiinp++8FeNh2sDHUpSqleKCsrY/LkyUyePJnhw4eTlZXV9trr9QbUxu23386uXbtOu82zzz7LkiVLglEyF1xwAZs2bQpKW2dC2H0pWjjajQis3VvO+Welh7ocpVSA0tPT28Lx8ccfJzExkYcffrjDNsYYjDHYbF2fay5evLjH/dx33339LzZMhd0Zekq8k3OHJbFuX3moS1FKBcGePXvIy8vj7rvvpqCggMOHDzN//nwKCwuZMGECTz75ZNu2J86YfT4fqampLFiwgEmTJnH++edz7NgxAB599FEWLlzYtv2CBQsoKiri3HPP5eOPPwagrq6Or371q0yaNIm5c+dSWFjY45n4Sy+9xMSJE8nLy+O73/0uAD6fj69//ettyxctWgTAz3/+c3Jzc5k0aRLz5s0L+mfWnbA7QweY5nGzbH0xzS2tOO1hd0xSKuSe+Ot2dpRUB7XN3MxkHrt6Qp/eu2PHDhYvXsxzzz0HwFNPPYXb7cbn8zF79myuv/56cnNzO7ynqqqKmTNn8tRTT/HQQw/x4osvsmDBglPaNsawdu1aVqxYwZNPPsm7777LM888w/Dhw1m+fDmbN2+moKDgtPUVFxfz6KOPsn79elJSUrj00kt58803ycjI4Pjx42zduhWAykqrK/jHP/4x+/fvJyYmpm3ZmdBjGorIiyJyTES29bDdVBFpEZHrg1de14o86TQ0t7DtUNVA70opdQacddZZTJ06te31K6+8QkFBAQUFBezcuZMdO3ac8p64uDiuuOIKAM477zz27dvXZdvXXXfdKdt8+OGH3HTTTQBMmjSJCRNOfyBas2YNF198MUOGDMHpdHLzzTezevVqzj77bHbt2sWDDz7IypUrSUlJAWDChAnMmzePJUuW9OrCoP4K5Az9d8AvgT90t4GI2IEfASuDU9bpTfWkAbBuXzlTRqWdiV0qFVH6eiY9UBISEtqe7969m1/84hesXbuW1NRU5s2b1+V47JiYmLbndrsdn8/XZduxsbGnbGOM6VV93W2fnp7Oli1beOedd1i0aBHLly/n+eefZ+XKlbz//vu88cYb/OAHP2Dbtm3Y7fZe7bMvejxDN8asBnrqsL4fWA4cC0ZRPRma5MIzJIG1e7UfXalIU11dTVJSEsnJyRw+fJiVK4N/nnjBBRewbNkyALZu3drlXwDtTZ8+nVWrVlFWVobP52Pp0qXMnDmT0tJSjDF87Wtf44knnmDjxo20tLRQXFzMxRdfzE9+8hNKS0upr68P+u/QlX73oYtIFnAtcDEwtYdt5wPzAUaNGtWv/U7NSWPl9qO0thpsNr1YQqlIUVBQQG5uLnl5eYwZM4YZM2YEfR/3338/t956K/n5+RQUFJCXl9fWXdKV7OxsnnzySWbNmoUxhquvvpqrrrqKjRs38s1vfhNjDCLCj370I3w+HzfffDM1NTW0trbyyCOPkJSUFPTfoSsSyJ8eIpIDvGmMyeti3Z+BnxpjPhGR3/m3e7WnNgsLC01/bnDx6oZiHv7zZt799oWMG57c53aUihY7d+5k/PjxoS5jUPD5fPh8PlwuF7t37+ayyy5j9+7dOByDa5xIV//NRGSDMaawq+2DUX0hsNR/meoQ4EoR8RljXg9C292a5nED1nh0DXSlVG/U1tZyySWX4PP5MMbwm9/8ZtCFeV/0+zcwxrRNBdbuDH1AwxwgOy2O4cku1u4t59bzcwZ6d0qpCJKamsqGDRtCXUbQ9RjoIvIKMAsYIiLFwGOAE8AY89yAVnf6uijyuPnki7K2/iullIpmPQa6MWZuoI0ZY27rVzW9NNXjZsXmEg6U1zM6PaHnNyilVAQL68ssT/Sjr9Hhi0opFd6BfnZGImnxTh2PrpRShHmg22xCYY5bJ+pSKgzMmjXrlIuEFi5cyL333nva9yUmJgJQUlLC9dd3PbPIrFmz6GkY9MKFCztc4HPllVcGZZ6Vxx9/nKeffrrf7QRDWAc6WN0u+8vqOVqtN75VajCbO3cuS5cu7bBs6dKlzJ0b2Nd0mZmZvPpqj5e4dKtzoL/99tukpqb2ub3BKOwDvajdeHSl1OB1/fXX8+abb9LU1ATAvn37KCkp4YILLmgbF15QUMDEiRN54403Tnn/vn37yMuzrm1saGjgpptuIj8/nxtvvJGGhoa27e655562qXcfe+wxABYtWkRJSQmzZ89m9uzZAOTk5HD8+HEAfvazn5GXl0deXl7b1Lv79u1j/Pjx3HXXXUyYMIHLLrusw366smnTJqZPn05+fj7XXnstFRUVbfvPzc0lPz+/bVKw999/v+0GH1OmTKGmpqbPn+0JYT+SPndEMgkxdtbuLefqSZmhLkep8PDOAjiyNbhtDp8IVzzV7er09HSKiop49913ueaaa1i6dCk33ngjIoLL5eK1114jOTmZ48ePM336dObMmdPtcORf//rXxMfHs2XLFrZs2dJh+tsf/vCHuN1uWlpauOSSS9iyZQsPPPAAP/vZz1i1ahVDhgzp0NaGDRtYvHgxa9aswRjDtGnTmDlzJmlpaezevZtXXnmF3/72t9xwww0sX778tPOb33rrrTzzzDPMnDmT733vezzxxBMsXLiQp556ir179xIbG9vWzfP000/z7LPPMmPGDGpra3G5XL35tLsU9mfoDruNgtFpeoauVBho3+3SvrvFGMN3v/td8vPzufTSSzl06BBHjx7ttp3Vq1e3BWt+fj75+flt65YtW0ZBQQFTpkxh+/btPU689eGHH3LttdeSkJBAYmIi1113HR988AEAHo+HyZMnA6efohes+dkrKyuZOXMmAN/4xjdYvXp1W4233HILL730UtsVqTNmzOChhx5i0aJFVFZWBuVK1bA/QwcoynHz0//3DyrrvaTGx/T8BqWi3WnOpAfSV77yFR566CE2btxIQ0ND25n1kiVLKC0tZcOGDTidTnJycrqcMre9rs7e9+7dy9NPP826detIS0vjtttu67Gd081ndWLqXbCm3+2py6U7b731FqtXr2bFihV8//vfZ/v27SxYsICrrrqKt99+m+nTp/Pee+8xbty4PrV/QtifocPJfvR1+ypCXIlS6nQSExOZNWsWd9xxR4cvQ6uqqhg6dChOp5NVq1axf//+07Zz0UUXtd0Ietu2bWzZsgWwpt5NSEggJSWFo0eP8s4777S9Jykpqct+6osuuojXX3+d+vp66urqeO2117jwwgt7/bulpKSQlpbWdnb/xz/+kZkzZ9La2srBgweZPXs2P/7xj6msrKS2tpbPP/+ciRMn8sgjj1BYWMhnn33W6312FhFn6JNGphJjt7FuXzlfyh0W6nKUUqcxd+5crrvuug4jXm655RauvvpqCgsLmTx5co9nqvfccw+33347+fn5TJ48maKiIsC6+9CUKVOYMGHCKVPvzp8/nyuuuIIRI0awatWqtuUFBQXcdtttbW3ceeedTJky5bTdK935/e9/z9133019fT1jxoxh8eLFtLS0MG/ePKqqqjDG8J3vfIfU1FT+8z//k1WrVmG328nNzW27+1J/BDR97kDo7/S5nX3tuY/xthjeuC/4cycrFQl0+tzw09vpcyOiywWsbpdth6qoa+r6NlRKKRXpIibQp+a4aWk1fHrgzN1hWymlBpOICfTzRqdhE1i7tyzUpSg1aIWqi1X1Xl/+W0VMoCe5nEzITGGtzuuiVJdcLhdlZWUa6mHAGENZWVmvLzaKiFEuJ0zNcbNkzX6afC3EOuyhLkepQSU7O5vi4mJKS0tDXYoKgMvlIjs7u1fviahAL/K4efGjvWwtrqIwxx3qcpQaVJxOJx6Pp+cNVdiKmC4XgKk5aQDa7aKUikoRFejpibGcPTRR53VRSkWliAp0sLpdNuyroKVVv/hRSkWXyAv0HDc1TT52Hq4OdSlKKXVGRV6g6w0vlFJRKuICPTM1juy0OL3PqFIq6kRcoIPV7bJ2b7leQKGUiiqRGegeN2V1Xj4vrQt1KUopdcZEZKBPbbvhhXa7KKWiR4+BLiIvisgxEdnWzfpbRGSL//GxiEwKfpm9M2ZIAkMSY/SLUaVUVAnkDP13wOWnWb8XmGmMyQe+DzwfhLr6RUQo8rg10JVSUaXHQDfGrAa6TUZjzMfGmBM38/wE6N1sMgNkao6bQ5UNFFfUh7oUpZQ6I4Ldh/5N4J3uVorIfBFZLyLrB3rGtyLtR1dKRZmgBbqIzMYK9Ee628YY87wxptAYU5iRkRGsXXdp3PBkkmIdrN1b0fPGSikVAYIyfa6I5AMvAFcYYwbFLYPsNqEwJ03vYKSUihr9PkMXkVHAX4CvG2P+0f+SgqfIk87npXUcr20KdSlKKTXgAhm2+Arwd+BcESkWkW+KyN0icrd/k+8B6cCvRGSTiKwfwHp7pchjzY++XvvRlVJRoMcuF2PM3B7W3wncGbSKgmhiViqxDhtr9pZzed6IUJejlFIDKiKvFD0hxmGjYFSajnRRSkWFiA50sKYB2FFSTU1jc6hLUUqpARXxgT7N46bVwIb9OnxRKRXZIj7Qp4xKxWETnQZAKRXxIj7Q42Mc5GWlaKArpSJexAc6WN0uW4qraGxuCXUpSik1YKIi0KfmuPG2tLLpYGWoS1FKqQETNYEuAuu020UpFcGiItBT4p2cOyyJtToeXSkVwaIi0MGaTnfD/gp8La2hLkUppQZEVAV6vbeF7SXVoS5FKaUGRPQEeo51wwsdvqiUilRRE+hDk13kpMdrP7pSKmJFTaCDNdpl3b5yWltNqEtRSqmgi6pAL/K4qaxvZvex2lCXopRSQRdVgT7Nkw6g3S5KqYgUVYE+0h3HsORY/WJUKRWRoirQRYQiTzrr9pZjjPajK6UiS1QFOlj96EeqGzlY3hDqUpRSKqiiL9D949HX7C0LcSVKKRVcURfoY4cmkhrv1PuMKqUiTtQFus0mFI526xejSqmIE3WBDtYNL/aV1XOsujHUpSilVNBEZaAXefzzumi3i1IqgkRloE/ITCY+xq7dLkqpiBKVge6w2zhvdJoGulIqokRloIM1UdeuozVU1ntDXYpSSgVFj4EuIi+KyDER2dbNehGRRSKyR0S2iEhB8MsMviKPG2Ng/b6KUJeilFJBEcgZ+u+Ay0+z/gpgrP8xH/h1/8saeJNHphJjt+l4dKVUxOgx0I0xq4HTpd41wB+M5RMgVURGBKvAgeJy2snPTmGN9qMrpSJEMPrQs4CD7V4X+5edQkTmi8h6EVlfWloahF33T5HHzbZDVdR7faEuRSml+i0YgS5dLOtyKkNjzPPGmEJjTGFGRkYQdt0/RR43vlbDpwcqQ12KUkr1WzACvRgY2e51NlAShHYH3Hmj07AJ2u2ilIoIwQj0FcCt/tEu04EqY8zhILQ74JJcTnIzk1mnga6UigCOnjYQkVeAWcAQESkGHgOcAMaY54C3gSuBPUA9cPtAFTsQpua4eXnNAby+VmIcUTssXykVAXoMdGPM3B7WG+C+oFV0hk3zuFn80T62HqrivNFpoS5HKaX6LOpPSaf6b3ih0wAopcJd1Ad6emIsZ2UksFbvYKSUCnNRH+gARZ501u+voKVVbxytlApfGuhAkSeNmkYfnx2pDnUpSinVZxroWGfogA5fVEqFNQ10ICs1jqzUOL2DkVIqrGmg+xV5rBtHW6MwlVIq/Gig+xV53Byv9bL3eF2oS1FKqT7RQPfT8ehKqXCnge53VkYC6Qkx2o+ulApbGuh+ItLWj66UUuFIA72dqTluiisaKKlsCHUpSinVaxro7RR5rH50vc+oUiocaaC3M35EMkmxDr3hhVIqLGmgt2O3CeflpOkVo0qpsKSB3kmRx83uY7WU1TaFuhSllOoVDfROinJO9KNXhLgSpZTqHQ30TiZmpxDrsOkXo0qpsKOB3kmsw86UUak6Hl0pFXY00LtQlONme0kVtU2+UJeilFIB00DvQpEnnVYDG/ZrP7pSKnxooHdhyqhU7DbR+4wqpcKKBnoXEmId5GWlsG6vnqErpcKHBno3pnncbDpYSWNzS6hLUUqpgGigd2NqjhtvSytbiqtCXYpSSgVEA70bU3PSALQfXSkVNgIKdBG5XER2icgeEVnQxfpRIrJKRD4VkS0icmXwSz2zUuNjOHdYkk7UpZQKGz0GuojYgWeBK4BcYK6I5Hba7FFgmTFmCnAT8KtgFxoKRR43G/dX4GtpDXUpSinVo0DO0IuAPcaYL4wxXmApcE2nbQyQ7H+eApQEr8TQKfK4qfO2sONwdahLUUqpHgUS6FnAwXavi/3L2nscmCcixcDbwP1BqS7ETtzwQqcBUEqFg0ACXbpYZjq9ngv8zhiTDVwJ/FFETmlbROaLyHoRWV9aWtr7as+wYckuRqfHa6ArpcJCIIFeDIxs9zqbU7tUvgksAzDG/B1wAUM6N2SMed4YU2iMKczIyOhbxWdYUY6bdfvKaW3tfAxTSqnBJZBAXweMFRGPiMRgfem5otM2B4BLAERkPFagD/5T8ABM9bipqG/m89LaUJeilFKn1WOgG2N8wLeAlcBOrNEs20XkSRGZ49/sX4C7RGQz8ApwmzEmIk5pp/n70XX4olJqsHMEspEx5m2sLzvbL/teu+c7gBnBLa0bNUdh9U/g//wQHLEDvrtR7niGJsWydm8586aPHvD9KaVUX4XflaIHP4F1v4UV98MZ+CNARCjyuFm7t5wI+aNDKRWhwi/Qc6+B2Y/Clj/B+z86I7uc5nFzpLqR4oqGM7I/pZTqi/ALdICLHobJt8Df/gs2/2nAdzdVx6MrpcJAeAa6CHx5IeRcCG/cB/s+HNDdnTM0iZQ4pwa6UmpQC89AB3DEwI1/BLcHlt4Cx3cP2K5sNmFqThpr92mgK6UGr/ANdIC4NLh5GdgcsOR6qDs+YLsq8rjZe7yOYzWNA7YPpZTqj/AOdLDO0OcuhZojsPRmaB6YwC3ypAPobemUUoNW+Ac6wMipcO1zcHANvH4PtAZ/utsJmcnEOe2s024XpdQgFdCFRWFhwrVQsR/ee8w6a7/kez2/pxecdhvnjU7TK0aVUoNWZJyhnzDjQSj4BnzwU9j4x6A3X+Rx89mRaqoamoPetlJK9VdkBboIXPVTGDMb3vw2fPG3oDY/NceNMbBhv56lK6UGn8gKdAC7E274PaSPhT/dCsc+C1rTU0al4rSLdrsopQalyAt0AFcK3LIMnC54+WtQeyw4zTrt5Gensk4DXSk1CEVmoAOkjrKGM9aWwis3gbc+KM0WedxsKa6iwdsSlPaUUipYIjfQAbIK4KsvwKGN8No/B2U4Y5HHja/V8OkBHY+ulBpcIjvQAcZ/2Zo7fecKa0hjP503Og0RdBoApdSgEznj0E9n+r1Q/gV8vMgao154R5+bSnY5yR2RrBN1KaUGncg/QwdrOOPlP4Kxl8FbD8Oe9/rV3NQcNxsPVOD1Bf+KVKWU6qvoCHQAuwOufxGG5sKy2+Do9j43Nc3jprG5lW0lVcGrTyml+il6Ah0gNglu/hPEJsKSG6D6cJ+a0RteKKUGo+gKdICULCvUGyrglRvBW9frJoYkxjImI0HHoyulBpXoC3SAEZOs7pcjW2H5ndDa+zHl0zxu/v5Fmc6+qJQaNKIz0AHOvdz6onTX2/A/j/b67fMvOouMpFhuev4Tnl/9OcaYAShSKaUCF72BDjBtvjWk8ZNfwZrne/VWz5AE/nr/BVyWO4z/+/Zn3PWHDVTV6yyMSqnQie5AB7jsB3DulfDuI7Dr3V69Ndnl5Fe3FPDY1bm8/49jXPXMB2wprhygQpVS6vQ00G12a3qA4RPh1Tvg8OZevV1EuH2Gh2X/fD7GwPW//jt/+Ps+7YJRSp1xGugAMQkw90/WTadfvhGqDvW6iSmj0njz/guYcXY633tjO/e/8im1Tb4BKFYppboWUKCLyOUisktE9ojIgm62uUFEdojIdhF5ObhlngHJI6wpd5tqrVBvqul1E2kJMfz3N6byyOXjeGfbEeY88yE7D1cPQLFKKXWqHgNdROzAs8AVQC4wV0RyO20zFvh3YIYxZgLw7QGodeANmwA3/A6O7bC6X1p6f4Ztswn3zDqLl++cRm2Tj688+xHL1h0Mfq1KKdVJIGfoRcAeY8wXxhgvsBS4ptM2dwHPGmMqAIwxwbmjRCicfal1G7vd/2N9UdrHvvBpY9J564ELKcxJ49+Wb+HhP2/WOdSVUgMqkEDPAtqfYhb7l7V3DnCOiHwkIp+IyOVdNSQi80VkvYisLy0t7VvFZ0Lh7fBPD8C6F6whjX2UkRTLH+6YxoOXjGX5xmK+8uxH7DlWG8RClVLqpEACXbpY1vm01QGMBWYBc4EXRCT1lDcZ87wxptAYU5iRkdHbWs+sS5+A8XNg5X/Azjf73IzdJnznS+fw+9uLKK1tYs4vP+SNTb3/0lUppXoSSKAXAyPbvc4GSrrY5g1jTLMxZi+wCyvgw5fNBtf+xrrr0fI7rbse9cNF52Tw9gMXMiEzmQeXbuI/XttKY7N2wSilgieQQF8HjBURj4jEADcBKzpt8zowG0BEhmB1wXwRzEJDIibeui9pYoZ1X9LKA/1qbniKi5fvms4/zxzDkjUHuP65jzlQFpx7nSqlVI+BbozxAd8CVgI7gWXGmO0i8qSIzPFvthIoE5EdwCrgX40xZQNV9BmVOBRu/jM0N1rDGRv7Nwe6027j368Yz29vLeRAWT1XPfMB7247EqRilVLRTEJ1RWNhYaFZv359SPbdJ1/8DV76KuRcCLf8GezOfjd5sLyeb728kc3FVdx5gYdHrhiH067XeimluiciG4wxhV2t0/QI1JhZ8OWF8MUqeOuhPg9nbG+kO55ld5/Pbf+Uwwsf7uXG3/ydksqGfrerlIpOGui9UfB1uPBfYOMf4KNfBKXJWIedx+dM4Jc3T+EfR2u5atEHrNoVvsP4lVKho4HeW7MfhbyvwnuPwba/BK3ZL+dnsuJbMxiW7OL2xet4euUufC16E2qlVOA00HvLZoNrfgUjp8Grt1v96nveC0oXzJiMRF6/bwY3TR3JL1ftYd5/r+FYdWMQilZKRQMN9L5wuuCWV2H2f8DhLVao/2o6rF8Mzf3rA3c57Tz11Xye/tokNh2s5MpFH/Lx58eDVLhSKpLpKJf+8jVZXS+fPGvdozTODYV3wNQ7rRkc+2HXkRruXbKBvcfreOhL53DvrLOx2bq6cFcpFS1ON8pFAz1YjIH9H1tzv3z2lnXjjAnXwfn3QuaUPjdb1+Tju69t5Y1NJVx0TgYLb5yMOyEmiIUrpcKJBvqZVr4X1vwGPv0jeGth1PnWvUvHXWUFfS8ZY3h57QGe+OsO0hNi+OXNUzhvtHsACldKDXYa6KHSWAWfvgRrnrOmDUgdBdPuhinzwJXS6+a2Hari3iUbKalsYMEV4/jmBR5EtAtGqWiigR5qrS1WN8wnv4YDH0NMkhXq0+aDe0yvmqpubOZf/7yZlduP8qXcYTz9tUmkxPX/qlWlVHjQQB9MSj61gn3bcivox10F0++B0TMgwLNtYwwvfrSP/3p7JyNSXTx9/SSKPG49W1cqCmigD0bVh60baKx/ERrKYXi+1c+edx04YgNqYuOBCr61ZCMlVY3kpMczZ1ImcyZncvbQpAEuXikVKhrog5m3HrYus87aSz+DxGHWkMfCOyBhSI9vr2ls5p2tR1ixuYSPPz9Oq4HcEclcMzmTqydlkpkadwZ+CaXUmaKBHg6Mgc//1xr2uOc9sMdC/g3WWfuw3J7fDxyrbuTNLYdZsbmETQcrASjKcTNnciZXThyhwx2VigAa6OGmdJd1xr55KfgarJkep99n3cDaFtjFvfvL6lixqYQ3Npew51gtDptw4dghXDM5iy/lDiMh1jGgv4JSamBooIer+nLY8DtY+1uoKYH0sTD9bpg0F2ISAmrCGMPOwzWs2FzCXzeXcKiyAZfTxqXjhzFnUiYzz80g1tH7sfFKqdDQQA93Lc2w/XVreoGST8GVCufdBkV3QUp2wM20tho2HKhgxaYS3tp6mPI6L8kuB1dOHMGcSZlMG5OOXacWUGpQ00CPFMbAwTVWP/vOvwICudfAlFsg6zyISwu4qeaWVj7ac5wVm0pYuf0Idd4WhibF8uX8TK6ZnEl+dooOg1RqENJAj0SVB6zpBTb+AZqqrWXuMda8MSceIyZBbM9DGBu8LfzvZ8d4Y9Mh/rarFG9Lqw6DVGqQ0kCPZE21ULzW6oop+RRKNkHVQf9KgSHnnAz4rAIYlgcx8d02V9XQzMptOgxSqcFKAz3a1B6zgr0t5DdC7VFrndhh6HjInAyZBVbQD5vQ5cVM3Q2DvHpyJlfpMEilQkIDPdoZAzWHrXA/tPFk0DeUW+vtMVaot++uyRgP9pNDG7sbBjlnciZfyh1Oog6DVOqM0EBXpzLG6ocv2dixu+ZEf7zDZU1H0L67Jv1sjNi6HAZ5yfhhzDong4nZKZydkYjDrsb3XEoAAAukSURBVDfDUmogaKCrwLS2QvkX7QL+Uzi8GZrrrPUxidYXrf6Qbx0xhQ01qazYfLhtGCRArMPG+BHJTMxKIS8rmbysFMYOTSLGoSGvVH9poKu+a22B4/84GfCHNlq32mtpsta7UtrCvTR2JHsakthaHc/a4zGsPdJKbVMLADF2G+NGJJGXlUJeZgoTs1I4Z3iiXtSkVC9poKvgammGYztPfuFa8ikc3Q6tvg6bGUccvoRhVDvSOWLc7PMmsaM2kQPeZI6aNMps6SQPHcm52UOZkGWF/LjhSbicGvJKdUcDXQ08XxNUl1hfvtYctqYHrjkMNUc6LvM1nPLWKhI40prGUZPGMdw0xw8l1p1N2rBRDM/2kOM5m7jUER2+pFUqWp0u0AP6FyIilwO/AOzAC8aYp7rZ7nrgz8BUY4ymdTRxxILbYz26Y4x1W76aI9bcNDVHoLqE5JojOMuLGVZ+CFvdThKaPsB+uBUOA5ust7Zgo9aRRnPcMOypmSQOycaZmgVJI6xHsv9nXFrANwpRKtL0GOgiYgeeBb4EFAPrRGSFMWZHp+2SgAeANQNRqIoAIhCXaj2Gjju5GIj3PwBobcHUlXL88H4O7v+cssP7qS87iKk+QkrlcYZX/YOhB9aQLjWn7sMea80pnzjU+pk0rOPrE88ThoLTdQZ+aaXOnEDO0IuAPcaYLwBEZClwDbCj03bfB34MPBzUClX0sdmRpOFkJA0n45xpHVYdrW5ka3EVK0uq+Ky4lCOH9mOrPcpwKWeYVHCus5ZRvhqGVleRVrmLRN/fiW0q73o/rhRIHH5q2Hc4IAyHOHfA0xYrFUqBBHoWcLDd62Kgw78yEZkCjDTGvCki3Qa6iMwH5gOMGjWq99WqqDcs2cWwXBeX5g4DzgFmcKymke2Hqtl6qIpVJVUcKG+guLyemibrS1oHPtzUMNJZzbjEes6Kq2NUTC3D7FWkU0lyQzlxlRuw1Zci3tpTdyp2f8AP7fpsv/1BITbxjH4eSrUXSKB31SHZ9k2qiNiAnwO39dSQMeZ54HmwvhQNrESlTm9okouh41zMHje0w/KqhmaKK+o5VNFAcdujnk2VDRQfbaCqobnD9i6njbNSIDepkbHxdYyOrSPTUc0QqSS1pRxXUxlSe9Qatll7DEzLqcU4E/xdOhnWnDkOV7tHrPXT2em1IxYcce1eB7CN3anfFahTBBLoxcDIdq+zgZJ2r5OAPOBv/ulWhwMrRGSOfjGqQiklzklKXAoTMlO6XF/d2NwW9ocq6ttCf2dlPe8dTaCivmPgxzpsZKXFkZ0WT3ZOLGclevG4asl21jDcVkWirxxb3TFr3pzaY9DcAA0V0NwIvkZrJJCv8eSjP8TWReB3d/CIa/fT/3C4Oj2P72abdst1lNGgF8h/oXXAWBHxAIeAm4CbT6w0xlQBbXczFpG/AQ9rmKvBLtnlJHmEk/EjkrtcX9vk8we+FfaHKk8+33aoqu3KWGvwl5sY+xCy0iaRnRZHVmocGUmxpCfEkJ4YS3piDEMSrdep8THYBWjxWsHeXeCfeN3ceVlDu22brANH+9e+But5Y6X/vQ3Wz+YG63mn6wUCZnO2C3p/2Hc4MPgPHB2et/srxe605g2yx/TjuRNsDv3rpBs9Broxxici3wJWYv2f+6IxZruIPAmsN8asGOgilQqFxFgH5w5P4tzhXc8HX+/1tevO8Z/hV/rP8nceo7yuidYuOhZtAu6EGNITYhmSZP08GfipJw8AydbPoN//taXZH+7+kD8R9G3h39DF+kZoru9+m8Zq8B3rur2B0FPoB3SAiD25zBHbbpl/vSO2f9uG4KCjFxYpNUBaWw2VDc2U1TZRWttEWa2Xstomyuq8HG/3vMy/7sSXuJ3FOe2kJ1pn+kMSYtqepyf4DwKJJw8O7viYwTUxmjEnQ7/VZ/1V0uK1Diotze2eewN83sWy1ubA2vD5t/V52633P/r6V8vp2Jzdh/+Ur8P59/ap2X5fWKSU6j2bTXAnxOBOiGHssJ7v+tTY3NIh4I93CvzjdV6OVDeyraSKslovvq5O/4G0eGdb4KcnWl087vgYUuOdpMXHkJbgJDU+xnoe7yTZ5cQ2UPeSFTnZJTOYtbZ2Ogg0nXzuazrN8hMHjHbPOyz3djygnHgenz4gv4YGulKDhMtpJyvV6n/viTGG6gYfx+tOnvkf73wwqPXy2ZEaKuubqaz3dtn9A1YXUGr7wI8/EfhO0hJiOi07+TyiZs+02cDmCvuLzTTQlQpDIkJKvJOUeCdnZfS8fWuroabRR0W9l4p6L5X1zf7nzVTUdVx2qLKR7SXVVNR7aWxu7bbNhBh7W+Cf/mAQQ0qck0SXg4RYu86wOYA00JWKAjbbyQNADgkBv6/B29LlQaCyzv+z3ku5f9mB8noq6rxUN56+PzrGbiMh1m4FfIyDJJeDhFgHif5H++eJ/nVJXS7Xg0NnGuhKqW7FxdiJi4nr1c3BfS2tVDU0nwz8Oi9VDc3UNfmo87ZQ0+iznjf5qPH/LK/zcqC8ntoT67xdXLTVBaddenUQiI+xkxBj/YyPdZAQYyfOvywuxk6sw4aE8ZBIDXSlVFA57Db/0MtTbzweqNZWQ53XR60/8GubWqhttF6fXOZ/NHZ8XeE/ONSdWBfgwQHAbhMr7NuFfEKMg/hYu3+5o+1n28Egtutlcc6T687UgUIDXSk16NhsQpLLSZLL2e+2Thwc6ppaqPP6qPf/bPCefF3vtYK/w7LmFuqbfNR5rYNEcUXH9d6W7r9fOOX3EU4eIGId3Fw0irsuGtPv360zDXSlVEQL5sGhveaWVuq91sGg3tvSdmCoPxH63hMHhJMHAutnCxlJff/r5XQ00JVSqg+cdhspcTZS4oJ7oOiPCBpIqpRS0U0DXSmlIoQGulJKRQgNdKWUihAa6EopFSE00JVSKkJooCulVITQQFdKqQgRsjsWiUgpsL+Pbx8CHA9iOeFOP4+O9PM4ST+LjiLh8xhtjOly0uSQBXp/iMj67m7BFI308+hIP4+T9LPoKNI/D+1yUUqpCKGBrpRSESJcA/35UBcwyOjn0ZF+HifpZ9FRRH8eYdmHrpRS6lTheoaulFKqEw10pZSKEGEX6CJyuYjsEpE9IrIg1PWEkoiMFJFVIrJTRLaLyIOhrinURMQuIp+KyJuhriXURCRVRF4Vkc/8/4+cH+qaQkVEvuP/N7JNRF4REVeoaxoIYRXoImIHngWuAHKBuSKSG9qqQsoH/IsxZjwwHbgvyj8PgAeBnaEuYpD4BfCuMWYcMIko/VxEJAt4ACg0xuQBduCm0FY1MMIq0IEiYI8x5gtjjBdYClwT4ppCxhhz2Biz0f+8BusfbFZoqwodEckGrgJeCHUtoSYiycBFwH8DGGO8xpjK0FYVUg4gTkQcQDxQEuJ6BkS4BXoWcLDd62KiOMDaE5EcYAqwJrSVhNRC4N+AwG/HHrnGAKXAYn8X1AsikhDqokLBGHMIeBo4ABwGqowx/xPaqgZGuAW6dLEs6sddikgisBz4tjGmOtT1hIKIfBk4ZozZEOpaBgkHUAD82hgzBagDovI7JxFJw/pL3gNkAgkiMi+0VQ2McAv0YmBku9fZROifToESESdWmC8xxvwl1PWE0Axgjojsw+qKu1hEXgptSSFVDBQbY078xfYqVsBHo0uBvcaYUmNMM/AX4J9CXNOACLdAXweMFRGPiMRgfbGxIsQ1hYyICFYf6U5jzM9CXU8oGWP+3RiTbYzJwfr/4n+NMRF5FhYIY8wR4KCInOtfdAmwI4QlhdIBYLqIxPv/zVxChH5B7Ah1Ab1hjPGJyLeAlVjfVL9ojNke4rJCaQbwdWCriGzyL/uuMebtENakBo/7gSX+k58vgNtDXE9IGGPWiMirwEaskWGfEqFTAOil/0opFSHCrctFKaVUNzTQlVIqQmigK6VUhNBAV0qpCKGBrpRSEUIDXSmlIoQGulJKRYj/D/DVrI2JqNGmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot the results of the training.\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.plot(history.history['loss'], label=\"Training loss\")\n",
    "plt.plot(history.history['val_loss'], label=\"Validation loss\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the encoder model from the tensors we previously declared.\n",
    "encoder_model = Model(encoder_inputs, [encoder_out, state_h, state_c])\n",
    "\n",
    "# Generate a new set of tensors for our new inference decoder. Note that we are using new tensors, \n",
    "# this does not preclude using the same underlying layers that we trained on. (e.g. weights/biases).\n",
    "\n",
    "inf_decoder_inputs = Input(shape=(None,), name=\"inf_decoder_inputs\")\n",
    "# We'll need to force feed the two state variables into the decoder each step.\n",
    "state_input_h = Input(shape=(units*2,), name=\"state_input_h\")\n",
    "state_input_c = Input(shape=(units*2,), name=\"state_input_c\")\n",
    "decoder_res, decoder_h, decoder_c = decoder_lstm(\n",
    "    decoder_emb(inf_decoder_inputs), \n",
    "    initial_state=[state_input_h, state_input_c])\n",
    "inf_decoder_out = decoder_d2(decoder_d1(decoder_res))\n",
    "inf_model = Model(inputs=[inf_decoder_inputs, state_input_h, state_input_c], \n",
    "                  outputs=[inf_decoder_out, decoder_h, decoder_c])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converts the given sentence (just a string) into a vector of word IDs\n",
    "# Output is 1-D: [timesteps/words]\n",
    "\n",
    "def sentence_to_vector(sentence, lang):\n",
    "\n",
    "    pre = sentence\n",
    "    vec = np.zeros(len_input)\n",
    "    sentence_list = [lang.word2idx[s] for s in pre.split(' ')]\n",
    "    for i,w in enumerate(sentence_list):\n",
    "        vec[i] = w\n",
    "    return vec\n",
    "\n",
    "# Given an input string, an encoder model (infenc_model) and a decoder model (infmodel),\n",
    "def translate(input_sentence, infenc_model, infmodel):\n",
    "    sv = sentence_to_vector(input_sentence, input_lang)\n",
    "    sv = sv.reshape(1,len(sv))\n",
    "    [emb_out, sh, sc] = infenc_model.predict(x=sv)\n",
    "    \n",
    "    i = 0\n",
    "    start_vec = target_lang.word2idx[\"<start>\"]\n",
    "    stop_vec = target_lang.word2idx[\"<end>\"]\n",
    "    \n",
    "    cur_vec = np.zeros((1,1))\n",
    "    cur_vec[0,0] = start_vec\n",
    "    cur_word = \"<start>\"\n",
    "    output_sentence = \"\"\n",
    "\n",
    "    while cur_word != \"<end>\" and i < (len_target-1):\n",
    "        i += 1\n",
    "        if cur_word != \"<start>\":\n",
    "            output_sentence = output_sentence + \" \" + cur_word\n",
    "        x_in = [cur_vec, sh, sc]\n",
    "        [nvec, sh, sc] = infmodel.predict(x=x_in)\n",
    "        cur_vec[0,0] = np.argmax(nvec[0,0])\n",
    "        cur_word = target_lang.idx2word[np.argmax(nvec[0,0])]\n",
    "    return output_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Input seq</th>\n",
       "      <th>Pred. Seq</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>fo</td>\n",
       "      <td>r</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>rando</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>if</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>im</td>\n",
       "      <td>port</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>imp</td>\n",
       "      <td>ort</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>mat</td>\n",
       "      <td>h</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>el</td>\n",
       "      <td>se</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Input seq Pred. Seq\n",
       "0  fo         r      \n",
       "1  rando      m      \n",
       "2  if                \n",
       "3  im         port   \n",
       "4  imp        ort    \n",
       "5  mat        h      \n",
       "6  el         se     "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = [\n",
    "    'fo',\n",
    "    'rando',\n",
    "    'if',\n",
    "    'im',\n",
    "    'imp',\n",
    "    'mat',\n",
    "    'el'\n",
    "]\n",
    "  \n",
    "\n",
    "import pandas as pd\n",
    "output = []  \n",
    "for t in test:  \n",
    "  output.append({\"Input seq\":t.lower(), \"Pred. Seq\":translate(t.lower(), encoder_model, inf_model)})\n",
    "\n",
    "results_df = pd.DataFrame.from_dict(output) \n",
    "results_df.head(len(test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "TensorflowGPU",
   "language": "python",
   "name": "tf_gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}