nwp_lstm_multi_word-mix.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Currently Loaded Modules:\n",
      "  1) anaconda3/5.1.0-gcc/8.3.1     4) cudnn/8.0.0.180-11.0-linux-x64-gcc/7.5.0\n",
      "  2) anaconda3/2019.10-gcc/8.3.1   5) openjdk/1.8.0_222-b10-gcc/8.3.1\n",
      "  3) cuda/11.0.3-gcc/7.5.0         6) hadoop/3.2.1-gcc/8.3.1\n",
      "\n",
      " \n",
      "\n"
     ]
    }
   ],
   "source": [
    "!module list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.4.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Input, LSTM, Dense, Embedding, Flatten, TimeDistributed, Dropout, LSTMCell, RNN, Bidirectional, Concatenate, Layer\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "from tensorflow.python.keras.utils import tf_utils\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "import pickle\n",
    "\n",
    "\n",
    "import unicodedata\n",
    "import re\n",
    "import os\n",
    "import time\n",
    "import shutil\n",
    "import requests\n",
    "import tarfile\n",
    "import glob\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string, os\n",
    "tf.__version__\n",
    "\n",
    "# gpus = tf.config.list_physical_devices('GPU')\n",
    "# gpu = gpus[0]\n",
    "\n",
    "# tf.config.experimental.set_memory_growth(gpu, True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU'),\n",
       " PhysicalDevice(name='/physical_device:GPU:1', device_type='GPU')]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "physical_devices = tf.config.experimental.list_physical_devices('GPU')\n",
    "physical_devices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.config.experimental.set_memory_growth(physical_devices[0], True)\n",
    "tf.config.experimental.set_memory_growth(physical_devices[1], True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = \"Project_CodeNet_LangClass.tar.gz\"\n",
    "data_url = f\"https://dax-cdn.cdn.appdomain.cloud/dax-project-codenet/1.0.0/{file_name}\"\n",
    "\n",
    "# Download tar archive to local disk\n",
    "with open(file_name, \"wb\") as f:\n",
    "    f.write(requests.get(data_url).content)\n",
    "    \n",
    "# Extract contents of archive to local disk\n",
    "if os.path.exists(\"data\"):\n",
    "    shutil.rmtree(\"data\")    \n",
    "with tarfile.open(file_name) as tfile:\n",
    "    tfile.extractall()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data:\n",
      "test  train\n",
      "\n",
      "data/train:\n",
      "C  C#  C++  D  Haskell\tJava  JavaScript  PHP  Python  Rust\n"
     ]
    }
   ],
   "source": [
    "!ls data data/train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "langs = [\n",
    "  \"C\",\n",
    "  \"C#\",\n",
    "  \"C++\",\n",
    "  \"D\",\n",
    "  \"Haskell\",\n",
    "  \"Java\",\n",
    "  \"JavaScript\",\n",
    "  \"PHP\",\n",
    "  \"Python\",\n",
    "  \"Rust\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import argparse\n",
    "import re\n",
    "from tokenize import tokenize, untokenize, COMMENT, STRING, NEWLINE, ENCODING, ENDMARKER, NL, INDENT, NUMBER\n",
    "from io import BytesIO\n",
    "import json\n",
    "\n",
    "lits = json.load(open(\"literals.json\"))\n",
    "\n",
    "def process_string(token, special_chars={\" \": \"U+0020\", \",\": \"U+002C\"}):\n",
    "    str_quote_options = [\"'''\", '\"\"\"', \"'\", '\"']\n",
    "    start_quote = \"\"\n",
    "    end_quote = \"\"\n",
    "    qualifier_regex = r\"^[a-z]+\"\n",
    "    qualifier_match = re.search(qualifier_regex, token)\n",
    "    # string qualifiers like 'r' for regex, 'f' for formatted string, 'b' for bytes, 'u' for unicode, etc (or combination of them)\n",
    "    qualifier = \"\" if not qualifier_match else qualifier_match[0]\n",
    "    # token string without qualifiers\n",
    "    token_string = re.sub(qualifier_regex, \"\", token)\n",
    "    # string literal without quotes\n",
    "    str_lit = token_string\n",
    "    for q in str_quote_options:\n",
    "        if token_string.startswith(q):\n",
    "            start_quote = q\n",
    "            str_lit = str_lit[len(q) :]\n",
    "            if token_string.endswith(q):\n",
    "                end_quote = q\n",
    "                str_lit = str_lit[: -len(q)]\n",
    "            break\n",
    "    # if start_quote in str_quote_options[:2]:\n",
    "    #     return \"\"\n",
    "    for sc in special_chars:\n",
    "        str_lit = str_lit.replace(sc, special_chars[sc])\n",
    "    return (\n",
    "        f\"{qualifier}{start_quote}<STR_LIT:{str_lit}>{end_quote}\"\n",
    "        if str_lit in lits['str']\n",
    "        else f\"{qualifier}{start_quote}<STR_LIT>{end_quote}\"\n",
    "    )\n",
    "\n",
    "def py_tokenize(file_type):\n",
    "    file_paths = glob.glob(os.path.join(os.getcwd(),\"data/\"+file_type+\"/Python\",\"*.*\"))\n",
    "    wf = open(os.path.join(os.getcwd(), f\"{file_type}.txt\"), 'w')\n",
    "    local_corpus = []\n",
    "    for path in file_paths:\n",
    "        try:\n",
    "            code = open(path).read()\n",
    "            token_gen = tokenize(BytesIO(bytes(code, \"utf8\")).readline)\n",
    "            out_tokens = []\n",
    "            prev_eol = False\n",
    "            for toknum, tokval, _, _, _ in token_gen:\n",
    "                tokval = \" \".join(tokval.split())\n",
    "                if toknum == STRING:\n",
    "                    add_token = process_string(tokval)\n",
    "                    out_tokens.append(add_token)\n",
    "                    prev_eol = False\n",
    "                elif toknum == NUMBER:\n",
    "                    if tokval in lits['num']:\n",
    "                        out_tokens.append(f\"<NUM_LIT:{tokval}>\")\n",
    "                    else:\n",
    "                        out_tokens.append(f\"<NUM_LIT>\")\n",
    "                    prev_eol = False\n",
    "                elif toknum in [NEWLINE, NL]:\n",
    "                    if not prev_eol:\n",
    "                        out_tokens.append(\"<EOL>\")\n",
    "                        prev_eol = True\n",
    "                elif toknum in [COMMENT, INDENT, ENCODING, ENDMARKER] or len(tokval) == 0:\n",
    "                    continue\n",
    "                else:\n",
    "                    out_tokens.append(tokval)\n",
    "                    prev_eol = False\n",
    "            if out_tokens[0] == \"<EOL>\":\n",
    "                out_tokens = out_tokens[1:]\n",
    "            if out_tokens[-1] == \"<EOL>\":\n",
    "                out_tokens = out_tokens[:-1]\n",
    "        except Exception:\n",
    "            out_tokens = []\n",
    "#         local_corpus.extend((\" \".join(out_tokens)).split('<EOL>'))\n",
    "#         out_tokens = [\"<s>\"] + out_tokens + [\"</s>\"]\n",
    "        out = \" \".join(out_tokens)\n",
    "        local_corpus.append(out)\n",
    "        wf.write(out+\"\\n\")\n",
    "    print(f\"{file_type}: are done\")\n",
    "    wf.close()\n",
    "    return local_corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train: are done\n"
     ]
    }
   ],
   "source": [
    "# file_list = glob.glob(os.path.join(os.getcwd(),\"data/train/Python\",\"*.*\"))\n",
    "# corpus = []\n",
    "\n",
    "# for file_path in file_list:\n",
    "#     with open(file_path) as f_input:\n",
    "#         corpus.extend([line for line in f_input])\n",
    "        \n",
    "# print(corpus[40:50])\n",
    "# py_tokenize(\"test\")\n",
    "corpus = py_tokenize(\"train\")\n",
    "# corpus2 = pd.DataFrame(corpus,columns=['code'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# corpus2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# corpus2['tokenized']=corpus2['code'].str.split()\n",
    " \n",
    "# corpus2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "import sys <EOL> def prod2d ( vec1 , vec2 ) : <EOL> return vec1 [ <NUM_LIT:0> ] * vec2 [ <NUM_LIT:1> ] - vec1 [ <NUM_LIT:1> ] * vec2 [ <NUM_LIT:0> ] <EOL> lineNumber = <NUM_LIT:0> <EOL> for line in sys . stdin . readlines ( ) : <EOL> lineNumber += <NUM_LIT:1> <EOL> List = map ( float , line . strip \n"
     ]
    }
   ],
   "source": [
    "full_corpus = ''.join(corpus)\n",
    "print(full_corpus[0:300])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'import sys <EOL> def prod2d ( vec1 , vec2 ) : <EOL> return vec1 [ <NUM_LIT:0> ] * vec2 [ <NUM_LIT:1> ] - vec1 [ <NUM_LIT:1> ] * vec2 [ <NUM_LIT:0> ] <EOL> lineNumber = <NUM_LIT:0> <EOL> for line in sys . stdin . readlines ( ) : <EOL> lineNumber += <NUM_LIT:1> <EOL> List = map ( float , line . strip ( ) . split ( ) ) <EOL> nodes = [ [ List [ <NUM_LIT:0> ] , List [ <NUM_LIT:1> ] ] , [ List [ <NUM_LIT:2> ] , List [ <NUM_LIT:3> ] ] , [ List [ <NUM_LIT:4> ] , List [ <NUM_LIT:5> ] ] ] <EOL> point = [ List [ <NUM_LIT:6> ] , List [ <NUM_LIT:7> ] ] <EOL> ABvec = [ nodes [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] - nodes [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] , nodes [ <NUM_LIT:1> ] [ <NUM_LIT:1> ] - nodes [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] ] <EOL> BCvec = [ nodes [ <NUM_LIT:2> ] [ <NUM_LIT:0> ] - nodes [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] , nodes [ <NUM_LIT:2> ] [ <NUM_LIT:1> ] - nodes [ <NUM_LIT:1> ] [ <NUM_LIT:1> ] ] <EOL> CAvec = [ nodes [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] - nodes [ <NUM_LIT:2> ] [ <NUM_LIT:0> ] , nodes [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] - nodes [ <NUM_LIT:2> ] [ <NUM_LIT:1> ] ] <EOL> APvec = [ point [ <NUM_LIT:0> ] - nodes [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] , point [ <NUM_LIT:1> ] - nodes [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] ] <EOL> BPvec = [ point [ <NUM_LIT:0> ] - nodes [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] , point [ <NUM_LIT:1> ] - nodes [ <NUM_LIT:1> ] [ <NUM_LIT:1> ] ] <EOL> CPvec = [ point [ <NUM_LIT:0> ] - nodes [ <NUM_LIT:2> ] [ <NUM_LIT:0> ] , point [ <NUM_LIT:1> ] - nodes [ <NUM_LIT:2> ] [ <NUM_LIT:1> ] ] <EOL> a = prod2d ( CAvec , APvec ) <EOL> b = prod2d ( ABvec , BPvec ) <EOL> c = prod2d ( BCvec , CPvec ) <EOL> if a > <NUM_LIT:0> and b > <NUM_LIT:0> and c > <NUM_LIT:0> : print \"<STR_LIT>\" <EOL> elif a < <NUM_LIT:0> and b < <NUM_LIT:0> and c < <NUM_LIT:0> : print \"<STR_LIT>\" <EOL> else : print \"<STR_LIT>\"'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['import sys ', ' def prod2d ( vec1 , vec2 ) : ', ' return vec1 [ <NUM_LIT:0> ] * vec2 [ <NUM_LIT:1> ] - vec1 [ <NUM_LIT:1> ] * vec2 [ <NUM_LIT:0> ] ', ' lineNumber = <NUM_LIT:0> ', ' for line in sys . stdin . readlines ( ) : ', ' lineNumber += <NUM_LIT:1> ', ' List = map ( float , line . strip ( ) . split ( ) ) ', ' nodes = [ [ List [ <NUM_LIT:0> ] , List [ <NUM_LIT:1> ] ] , [ List [ <NUM_LIT:2> ] , List [ <NUM_LIT:3> ] ] , [ List [ <NUM_LIT:4> ] , List [ <NUM_LIT:5> ] ] ] ', ' point = [ List [ <NUM_LIT:6> ] , List [ <NUM_LIT:7> ] ] ', ' ABvec = [ nodes [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] - nodes [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] , nodes [ <NUM_LIT:1> ] [ <NUM_LIT:1> ] - nodes [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] ] ', ' BCvec = [ nodes [ <NUM_LIT:2> ] [ <NUM_LIT:0> ] - nodes [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] , nodes [ <NUM_LIT:2> ] [ <NUM_LIT:1> ] - nodes [ <NUM_LIT:1> ] [ <NUM_LIT:1> ] ] ', ' CAvec = [ nodes [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] - nodes [ <NUM_LIT:2> ] [ <NUM_LIT:0> ] , nodes [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] - nodes [ <NUM_LIT:2> ] [ <NUM_LIT:1> ] ] ', ' APvec = [ point [ <NUM_LIT:0> ] - nodes [ <NUM_LIT:0> ] [ <NUM_LIT:0> ] , point [ <NUM_LIT:1> ] - nodes [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] ] ', ' BPvec = [ point [ <NUM_LIT:0> ] - nodes [ <NUM_LIT:1> ] [ <NUM_LIT:0> ] , point [ <NUM_LIT:1> ] - nodes [ <NUM_LIT:1> ] [ <NUM_LIT:1> ] ] ', ' CPvec = [ point [ <NUM_LIT:0> ] - nodes [ <NUM_LIT:2> ] [ <NUM_LIT:0> ] , point [ <NUM_LIT:1> ] - nodes [ <NUM_LIT:2> ] [ <NUM_LIT:1> ] ] ', ' a = prod2d ( CAvec , APvec ) ', ' b = prod2d ( ABvec , BPvec ) ', ' c = prod2d ( BCvec , CPvec ) ', ' if a > <NUM_LIT:0> and b > <NUM_LIT:0> and c > <NUM_LIT:0> : print \"<STR_LIT>\" ', ' elif a < <NUM_LIT:0> and b < <NUM_LIT:0> and c < <NUM_LIT:0> : print \"<STR_LIT>\" ']\n",
      "3815\n"
     ]
    }
   ],
   "source": [
    "corpus_new = []\n",
    "for code in corpus:\n",
    "    corpus_new.extend(code.split('<EOL>'))\n",
    "print(corpus_new[0:20])\n",
    "print(len(corpus_new))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Training sequences:\n",
      " [45, 46]\n",
      "\n",
      "Padded training sequences:\n",
      " [45 46  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0\n",
      "  0  0]\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "num_words = 200\n",
    "oov_token = '<UNK>'\n",
    "pad_type = 'post'\n",
    "trunc_type = 'post'\n",
    "\n",
    "tokenizer = Tokenizer(num_words=num_words, filters='')\n",
    "tokenizer.fit_on_texts(corpus_new)\n",
    "\n",
    "# Get our training data word index\n",
    "word_index = tokenizer.word_index\n",
    "\n",
    "# Encode training data sentences into sequences\n",
    "train_sequences = tokenizer.texts_to_sequences(corpus_new)\n",
    "\n",
    "# Get max training sequence length\n",
    "maxlen = max([len(x) for x in train_sequences])\n",
    "\n",
    "# Pad the training sequences\n",
    "train_padded = pad_sequences(train_sequences, padding=pad_type, truncating=trunc_type, maxlen=maxlen)\n",
    "\n",
    "# Output the results of our work\n",
    "# print(\"Word index:\\n\", word_index)\n",
    "print(\"\\nTraining sequences:\\n\", train_sequences[0])\n",
    "print(\"\\nPadded training sequences:\\n\", train_padded[0])\n",
    "# print(\"\\nPadded training shape:\", train_padded.shape)\n",
    "# print(\"Training sequences data type:\", type(train_sequences))\n",
    "# print(\"Padded Training sequences data type:\", type(train_padded))\n",
    "\n",
    "pickle.dump(tokenizer, open('tokenizer-mix.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "746\n"
     ]
    }
   ],
   "source": [
    "vocab_size = len(tokenizer.word_index) + 1\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OrderedDict([('import', 100), ('sys', 97), ('def', 300), ('prod2d', 4), ('(', 2531), ('vec1', 11), (',', 2080), ('vec2', 11), (')', 2531), (':', 1435), ('return', 290), ('[', 2125), ('<num_lit:0>', 827), (']', 2125), ('*', 435), ('<num_lit:1>', 1341), ('-', 800), ('linenumber', 5), ('=', 1346), ('for', 415), ('line', 76), ('in', 422), ('.', 1012), ('stdin', 47), ('readlines', 10), ('+=', 355), ('list', 73), ('map', 53), ('float', 55), ('strip', 19), ('split', 85), ('nodes', 19), ('<num_lit:2>', 594), ('<num_lit:3>', 113), ('<num_lit:4>', 43), ('<num_lit:5>', 15), ('point', 55), ('<num_lit:6>', 15), ('<num_lit:7>', 19), ('abvec', 2), ('bcvec', 2), ('cavec', 2), ('apvec', 2), ('bpvec', 2), ('cpvec', 2), ('a', 546), ('b', 423), ('c', 295), ('if', 487), ('>', 92), ('and', 111), ('print', 185), ('\"<str_lit>\"', 74), ('elif', 57), ('<', 120), ('else', 108), ('gcd', 31), ('%', 49), ('lcm', 19), ('/', 161), ('data', 31), ('int', 124), ('d', 92), ('e', 70), ('f', 58), ('==', 249), ('y', 794), ('<num_lit:1.0>', 17), ('x', 887), ('round', 25), ('continue', 18), ('ix', 3), ('jx', 3), ('iy', 3), ('jy', 3), ('solve', 6), ('agn', 3), ('+', 713), ('<num_lit:0.>', 22), ('while', 51), ('try', 46), ('n', 178), ('input', 85), ('i', 561), ('xrange', 36), ('t', 73), ('p', 171), ('q', 20), ('s', 145), ('raw_input', 11), ('j', 242), ('<num_lit:1.>', 6), ('r', 80), ('**', 156), ('<num_lit:0.5>', 13), ('except', 46), ('eoferror', 23), ('break', 59), ('sheet', 10), ('_', 16), ('range', 263), ('<num_lit:10>', 124), ('small_range', 2), ('middle_range', 2), ('large_range', 2), ('drop', 24), ('drop_range', 2), ('dx', 16), ('dy', 16), ('newx', 3), ('newy', 3), ('<=', 86), ('<num_lit:9>', 58), ('true', 84), ('\"<str_lit:u+002c>\"', 13), ('zero_cnt', 3), ('max_ink', 4), ('ink', 10), ('count', 28), ('-=', 11), ('tmp', 33), ('\"\"\"<str_lit>\"\"\"', 6), ('math', 84), ('segment_sieve', 2), ('begin', 9), ('end', 34), ('assert', 2), ('sqrt_end', 4), ('ceil', 3), ('sqrt', 37), ('is_prime_small', 5), ('false', 47), ('is_prime', 24), ('k', 60), ('max', 52), ('//', 14), ('primes', 4), ('cond', 2), ('enumerate', 15), ('main', 42), ('ns', 3), ('max_n', 2), ('sum', 20), ('__name__', 31), (\"'<str_lit:__main__>'\", 20), ('my_round', 1), ('floor', 1), ('copysign', 1), ('class', 38), ('equation', 2), ('__init__', 22), ('self', 548), ('calc', 2), ('temp', 16), ('temp_a', 5), ('temp_b', 5), ('zerodivisionerror', 1), ('ans', 42), ('format', 23), ('append', 66), ('num', 88), ('\"<str_lit:__main__>\"', 11), ('from', 24), ('itertools', 4), ('accumulate', 2), ('readline', 17), ('list2d', 1), ('list3d', 1), ('list4d', 1), ('none', 23), ('is', 17), ('yes', 2), (\"'<str_lit>'\", 141), ('no', 42), ('setrecursionlimit', 5), ('inf', 4), ('mod', 5), ('eratosthenes_sieve', 2), ('table', 20), ('prime_list', 14), ('<num_lit>', 305), ('acc', 2), ('x1', 85), ('y1', 77), ('x2', 68), ('y2', 64), ('x3', 54), ('y3', 52), ('z1', 4), ('z2', 4), ('z3', 4), ('a11', 7), ('a12', 7), ('a13', 7), ('a21', 7), ('a22', 7), ('a23', 7), ('a31', 7), ('a32', 7), ('a33', 7), ('det', 10), ('l', 72), ('m', 34), ('sol', 2), ('array', 30), ('len', 78), ('res', 14), ('pi', 12), ('output', 21), ('\"<str_lit:u+0020>\"', 17), ('!=', 37), ('ttl', 2), ('<num_lit:12>', 9), ('size', 68), ('index_lis', 6), ('add_x', 2), ('add_y', 2), ('_x', 4), ('_y', 4), ('check', 21), ('total', 14), ('max_num', 4), ('lis', 3), (\"'<str_lit:u+002c>'\", 11), ('w', 5), ('o', 1), ('find', 2), (\"'<str_lit:u+0020>'\", 14), ('or', 32), (\"'<str_lit:.>'\", 1), ('chr', 1), ('ord', 10), (\"'<str_lit:a>'\", 2), ('decimal', 10), ('input_line', 7), ('multiplicand1', 9), ('multiplicand2', 9), ('calculate1', 2), ('after_y1', 6), ('after_y2', 6), ('after_p', 6), ('after_q', 6), ('calculate2', 2), ('calculate3', 2), ('>=', 62), ('abs', 47), ('circle', 10), ('v', 8), ('xcenter', 2), ('ycenter', 2), ('radius', 2), ('xc', 2), ('yc', 2), ('ra', 2), ('loop', 3), ('max_total', 4), ('get_input', 5), ('yield', 4), ('join', 6), ('calcgcd', 2), ('large', 15), ('small', 19), ('temp_small', 4), ('calclcm', 2), ('bisect', 4), ('bisect_right', 2), ('max_number', 8), ('prime_flag_list', 16), ('paper', 53), ('white_space', 2), ('most_dark', 2), ('pass', 8), ('filter', 1), ('out_of_paper', 2), ('tuple', 1), ('dropinc', 4), ('coord', 4), ('<num_lit:8>', 20), ('object', 15), ('__repr__', 2), ('p1', 43), ('p2', 37), ('epsilon', 9), ('is_over', 7), ('run', 4), ('xp', 21), ('yp', 17), ('p3', 24), ('l_12', 3), ('l_23', 3), ('l_31', 3), ('&=', 2), ('points_list', 89), ('sort', 3), ('py', 40), ('px', 51), ('a2_s', 8), ('a2', 16), ('m13x', 8), ('m13y', 8), ('b2', 10), ('a1_s', 4), ('a1', 11), ('m12x', 4), ('m12y', 4), ('b1', 7), ('pow', 3), ('heapq', 4), ('collections', 7), ('deque', 3), ('enum', 2), ('_heapq', 1), ('heappush', 2), ('heappop', 2), ('copy', 5), ('big_num', 1), ('huge_num', 1), ('eps', 9), ('num_query', 2), ('reverse', 3), ('add', 9), ('digit', 5), ('pop', 3), ('s1i', 3), ('s1j', 2), ('s2i', 3), ('s2j', 2), ('s3i', 3), ('s3j', 2), ('xx', 12), ('yy', 12), ('cnt', 43), ('inputline', 3), ('mymath', 16), ('pnum_eratosthenes', 6), ('ptable', 15), ('plist', 15), ('mul', 5), ('row', 14), ('ab', 22), ('calc_digit', 2), ('data1', 10), ('data2', 12), ('data_size2', 4), ('str', 9), ('data_size1', 4), ('two_number_sum', 2), ('result', 32), ('edge', 8), ('area', 43), ('lawofcosines', 8), ('acos', 6), (';', 76), ('is_same', 2), ('triangle', 24), ('edgea', 25), ('edgeb', 22), ('edgec', 22), ('anglea', 10), ('angleb', 7), ('anglec', 13), ('circumscribeadcircleradius', 3), ('sin', 17), ('circumscribedcirclecenter', 5), ('inscribedcircleradius', 2), ('inscribedcirclecenter', 2), ('points', 23), ('edges', 6), ('isinner', 3), ('cross', 4), ('lambda', 7), ('c1', 10), ('c2', 10), ('pnum_check', 2), ('is_integer', 4), ('valueerror', 4), ('dist', 2), ('printa', 2), ('*=', 7), ('xg', 4), ('yg', 4), ('xi', 4), ('yi', 3), ('xj', 4), ('yj', 3), ('line12', 4), ('line23', 4), ('line13', 4), ('<num_lit:11>', 6), ('fabs', 6), ('calcu_cirucumcenter', 2), ('<num_lit:0.0>', 4), ('min', 10), ('extend', 1), ('prim_no', 20), ('{', 11), ('}', 11), ('get', 24), ('not', 14), ('max_check', 8), ('prim_vals', 24), ('num_data', 8), ('sorted_num_data', 4), ('sorted', 6), ('prim_num', 6), ('start_num', 12), ('<num_lit:1000>', 7), ('g', 9), (\"'''<str_lit>'''\", 2), ('limit', 2), ('vector2', 7), ('namedtuple', 3), ('__slots__', 1), ('__add__', 2), ('other', 15), ('__sub__', 2), ('__mul__', 3), ('__neg__', 1), ('__pos__', 1), ('__abs__', 1), ('dotproduct', 1), ('x4', 7), ('y4', 7), ('z', 4), ('vector_ab', 2), ('vector_cd', 2), ('cos', 2), ('radians', 2), ('degrees', 3), ('cosa', 11), ('vertex', 11), ('bc', 13), ('ca', 13), ('cosb', 9), ('cosc', 9), ('sina', 7), ('sinb', 6), ('sinc', 8), ('sin2a', 15), ('sin2b', 15), ('sin2c', 15), ('circumscribed', 3), ('eval', 1), ('prod_mat_vec', 2), ('vec', 12), ('ret', 3), ('changerow', 3), ('i1', 4), ('i2', 4), ('buf', 2), ('matrixinverse3x3', 2), ('a_src', 2), ('/=', 5), ('output3x3', 1), ('lmn', 6), ('cmath', 8), ('complex', 3), ('__str__', 4), ('real', 7), ('imag', 6), ('angle', 8), ('t1', 10), ('t2', 10), ('t3', 3), ('string', 8), ('fractions', 3), ('re', 3), ('random', 2), ('time', 2), ('<num_lit:20>', 7), ('li', 2), ('lf', 3), ('ls', 2), ('matrix', 7), ('col', 5), ('__iter__', 2), ('__getitem__', 2), ('aa', 6), ('bb', 6), ('ai', 2), ('__truediv__', 1), ('lu', 2), ('deepcopy', 2), ('u', 6), ('solve_se', 2), ('sa', 2), ('read', 1), (\"'<str_lit:\\\\n>'\", 3), ('pap', 5), ('brot', 2), ('raise', 1), ('bmax', 4), ('wcount', 3), ('is_prime_2', 2), ('prime_count', 1), ('l2', 3), ('prime_count_2', 2), ('n1', 13), ('n2', 12), ('kotae', 6), ('shorter', 3), ('longer', 2), ('idx', 9), ('kuriagari', 6), ('\"<str_lit:1>\"', 1), ('norm2d', 2), ('normalize', 3), ('mass', 79), ('white', 3), ('__future__', 1), ('division', 1), ('absolute_import', 1), ('print_function', 1), ('unicode_literals', 1), ('operator', 2), ('attrgetter', 2), ('linear', 5), ('gradient', 3), ('y_intercept', 2), ('takeout', 2), ('seq', 5), ('gen', 2), ('next', 4), ('key', 2), (\"'<str_lit:x>'\", 1), ('circumscribedcircleradius', 2), ('center', 9), ('func', 2), ('h', 6), ('fx', 12), ('dataset', 2), ('a_tmp1', 2), ('a_tmp2', 2), ('flagx', 2), ('flagy', 2), ('item', 4), ('masu', 6), ('access', 10), ('kosu', 3), ('komax', 4), ('arr', 22), ('<num_lit:100>', 2), ('stl1', 2), ('stl2', 2), ('stl3', 3), ('lst', 20), ('point1', 2), ('point2', 2), ('point3', 2), ('pointlst', 8), ('bdr1', 3), ('bdr3', 6), ('bdr2', 3), ('start', 11), ('pmb', 5), ('vector', 16), ('@', 3), ('staticmethod', 1), ('cross_product', 11), ('isin', 2), ('ap', 3), ('bp', 3), ('cp', 7), ('gauss', 2), ('div', 2), ('r2', 3), ('head', 2), ('length', 2), ('solve_sim_equ', 2), ('circumscribed_circle', 2), ('get_equ_coef', 3), ('h_x', 2), ('h_y', 2), ('coef', 2), ('vs', 7), ('tmp_lst', 8), ('compress', 2), ('accumurate_lst', 4), ('calc_accumurate_lst', 2), ('max_value', 4), ('to', 4), ('frm', 3), ('value', 3), ('exit', 2), ('accum', 5), ('lim', 3), ('index', 8), ('smallink', 2), ('bigink', 2), ('answermap', 53), ('<num_lit:15>', 1), ('<num_lit:16>', 1), ('<num_lit:30>', 7), ('<num_lit:32>', 1), ('<num_lit:50>', 1), ('inputs', 3), ('lines', 2), ('gx', 5), ('gy', 5), ('mediam', 3), ('ax', 2), ('ay', 2), ('bx', 2), ('by', 2), ('cx', 2), ('cy', 2), ('pax', 2), ('pay', 2), ('pbx', 2), ('pby', 2), ('pcx', 2), ('pcy', 2), ('pa', 2), ('pb', 2), ('pc', 2), ('cp1', 8), ('cp2', 8), ('cp3', 8), ('sig1', 3), ('sig2', 3), ('sig3', 3), ('width', 4), ('calc_cos', 4), ('calc_sin', 4), ('calc_2sin', 4), ('grid', 42), ('middle', 2), ('__', 1), ('ink_size', 3), ('max_element', 4), ('functools', 1), ('dd', 1), ('ddn', 1), ('li_', 1), ('pf', 1), ('flush', 1), ('distance', 2), ('intersection', 2), ('ksi', 2), ('eta', 2), ('delta', 4), ('ramda', 5), ('mu', 3), ('circumcenters', 2), ('s1', 5), ('s2', 5), ('p4', 2), ('rr', 3), ('solution', 2), ('combinationsum', 2), ('target', 3), ('candidates', 5), ('stack', 4), ('duplicated', 2), ('nums', 18), ('recursion', 3), ('permuteunique', 2), ('combination', 3), ('pnum_fermat', 1), ('pnum', 3), ('n_pn', 2), ('upper', 1), ('check_exists', 4), ('word_set', 2), ('rotate_dict', 8), ('search', 4), ('word', 19), ('rotated_word', 2), ('alphabets', 5), ('ascii_lowercase', 2), ('frequency', 2), ('freq_chr_offset', 2), ('ss', 3), ('sl3', 2), ('sl4', 3), ('set', 2), ('order', 2), ('offset', 2), ('zip', 1), ('isalpha', 1), ('isplus', 4), ('maxvalue', 8), ('isleft', 4), ('tmpvector', 2), ('directedvector', 5), ('property', 2), ('getdatasets', 2), ('evaluate', 3), ('all', 1), ('any', 1), ('defaultdict', 1), ('counter', 4), ('hypot', 1), ('factorial', 1), ('permutations', 1), ('combinations', 1), ('product', 1), ('itemgetter', 1), ('ascii_uppercase', 1), ('digits', 1), ('bisect_left', 1), ('primes_for', 2), ('direction_vector', 7), ('->', 2), ('v1', 3), ('v2', 3), ('points_receive', 3), ('p1_to_p2', 2), ('p2_to_p3', 2), ('p3_to_p1', 2), ('p1_to_pp', 2), ('p2_to_pp', 2), ('p3_to_pp', 2), ('init', 2), ('drop_small', 3), ('drop_medium', 3), ('drop_large', 2), ('count_white', 2), ('find_max', 2), ('mx', 4), ('global', 2), ('small_inc', 2), ('cells', 36), ('mapsize', 17), ('medium_inc', 2), ('large_inc', 2), ('medium', 5), ('max_d', 4), ('small_ink', 2), ('mid_ink', 3), ('big_ink', 2), ('nu', 3), ('max_val', 4), ('flag', 13), ('thickest', 4), ('alpha', 4), ('this', 3), ('that', 3), ('the', 3), ('di', 29), ('maxcnt', 4), ('x_inp', 2), ('y_inp', 2), ('p_trim', 2), ('p_flatten', 3)])\n"
     ]
    }
   ],
   "source": [
    "print(tokenizer.word_counts)\n",
    "# print(tokenizer.document_count)\n",
    "# print(tokenizer.word_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test: are done\n",
      "['import math , string , itertools , fractions , heapq , collections , re , array , bisect , sys , random , time , copy ', ' sys . setrecursionlimit ( <NUM_LIT:10> ** <NUM_LIT:7> ) ', ' inf = <NUM_LIT:10> ** <NUM_LIT:20> ', ' mod = <NUM_LIT:10> ** <NUM_LIT:9> + <NUM_LIT:7> ', ' def LI ( ) : return [ int ( x ) for x in sys . stdin . readline ( ) . split ( ) ] ', ' def LF ( ) : return [ float ( x ) for x in sys . stdin . readline ( ) . split ( ) ] ', ' def LS ( ) : return sys . stdin . readline ( ) . split ( ) ', ' def I ( ) : return int ( sys . stdin . readline ( ) ) ', ' def F ( ) : return float ( sys . stdin . readline ( ) ) ', ' def S ( ) : return input ( ) ', ' class Matrix ( ) : ', ' def __init__ ( self , A ) : ', ' self . A = A ', ' self . row = len ( A ) ', ' self . col = len ( A [ <NUM_LIT:0> ] ) ', ' def __iter__ ( self ) : ', ' return self . A . __iter__ ( ) ', ' def __getitem__ ( self , i ) : ', ' return self . A . __getitem__ ( i ) ', ' def __add__ ( self , B ) : ']\n",
      "759\n"
     ]
    }
   ],
   "source": [
    "test_corpus = []\n",
    "for code in py_tokenize(\"test\"):\n",
    "    test_corpus.extend(code.split('<EOL>'))\n",
    "print(test_corpus[0:20])\n",
    "print(len(test_corpus))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[80, 46, 1, 28, 2, 6, 3, 7, 1, 29, 4, 12, 5, 21, 4, 9, 5, 13, 4, 9, 5, 21, 4, 12, 5, 1, 8, 12, 1, 24, 61, 23, 46, 10, 90, 10, 2, 3, 7, 1, 25, 9, 1, 63, 8, 81, 2, 77, 6, 61]\n"
     ]
    }
   ],
   "source": [
    "tokenizer2 = Tokenizer(num_words=num_words, filters='')\n",
    "tokenizer2.fit_on_texts([full_corpus])\n",
    "\n",
    "sequence_data = tokenizer2.texts_to_sequences([full_corpus])[0]\n",
    "print(sequence_data[:50])\n",
    "\n",
    "pickle.dump(tokenizer2, open('tokenizer-mix1.pkl', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "774\n"
     ]
    }
   ],
   "source": [
    "vocab_size = len(tokenizer2.word_index) + 1\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(tokenizer2.word_counts)\n",
    "# print(tokenizer.document_count)\n",
    "# print(tokenizer.word_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Length of sequences are:  38387\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[80, 46],\n",
       "       [46,  1],\n",
       "       [ 1, 28],\n",
       "       [28,  2],\n",
       "       [ 2,  6],\n",
       "       [ 6,  3],\n",
       "       [ 3,  7],\n",
       "       [ 7,  1],\n",
       "       [ 1, 29],\n",
       "       [29,  4]])"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequences = []\n",
    "\n",
    "for i in range(1, len(sequence_data)):\n",
    "    words = sequence_data[i-1:i+1]\n",
    "    sequences.append(words)\n",
    "    \n",
    "print(\"The Length of sequences are: \", len(sequences))\n",
    "sequences = np.array(sequences)\n",
    "sequences[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = []\n",
    "y = []\n",
    "\n",
    "for i in sequences:\n",
    "    X.append(i[0])\n",
    "    y.append(i[1])\n",
    "    \n",
    "X = np.array(X)\n",
    "y = np.array(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Data is:  [80 46  1 28  2]\n",
      "The responses are:  [46  1 28  2  6]\n"
     ]
    }
   ],
   "source": [
    "print(\"The Data is: \", X[:5])\n",
    "print(\"The responses are: \", y[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 1., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 1., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = to_categorical(y, num_classes=vocab_size)\n",
    "y[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Embedding(vocab_size, 10, input_length=1))\n",
    "model.add(LSTM(1000, return_sequences=True))\n",
    "model.add(LSTM(1000))\n",
    "model.add(Dense(1000, activation=\"relu\"))\n",
    "model.add(Dense(vocab_size, activation=\"softmax\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_3 (Embedding)      (None, 1, 10)             7740      \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 1, 1000)           4044000   \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (None, 1000)              8004000   \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1000)              1001000   \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 774)               774774    \n",
      "=================================================================\n",
      "Total params: 13,831,514\n",
      "Trainable params: 13,831,514\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "from tensorflow.keras.callbacks import ReduceLROnPlateau\n",
    "from tensorflow.keras.callbacks import TensorBoard\n",
    "\n",
    "checkpoint = ModelCheckpoint(\"nextword-mix1.h5\", monitor='loss', verbose=1,\n",
    "    save_best_only=True, mode='auto')\n",
    "\n",
    "reduce = ReduceLROnPlateau(monitor='loss', factor=0.2, patience=3, min_lr=0.0001, verbose = 1)\n",
    "\n",
    "logdir='logsnextword1'\n",
    "tensorboard_Visualization = TensorBoard(log_dir=logdir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=\"categorical_crossentropy\", optimizer=Adam(lr=0.001))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "600/600 [==============================] - 13s 17ms/step - loss: 4.0644\n",
      "\n",
      "Epoch 00001: loss improved from inf to 3.47923, saving model to nextword-mix1.h5\n",
      "Epoch 2/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.8405\n",
      "\n",
      "Epoch 00002: loss improved from 3.47923 to 2.79597, saving model to nextword-mix1.h5\n",
      "Epoch 3/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.6715\n",
      "\n",
      "Epoch 00003: loss improved from 2.79597 to 2.66644, saving model to nextword-mix1.h5\n",
      "Epoch 4/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.5968\n",
      "\n",
      "Epoch 00004: loss improved from 2.66644 to 2.60510, saving model to nextword-mix1.h5\n",
      "Epoch 5/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.5662\n",
      "\n",
      "Epoch 00005: loss improved from 2.60510 to 2.57011, saving model to nextword-mix1.h5\n",
      "Epoch 6/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.5408\n",
      "\n",
      "Epoch 00006: loss improved from 2.57011 to 2.53698, saving model to nextword-mix1.h5\n",
      "Epoch 7/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.5295\n",
      "\n",
      "Epoch 00007: loss improved from 2.53698 to 2.51989, saving model to nextword-mix1.h5\n",
      "Epoch 8/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4876\n",
      "\n",
      "Epoch 00008: loss improved from 2.51989 to 2.50140, saving model to nextword-mix1.h5\n",
      "Epoch 9/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4884\n",
      "\n",
      "Epoch 00009: loss improved from 2.50140 to 2.49163, saving model to nextword-mix1.h5\n",
      "Epoch 10/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4724\n",
      "\n",
      "Epoch 00010: loss improved from 2.49163 to 2.47961, saving model to nextword-mix1.h5\n",
      "Epoch 11/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4730\n",
      "\n",
      "Epoch 00011: loss improved from 2.47961 to 2.47192, saving model to nextword-mix1.h5\n",
      "Epoch 12/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4526\n",
      "\n",
      "Epoch 00012: loss improved from 2.47192 to 2.46567, saving model to nextword-mix1.h5\n",
      "Epoch 13/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4508\n",
      "\n",
      "Epoch 00013: loss improved from 2.46567 to 2.46025, saving model to nextword-mix1.h5\n",
      "Epoch 14/150\n",
      "600/600 [==============================] - 11s 18ms/step - loss: 2.4482\n",
      "\n",
      "Epoch 00014: loss improved from 2.46025 to 2.45432, saving model to nextword-mix1.h5\n",
      "Epoch 15/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4522\n",
      "\n",
      "Epoch 00015: loss improved from 2.45432 to 2.44970, saving model to nextword-mix1.h5\n",
      "Epoch 16/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4477\n",
      "\n",
      "Epoch 00016: loss improved from 2.44970 to 2.44388, saving model to nextword-mix1.h5\n",
      "Epoch 17/150\n",
      "600/600 [==============================] - 11s 18ms/step - loss: 2.4355\n",
      "\n",
      "Epoch 00017: loss improved from 2.44388 to 2.43970, saving model to nextword-mix1.h5\n",
      "Epoch 18/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4232\n",
      "\n",
      "Epoch 00018: loss improved from 2.43970 to 2.43510, saving model to nextword-mix1.h5\n",
      "Epoch 19/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4347\n",
      "\n",
      "Epoch 00019: loss improved from 2.43510 to 2.43413, saving model to nextword-mix1.h5\n",
      "Epoch 20/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4178\n",
      "\n",
      "Epoch 00020: loss improved from 2.43413 to 2.42954, saving model to nextword-mix1.h5\n",
      "Epoch 21/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4124\n",
      "\n",
      "Epoch 00021: loss improved from 2.42954 to 2.42785, saving model to nextword-mix1.h5\n",
      "Epoch 22/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4347\n",
      "\n",
      "Epoch 00022: loss improved from 2.42785 to 2.42448, saving model to nextword-mix1.h5\n",
      "Epoch 23/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4064\n",
      "\n",
      "Epoch 00023: loss improved from 2.42448 to 2.42273, saving model to nextword-mix1.h5\n",
      "Epoch 24/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4173\n",
      "\n",
      "Epoch 00024: loss improved from 2.42273 to 2.42169, saving model to nextword-mix1.h5\n",
      "Epoch 25/150\n",
      "600/600 [==============================] - 11s 18ms/step - loss: 2.4200\n",
      "\n",
      "Epoch 00025: loss improved from 2.42169 to 2.41885, saving model to nextword-mix1.h5\n",
      "Epoch 26/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4114\n",
      "\n",
      "Epoch 00026: loss improved from 2.41885 to 2.41819, saving model to nextword-mix1.h5\n",
      "Epoch 27/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4103\n",
      "\n",
      "Epoch 00027: loss improved from 2.41819 to 2.41546, saving model to nextword-mix1.h5\n",
      "Epoch 28/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3940\n",
      "\n",
      "Epoch 00028: loss improved from 2.41546 to 2.41420, saving model to nextword-mix1.h5\n",
      "Epoch 29/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4009\n",
      "\n",
      "Epoch 00029: loss improved from 2.41420 to 2.41351, saving model to nextword-mix1.h5\n",
      "Epoch 30/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3886\n",
      "\n",
      "Epoch 00030: loss improved from 2.41351 to 2.41150, saving model to nextword-mix1.h5\n",
      "Epoch 31/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4003\n",
      "\n",
      "Epoch 00031: loss improved from 2.41150 to 2.41020, saving model to nextword-mix1.h5\n",
      "Epoch 32/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3919\n",
      "\n",
      "Epoch 00032: loss improved from 2.41020 to 2.40861, saving model to nextword-mix1.h5\n",
      "Epoch 33/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4111\n",
      "\n",
      "Epoch 00033: loss improved from 2.40861 to 2.40780, saving model to nextword-mix1.h5\n",
      "Epoch 34/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3996\n",
      "\n",
      "Epoch 00034: loss improved from 2.40780 to 2.40759, saving model to nextword-mix1.h5\n",
      "Epoch 35/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4060\n",
      "\n",
      "Epoch 00035: loss improved from 2.40759 to 2.40612, saving model to nextword-mix1.h5\n",
      "Epoch 36/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3966\n",
      "\n",
      "Epoch 00036: loss improved from 2.40612 to 2.40484, saving model to nextword-mix1.h5\n",
      "Epoch 37/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4125\n",
      "\n",
      "Epoch 00037: loss did not improve from 2.40484\n",
      "Epoch 38/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4023\n",
      "\n",
      "Epoch 00038: loss improved from 2.40484 to 2.40282, saving model to nextword-mix1.h5\n",
      "Epoch 39/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4043\n",
      "\n",
      "Epoch 00039: loss improved from 2.40282 to 2.40266, saving model to nextword-mix1.h5\n",
      "Epoch 40/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3886\n",
      "\n",
      "Epoch 00040: loss improved from 2.40266 to 2.40071, saving model to nextword-mix1.h5\n",
      "Epoch 41/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3929\n",
      "\n",
      "Epoch 00041: loss did not improve from 2.40071\n",
      "Epoch 42/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3834\n",
      "\n",
      "Epoch 00042: loss improved from 2.40071 to 2.39936, saving model to nextword-mix1.h5\n",
      "Epoch 43/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3929\n",
      "\n",
      "Epoch 00043: loss did not improve from 2.39936\n",
      "Epoch 44/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3990\n",
      "\n",
      "Epoch 00044: loss improved from 2.39936 to 2.39930, saving model to nextword-mix1.h5\n",
      "Epoch 45/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3923\n",
      "\n",
      "Epoch 00045: loss improved from 2.39930 to 2.39787, saving model to nextword-mix1.h5\n",
      "Epoch 46/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3973\n",
      "\n",
      "Epoch 00046: loss improved from 2.39787 to 2.39720, saving model to nextword-mix1.h5\n",
      "Epoch 47/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3875\n",
      "\n",
      "Epoch 00047: loss improved from 2.39720 to 2.39652, saving model to nextword-mix1.h5\n",
      "Epoch 48/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3936\n",
      "\n",
      "Epoch 00048: loss improved from 2.39652 to 2.39648, saving model to nextword-mix1.h5\n",
      "Epoch 49/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3760\n",
      "\n",
      "Epoch 00049: loss improved from 2.39648 to 2.39499, saving model to nextword-mix1.h5\n",
      "Epoch 50/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3924\n",
      "\n",
      "Epoch 00050: loss did not improve from 2.39499\n",
      "Epoch 51/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3893\n",
      "\n",
      "Epoch 00051: loss improved from 2.39499 to 2.39461, saving model to nextword-mix1.h5\n",
      "Epoch 52/150\n",
      "600/600 [==============================] - 11s 18ms/step - loss: 2.3908\n",
      "\n",
      "Epoch 00052: loss did not improve from 2.39461\n",
      "Epoch 53/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3885\n",
      "\n",
      "Epoch 00053: loss improved from 2.39461 to 2.39362, saving model to nextword-mix1.h5\n",
      "Epoch 54/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3825\n",
      "\n",
      "Epoch 00054: loss improved from 2.39362 to 2.39351, saving model to nextword-mix1.h5\n",
      "Epoch 55/150\n",
      "600/600 [==============================] - 11s 18ms/step - loss: 2.3893\n",
      "\n",
      "Epoch 00055: loss improved from 2.39351 to 2.39233, saving model to nextword-mix1.h5\n",
      "Epoch 56/150\n",
      "600/600 [==============================] - 11s 18ms/step - loss: 2.3729\n",
      "\n",
      "Epoch 00056: loss improved from 2.39233 to 2.39209, saving model to nextword-mix1.h5\n",
      "Epoch 57/150\n",
      "600/600 [==============================] - 11s 18ms/step - loss: 2.3758\n",
      "\n",
      "Epoch 00057: loss did not improve from 2.39209\n",
      "Epoch 58/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3880\n",
      "\n",
      "Epoch 00058: loss improved from 2.39209 to 2.39132, saving model to nextword-mix1.h5\n",
      "Epoch 59/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.4008\n",
      "\n",
      "Epoch 00059: loss did not improve from 2.39132\n",
      "Epoch 60/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3890\n",
      "\n",
      "Epoch 00060: loss improved from 2.39132 to 2.39106, saving model to nextword-mix1.h5\n",
      "Epoch 61/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3915\n",
      "\n",
      "Epoch 00061: loss did not improve from 2.39106\n",
      "Epoch 62/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3751\n",
      "\n",
      "Epoch 00062: loss improved from 2.39106 to 2.39096, saving model to nextword-mix1.h5\n",
      "Epoch 63/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3851\n",
      "\n",
      "Epoch 00063: loss did not improve from 2.39096\n",
      "\n",
      "Epoch 00063: ReduceLROnPlateau reducing learning rate to 0.00020000000949949026.\n",
      "Epoch 64/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3828\n",
      "\n",
      "Epoch 00064: loss improved from 2.39096 to 2.37323, saving model to nextword-mix1.h5\n",
      "Epoch 65/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3542\n",
      "\n",
      "Epoch 00065: loss improved from 2.37323 to 2.37160, saving model to nextword-mix1.h5\n",
      "Epoch 66/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3773\n",
      "\n",
      "Epoch 00066: loss improved from 2.37160 to 2.37098, saving model to nextword-mix1.h5\n",
      "Epoch 67/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3743\n",
      "\n",
      "Epoch 00067: loss improved from 2.37098 to 2.37054, saving model to nextword-mix1.h5\n",
      "Epoch 68/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3755\n",
      "\n",
      "Epoch 00068: loss did not improve from 2.37054\n",
      "Epoch 69/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3568\n",
      "\n",
      "Epoch 00069: loss improved from 2.37054 to 2.37040, saving model to nextword-mix1.h5\n",
      "Epoch 70/150\n",
      "600/600 [==============================] - 11s 18ms/step - loss: 2.3710\n",
      "\n",
      "Epoch 00070: loss improved from 2.37040 to 2.37032, saving model to nextword-mix1.h5\n",
      "Epoch 71/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3819\n",
      "\n",
      "Epoch 00071: loss improved from 2.37032 to 2.37022, saving model to nextword-mix1.h5\n",
      "Epoch 72/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3702\n",
      "\n",
      "Epoch 00072: loss did not improve from 2.37022\n",
      "Epoch 73/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3722\n",
      "\n",
      "Epoch 00073: loss improved from 2.37022 to 2.37019, saving model to nextword-mix1.h5\n",
      "Epoch 74/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3620\n",
      "\n",
      "Epoch 00074: loss improved from 2.37019 to 2.37007, saving model to nextword-mix1.h5\n",
      "Epoch 75/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3744\n",
      "\n",
      "Epoch 00075: loss did not improve from 2.37007\n",
      "Epoch 76/150\n",
      "600/600 [==============================] - 11s 18ms/step - loss: 2.3753\n",
      "\n",
      "Epoch 00076: loss did not improve from 2.37007\n",
      "Epoch 77/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3855\n",
      "\n",
      "Epoch 00077: loss improved from 2.37007 to 2.36986, saving model to nextword-mix1.h5\n",
      "Epoch 78/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3754\n",
      "\n",
      "Epoch 00078: loss did not improve from 2.36986\n",
      "Epoch 79/150\n",
      "600/600 [==============================] - 11s 18ms/step - loss: 2.3659\n",
      "\n",
      "Epoch 00079: loss improved from 2.36986 to 2.36970, saving model to nextword-mix1.h5\n",
      "Epoch 80/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3699\n",
      "\n",
      "Epoch 00080: loss did not improve from 2.36970\n",
      "Epoch 81/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3779\n",
      "\n",
      "Epoch 00081: loss did not improve from 2.36970\n",
      "Epoch 82/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3639\n",
      "\n",
      "Epoch 00082: loss improved from 2.36970 to 2.36962, saving model to nextword-mix1.h5\n",
      "\n",
      "Epoch 00082: ReduceLROnPlateau reducing learning rate to 0.0001.\n",
      "Epoch 83/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3652\n",
      "\n",
      "Epoch 00083: loss improved from 2.36962 to 2.36691, saving model to nextword-mix1.h5\n",
      "Epoch 84/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3761\n",
      "\n",
      "Epoch 00084: loss improved from 2.36691 to 2.36677, saving model to nextword-mix1.h5\n",
      "Epoch 85/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3815\n",
      "\n",
      "Epoch 00085: loss improved from 2.36677 to 2.36671, saving model to nextword-mix1.h5\n",
      "Epoch 86/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3708\n",
      "\n",
      "Epoch 00086: loss did not improve from 2.36671\n",
      "Epoch 87/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3762\n",
      "\n",
      "Epoch 00087: loss improved from 2.36671 to 2.36660, saving model to nextword-mix1.h5\n",
      "Epoch 88/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3647\n",
      "\n",
      "Epoch 00088: loss did not improve from 2.36660\n",
      "Epoch 89/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3614\n",
      "\n",
      "Epoch 00089: loss did not improve from 2.36660\n",
      "Epoch 90/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3622\n",
      "\n",
      "Epoch 00090: loss improved from 2.36660 to 2.36658, saving model to nextword-mix1.h5\n",
      "Epoch 91/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3601\n",
      "\n",
      "Epoch 00091: loss improved from 2.36658 to 2.36658, saving model to nextword-mix1.h5\n",
      "Epoch 92/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3650\n",
      "\n",
      "Epoch 00092: loss did not improve from 2.36658\n",
      "Epoch 93/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3701\n",
      "\n",
      "Epoch 00093: loss did not improve from 2.36658\n",
      "Epoch 94/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3666\n",
      "\n",
      "Epoch 00094: loss improved from 2.36658 to 2.36640, saving model to nextword-mix1.h5\n",
      "Epoch 95/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3560\n",
      "\n",
      "Epoch 00095: loss did not improve from 2.36640\n",
      "Epoch 96/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3729\n",
      "\n",
      "Epoch 00096: loss did not improve from 2.36640\n",
      "Epoch 97/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3711\n",
      "\n",
      "Epoch 00097: loss did not improve from 2.36640\n",
      "Epoch 98/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3756\n",
      "\n",
      "Epoch 00098: loss did not improve from 2.36640\n",
      "Epoch 99/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3540\n",
      "\n",
      "Epoch 00099: loss did not improve from 2.36640\n",
      "Epoch 100/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3583\n",
      "\n",
      "Epoch 00100: loss did not improve from 2.36640\n",
      "Epoch 101/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3724\n",
      "\n",
      "Epoch 00101: loss did not improve from 2.36640\n",
      "Epoch 102/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3826\n",
      "\n",
      "Epoch 00102: loss did not improve from 2.36640\n",
      "Epoch 103/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3692\n",
      "\n",
      "Epoch 00103: loss did not improve from 2.36640\n",
      "Epoch 104/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3710\n",
      "\n",
      "Epoch 00104: loss improved from 2.36640 to 2.36636, saving model to nextword-mix1.h5\n",
      "Epoch 105/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3812\n",
      "\n",
      "Epoch 00105: loss did not improve from 2.36636\n",
      "Epoch 106/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3601\n",
      "\n",
      "Epoch 00106: loss improved from 2.36636 to 2.36629, saving model to nextword-mix1.h5\n",
      "Epoch 107/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3602\n",
      "\n",
      "Epoch 00107: loss did not improve from 2.36629\n",
      "Epoch 108/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3739\n",
      "\n",
      "Epoch 00108: loss did not improve from 2.36629\n",
      "Epoch 109/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3833\n",
      "\n",
      "Epoch 00109: loss did not improve from 2.36629\n",
      "Epoch 110/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3660\n",
      "\n",
      "Epoch 00110: loss did not improve from 2.36629\n",
      "Epoch 111/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3592\n",
      "\n",
      "Epoch 00111: loss improved from 2.36629 to 2.36628, saving model to nextword-mix1.h5\n",
      "Epoch 112/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3652\n",
      "\n",
      "Epoch 00112: loss improved from 2.36628 to 2.36622, saving model to nextword-mix1.h5\n",
      "Epoch 113/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3755\n",
      "\n",
      "Epoch 00113: loss did not improve from 2.36622\n",
      "Epoch 114/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3579\n",
      "\n",
      "Epoch 00114: loss improved from 2.36622 to 2.36621, saving model to nextword-mix1.h5\n",
      "Epoch 115/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3728\n",
      "\n",
      "Epoch 00115: loss did not improve from 2.36621\n",
      "Epoch 116/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3536\n",
      "\n",
      "Epoch 00116: loss did not improve from 2.36621\n",
      "Epoch 117/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3648\n",
      "\n",
      "Epoch 00117: loss did not improve from 2.36621\n",
      "Epoch 118/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3625\n",
      "\n",
      "Epoch 00118: loss did not improve from 2.36621\n",
      "Epoch 119/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3765\n",
      "\n",
      "Epoch 00119: loss did not improve from 2.36621\n",
      "Epoch 120/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3647\n",
      "\n",
      "Epoch 00120: loss improved from 2.36621 to 2.36619, saving model to nextword-mix1.h5\n",
      "Epoch 121/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3510\n",
      "\n",
      "Epoch 00121: loss did not improve from 2.36619\n",
      "Epoch 122/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3566\n",
      "\n",
      "Epoch 00122: loss improved from 2.36619 to 2.36616, saving model to nextword-mix1.h5\n",
      "Epoch 123/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3446\n",
      "\n",
      "Epoch 00123: loss did not improve from 2.36616\n",
      "Epoch 124/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3676\n",
      "\n",
      "Epoch 00124: loss did not improve from 2.36616\n",
      "Epoch 125/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3669\n",
      "\n",
      "Epoch 00125: loss improved from 2.36616 to 2.36613, saving model to nextword-mix1.h5\n",
      "Epoch 126/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3539\n",
      "\n",
      "Epoch 00126: loss did not improve from 2.36613\n",
      "Epoch 127/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3627\n",
      "\n",
      "Epoch 00127: loss did not improve from 2.36613\n",
      "Epoch 128/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3585\n",
      "\n",
      "Epoch 00128: loss did not improve from 2.36613\n",
      "Epoch 129/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3521\n",
      "\n",
      "Epoch 00129: loss did not improve from 2.36613\n",
      "Epoch 130/150\n",
      "600/600 [==============================] - 11s 18ms/step - loss: 2.3628\n",
      "\n",
      "Epoch 00130: loss did not improve from 2.36613\n",
      "Epoch 131/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3687\n",
      "\n",
      "Epoch 00131: loss did not improve from 2.36613\n",
      "Epoch 132/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3631\n",
      "\n",
      "Epoch 00132: loss did not improve from 2.36613\n",
      "Epoch 133/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3559\n",
      "\n",
      "Epoch 00133: loss improved from 2.36613 to 2.36611, saving model to nextword-mix1.h5\n",
      "Epoch 134/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3579\n",
      "\n",
      "Epoch 00134: loss improved from 2.36611 to 2.36607, saving model to nextword-mix1.h5\n",
      "Epoch 135/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3546\n",
      "\n",
      "Epoch 00135: loss improved from 2.36607 to 2.36607, saving model to nextword-mix1.h5\n",
      "Epoch 136/150\n",
      "600/600 [==============================] - 11s 18ms/step - loss: 2.3641\n",
      "\n",
      "Epoch 00136: loss improved from 2.36607 to 2.36600, saving model to nextword-mix1.h5\n",
      "Epoch 137/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3567\n",
      "\n",
      "Epoch 00137: loss did not improve from 2.36600\n",
      "Epoch 138/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3573\n",
      "\n",
      "Epoch 00138: loss did not improve from 2.36600\n",
      "Epoch 139/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3733\n",
      "\n",
      "Epoch 00139: loss did not improve from 2.36600\n",
      "Epoch 140/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3592\n",
      "\n",
      "Epoch 00140: loss did not improve from 2.36600\n",
      "Epoch 141/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3686\n",
      "\n",
      "Epoch 00141: loss did not improve from 2.36600\n",
      "Epoch 142/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3578\n",
      "\n",
      "Epoch 00142: loss did not improve from 2.36600\n",
      "Epoch 143/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3598\n",
      "\n",
      "Epoch 00143: loss did not improve from 2.36600\n",
      "Epoch 144/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3516\n",
      "\n",
      "Epoch 00144: loss did not improve from 2.36600\n",
      "Epoch 145/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3738\n",
      "\n",
      "Epoch 00145: loss did not improve from 2.36600\n",
      "Epoch 146/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3331\n",
      "\n",
      "Epoch 00146: loss did not improve from 2.36600\n",
      "Epoch 147/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3737\n",
      "\n",
      "Epoch 00147: loss improved from 2.36600 to 2.36594, saving model to nextword-mix1.h5\n",
      "Epoch 148/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3615\n",
      "\n",
      "Epoch 00148: loss did not improve from 2.36594\n",
      "Epoch 149/150\n",
      "600/600 [==============================] - 11s 18ms/step - loss: 2.3753\n",
      "\n",
      "Epoch 00149: loss did not improve from 2.36594\n",
      "Epoch 150/150\n",
      "600/600 [==============================] - 10s 17ms/step - loss: 2.3597\n",
      "\n",
      "Epoch 00150: loss did not improve from 2.36594\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1483609a3a90>"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X, y, epochs=150, batch_size=64, callbacks=[checkpoint, reduce, tensorboard_Visualization])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import load_model\n",
    "model = load_model('nextword-mix1.h5')\n",
    "tokenizer = pickle.load(open('tokenizer-mix1.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Predict_Next_Words(model, tokenizer, text):\n",
    "    \"\"\"\n",
    "        In this function we are using the tokenizer and models trained\n",
    "        and we are creating the sequence of the text entered and then\n",
    "        using our model to predict and return the the predicted word.\n",
    "    \n",
    "    \"\"\"\n",
    "    for i in range(3):\n",
    "        sequence = tokenizer.texts_to_sequences([text])[0]\n",
    "        sequence = np.array(sequence)\n",
    "        \n",
    "        preds = model.predict_classes(sequence)\n",
    "        print(preds)\n",
    "        predicted_word = \"\"\n",
    "        \n",
    "        for key, value in tokenizer.word_index.items():\n",
    "            if value == preds:\n",
    "                predicted_word = key\n",
    "                break\n",
    "        \n",
    "        print(predicted_word)\n",
    "        return predicted_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[56]\n",
      "math\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'math'"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Predict_Next_Words(model,tokenizer,\"import\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "TensorflowGPU",
   "language": "python",
   "name": "tf_gpu"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}