RePlutarch_TFembPub.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "RePlutarch_TFembPub.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.4"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oxKYH6wCQ9Hf",
        "colab_type": "text"
      },
      "source": [
        "<table class=\"tfo-notebook-buttons\" align=\"left\">\n",
        "<td>\n",
        "<a target=\"_blank\"  href=\"https://colab.research.google.com/github/mlai-demo/TextExplore/blob/master/RePlutarch_TFembPub.ipynb\">\n",
        "    <img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
        "</td><td>\n",
        "<a target=\"_blank\"  href=\"https://github.com/mlai-demo/TextExplore/blob/master/RePlutarch_TFembPub.ipynb\"><img width=32px src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a></td></table>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "PcIZDgAuNh7F",
        "colab": {}
      },
      "source": [
        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
        "\n",
        "# Install TensorFlow - works on Colab only\n",
        "try:\n",
        "  # %tensorflow_version only exists in Colab.\n",
        "  %tensorflow_version 2.x\n",
        "except Exception:\n",
        "  pass\n",
        "\n",
        "import tensorflow as tf"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "o_hemWWxNh9E",
        "outputId": "b315abea-b2fd-415a-fc88-3334c30db5bb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(tf.__version__)"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.0.0-rc0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "BJGnfegTNh9e",
        "colab": {}
      },
      "source": [
        "import os\n",
        "fpath = os.getcwd(); fpath"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "JkKkAhJaNh-s",
        "colab": {}
      },
      "source": [
        "# if using Google Colab\n",
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()\n",
        "\n",
        "for fn in uploaded.keys():\n",
        "    print('User uploaded file \"{name}\" with length {length} bytes'.format(\n",
        "      name=fn, length=len(uploaded[fn])))\n",
        "  \n",
        "# Click Files tab - the updloaded file(s) will be there"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "P5NS5q_SNh_1",
        "colab": {}
      },
      "source": [
        "import re\n",
        "\n",
        "corpus = open(fpath + '/Plutarch.txt',  'rb').read().lower().decode(encoding='utf-8')\n",
        "corpus = re.sub('\\n', ' ', corpus) #remove new line\n",
        "corpus = re.sub('\\r', ' ', corpus) #remove \"return\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "3c5UXe-nCfvS",
        "colab": {}
      },
      "source": [
        "import nltk\n",
        "from nltk.tokenize import sent_tokenize\n",
        "nltk.download('punkt') #need in Colab upon resetting the runtime\n",
        " \n",
        "# tokenize at sentence level\n",
        "sentences = nltk.sent_tokenize(corpus)\n",
        "#print(\"\\n---\\n\".join(sentences))\n",
        "print(\"The number of sentences is {}\".format(len(sentences)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "hOLDD9YoNh__",
        "colab": {}
      },
      "source": [
        "from nltk.tokenize import word_tokenize\n",
        "\n",
        "word_count = lambda sentence: len(word_tokenize(sentence))\n",
        "#print(min(sentences, key=word_count)) \n",
        "#print('\\n')\n",
        "#print(max(sentences, key=word_count)) \n",
        "longest_sentence = max(sentences, key=word_count)\n",
        "length_longest_sentence = len(word_tokenize(longest_sentence))\n",
        "print(\"The longest sentence has {} words\".format(length_longest_sentence))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "a12Qzu6VNiAH",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "tokenizer = Tokenizer()\n",
        "tokenizer.fit_on_texts(sentences)\n",
        "sent_numeric = tokenizer.texts_to_sequences(sentences)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "XnObx0kTHK9P",
        "colab": {}
      },
      "source": [
        "len(tokenizer.word_index.items())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "dPTXdBmmKdQe",
        "colab": {}
      },
      "source": [
        "word_index = {k:v for k,v in tokenizer.word_index.items()}\n",
        "word_index[\"<PAD>\"] = 0\n",
        "\n",
        "vocab_size = len(word_index)\n",
        "vocab_size"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "hus87becUVy3",
        "colab": {}
      },
      "source": [
        "for word in ['considering', 'therefore', 'great', 'oppose']:\n",
        "    print('{}: {}'.format(word, word_index[word]))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "y5C0OsHaUn4q",
        "colab": {}
      },
      "source": [
        "sent_numeric[2:4]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "sPH5YlhTUoIc",
        "colab": {}
      },
      "source": [
        "sentences[2:4]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "OeIR9L1OQSca",
        "colab": {}
      },
      "source": [
        "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n",
        "\n",
        "def decode_data(text):\n",
        "    return ' '.join([reverse_word_index.get(i, '?') for i in text])\n",
        "  \n",
        "#print(reverse_word_index)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "kjFQR80p-GuT",
        "colab": {}
      },
      "source": [
        "decode_data(sent_numeric[3])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "S5yC-2YiKmn_",
        "colab": {}
      },
      "source": [
        "sent_numeric[3]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "sfQeOGw_-Guc",
        "colab": {}
      },
      "source": [
        "maxLen = length_longest_sentence\n",
        "data = tf.keras.preprocessing.sequence.pad_sequences(sent_numeric,\n",
        "                                                        value=word_index[\"<PAD>\"],\n",
        "                                                        padding='post',\n",
        "                                                        maxlen=maxLen)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "kiKhvw28-Guk",
        "colab": {}
      },
      "source": [
        "data[0]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "t4Bfi5wg-Gun",
        "colab": {}
      },
      "source": [
        "decode_data(data[0])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EAC8TaROGI8P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# embedding layer by itself\n",
        "\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Embedding\n",
        "\n",
        "embedding_dim = 100\n",
        "\n",
        "model_justembed = Sequential()\n",
        "model_justembed.add(Embedding(vocab_size, embedding_dim, input_length=maxLen))\n",
        "\n",
        "model_justembed.compile('adam', 'mse')\n",
        "model_justembed.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dSQJamyxGI8T",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "output_array = model_justembed.predict(data)\n",
        "#output_array"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "GJyKJf-bWhX2",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras.layers import Dense"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "JoLyHsHU-Gus",
        "colab": {}
      },
      "source": [
        "embedding_dim=100\n",
        "\n",
        "model = tf.keras.Sequential([\n",
        "  layers.Embedding(vocab_size, embedding_dim, input_length=maxLen, mask_zero=True),\n",
        "  layers.GlobalAveragePooling1D(),\n",
        "  #layers.Dense(100, activation='relu'), #uncomment to compare the versions\n",
        "  layers.Dense(1, activation='sigmoid')\n",
        "])\n",
        "\n",
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "3ATFCAqBNiA0",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "adam = tf.keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, decay=0.0, amsgrad=False) \n",
        "model.compile(optimizer='adam',\n",
        "              loss='binary_crossentropy',\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "batch_size = 16989\n",
        "data_labels = np.zeros([batch_size, 1])\n",
        "\n",
        "history = model.fit(\n",
        "    data,\n",
        "    data_labels,\n",
        "    epochs=200,\n",
        "    batch_size=batch_size,\n",
        "    verbose = 0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "DOZgpRKO-Gux",
        "colab": {}
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "history_dict = history.history\n",
        "\n",
        "loss = history_dict['loss']\n",
        "epochs = range(1, len(loss) + 1)\n",
        "\n",
        "plt.figure(figsize=(12,9))\n",
        "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
        "plt.title('Training loss')\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "OfdrSJiI-Guz",
        "colab": {}
      },
      "source": [
        "e = model.layers[0]\n",
        "weights = e.get_weights()[0]\n",
        "print(weights.shape)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "A5QCXG_B-Gu2",
        "colab": {}
      },
      "source": [
        "import io\n",
        "\n",
        "out_v = io.open('vecs.tsv', 'w', encoding='utf-8')\n",
        "out_m = io.open('meta.tsv', 'w', encoding='utf-8')\n",
        "for word_num in range(vocab_size): \n",
        "    word = reverse_word_index[word_num]\n",
        "    embeddings = weights[word_num]\n",
        "    out_m.write(word + \"\\n\")\n",
        "    out_v.write('\\t'.join([str(x) for x in embeddings]) + \"\\n\")\n",
        "out_v.close()\n",
        "out_m.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "spnu2WTyGI82",
        "colab_type": "text"
      },
      "source": [
        "Go to projector.tensorflow.org and upload the two files"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "IbBiOm-xNiBA",
        "colab": {}
      },
      "source": [
        "f = open('vectors.tsv' ,'w')\n",
        "f.write('{} {}\\n'.format(vocab_size-1, embedding_dim)) "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "H3HJ95slNiBH",
        "colab": {}
      },
      "source": [
        "vectors = model.get_weights()[0]\n",
        "for words, i in tokenizer.word_index.items():\n",
        "    str_vec = ' '.join(map(str, list(vectors[i, :])))\n",
        "    f.write('{} {}\\n'.format(words, str_vec))\n",
        "f.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "shtizRH6-Gu8",
        "colab": {}
      },
      "source": [
        "# if running in Colab, this will download files to the local machine (if double-click does not work)\n",
        "try:\n",
        "  from google.colab import files\n",
        "except ImportError:\n",
        "   pass\n",
        "else:\n",
        "  files.download('vecs.tsv')\n",
        "  files.download('meta.tsv')\n",
        "  files.download('vectors.tsv')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "_uW6PvjhNiBU",
        "colab": {}
      },
      "source": [
        "import gensim"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "gEEwoTeZNiBe",
        "colab": {}
      },
      "source": [
        "w2v = gensim.models.KeyedVectors.load_word2vec_format('./vectors.tsv', binary=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "q6uywFKRNiBt",
        "colab": {}
      },
      "source": [
        "w2v.most_similar('rome')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "qOuk0pgn-GvE",
        "colab": {}
      },
      "source": [
        "round(w2v.similarity('rome', 'caesar'),4)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Tv8dcTkdEJ-U",
        "colab": {}
      },
      "source": [
        "round(w2v.similarity('pompey', 'caesar'),4)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "81X0a0c8UuAq",
        "colab": {}
      },
      "source": [
        "embedding_dim = 100\n",
        "\n",
        "model2 = tf.keras.Sequential([\n",
        "    layers.Embedding(vocab_size, embedding_dim, mask_zero=True),\n",
        "    #layers.Bidirectional(layers.LSTM(64, return_sequences=True)), #another LSTM layer - uncomment to compare\n",
        "    layers.Bidirectional(layers.LSTM(64)),  \n",
        "    layers.Dense(64, activation='relu'),\n",
        "    layers.Dense(1, activation='sigmoid')\n",
        "])\n",
        "\n",
        "model2.summary()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "TKN3G3aLNiCo",
        "scrolled": false,
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "\n",
        "adam = tf.keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, decay=0.0, amsgrad=False) \n",
        "model2.compile(optimizer='adam',\n",
        "              loss='binary_crossentropy',\n",
        "              metrics=['accuracy'])\n",
        "\n",
        "batch_size = 16989\n",
        "data_labels = np.zeros([batch_size, 1])\n",
        "\n",
        "history = model2.fit(\n",
        "    data,\n",
        "    data_labels,\n",
        "    epochs=20,\n",
        "    verbose = 0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "mCAPKf53fJEK",
        "outputId": "2fae9c45-5392-4598-d0d5-bd851aed1100",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "e2 = model2.layers[0]\n",
        "weights2 = e2.get_weights()[0]\n",
        "print(weights2.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(20242, 100)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "41VQmJjrNiCy",
        "colab": {}
      },
      "source": [
        "import io\n",
        "\n",
        "out_v = io.open('vecs2.tsv', 'w', encoding='utf-8')\n",
        "out_m = io.open('meta2.tsv', 'w', encoding='utf-8')\n",
        "for word_num in range(vocab_size):\n",
        "    word = reverse_word_index[word_num]\n",
        "    embeddings = weights2[word_num]\n",
        "    out_m.write(word + \"\\n\")\n",
        "    out_v.write('\\t'.join([str(x) for x in embeddings]) + \"\\n\")\n",
        "out_v.close()\n",
        "out_m.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zxRxo3VCGI9s",
        "colab_type": "text"
      },
      "source": [
        "Go to projector.tensorflow.org and upload the two files"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Sn7jaLxVNiC1",
        "colab": {}
      },
      "source": [
        "f = open('vectors2.tsv' ,'w')\n",
        "f.write('{} {}\\n'.format(vocab_size-1, 100))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "xng7X-BGNiC3",
        "colab": {}
      },
      "source": [
        "vectors2 = model2.get_weights()[0]\n",
        "for words, i in tokenizer.word_index.items():\n",
        "    str_vec = ' '.join(map(str, list(vectors2[i, :])))\n",
        "    f.write('{} {}\\n'.format(words, str_vec))\n",
        "f.close()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "JELEufg2NiC9",
        "colab": {}
      },
      "source": [
        "w2v2 = gensim.models.KeyedVectors.load_word2vec_format('./vectors2.tsv', binary=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "wPsdrLFXNiDB",
        "colab": {}
      },
      "source": [
        "w2v2.most_similar('rome')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ne-vB8bGlh0I",
        "colab": {}
      },
      "source": [
        "w2v2.most_similar('caesar')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "r69Ny8pPj6GX",
        "colab": {}
      },
      "source": [
        "round(w2v2.similarity('pompey', 'caesar'),4)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "gCGpYepqj5rm",
        "colab": {}
      },
      "source": [
        "round(w2v2.similarity('rome', 'caesar'),4)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "00P51yYpNiEW",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}