HW4/Solutions/LSTM_model_for_sentiment_analysis_task.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "LSTM_model_for_sentiment_analysis_task.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Lh0gPnUwOE7J",
        "outputId": "4c8bf95e-82e6-4465-8b36-c6d1f56506e4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=18l4Ca8KBkCo0-Eelt6dxAo6ve0lGEGuJ\n",
            "To: /content/dataset_annotated_sentiment.json\n",
            "\r  0% 0.00/1.74M [00:00<?, ?B/s]\r100% 1.74M/1.74M [00:00<00:00, 161MB/s]\n"
          ]
        }
      ],
      "source": [
        "'''\n",
        "download dataset\n",
        "'''\n",
        "\n",
        "# https://drive.google.com/file/d/18l4Ca8KBkCo0-Eelt6dxAo6ve0lGEGuJ/view?usp=sharing\n",
        "!gdown 18l4Ca8KBkCo0-Eelt6dxAo6ve0lGEGuJ"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "load train, eval, and test dataset to json format\n",
        "'''\n",
        "\n",
        "import json\n",
        "from pathlib import Path\n",
        "dataset = json.loads(Path('dataset_annotated_sentiment.json').read_text())\n",
        "train = dataset['train']\n",
        "eval = dataset['eval']\n",
        "test = dataset['test']"
      ],
      "metadata": {
        "id": "NSTONWkKPH0v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install hazm"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VlcFelAIPQ5r",
        "outputId": "26cfe822-243b-4b15-92ad-6b607b7e0a48"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: hazm in /usr/local/lib/python3.7/dist-packages (0.7.0)\n",
            "Requirement already satisfied: nltk==3.3 in /usr/local/lib/python3.7/dist-packages (from hazm) (3.3)\n",
            "Requirement already satisfied: libwapiti>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from hazm) (0.2.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from nltk==3.3->hazm) (1.15.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from hazm import Normalizer\n",
        "from hazm import Lemmatizer\n",
        "\n",
        "normalizer = Normalizer()\n",
        "lemmatizer = Lemmatizer()"
      ],
      "metadata": {
        "id": "tjL5xaSwPSuu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "normalize and lemmatize train & eval dataset and convert to list format, and also\n",
        "convert labels to numerical format\n",
        "'''\n",
        "\n",
        "augmented_train = []\n",
        "# this is correct. But it shows bad because of RTL\n",
        "labels = {'خنثی': 0, 'مثبت': 1, 'منفی': -1}\n",
        "for item in train:\n",
        "    for label in item['annotations']:\n",
        "        augmented_train.append((lemmatizer.lemmatize(normalizer.normalize(item['text'])),labels[label]))\n",
        "for item in eval:\n",
        "    for label in item['annotations']:\n",
        "        augmented_train.append((lemmatizer.lemmatize(normalizer.normalize(item['text'])),labels[label]))"
      ],
      "metadata": {
        "id": "8Oxz_ISRPWo1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "normalize and lemmatize test dataset and convert to list format, and also\n",
        "convert labels to numerical format\n",
        "'''\n",
        "\n",
        "augmented_test = []\n",
        "# this is correct. But it shows bad because of RTL\n",
        "labels = {'خنثی': 0, 'مثبت': 1, 'منفی': -1}\n",
        "for item in test:\n",
        "    for label in item['annotations']:\n",
        "        augmented_test.append((lemmatizer.lemmatize(normalizer.normalize(item['text'])),labels[label]))"
      ],
      "metadata": {
        "id": "wRSHfknCPY8s"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "convert train and test dataset to dataFrame format\n",
        "'''\n",
        "\n",
        "import pandas as pd\n",
        "augmented_train = pd.DataFrame(augmented_train)\n",
        "augmented_test = pd.DataFrame(augmented_test)"
      ],
      "metadata": {
        "id": "BQ1ZzRCuRVOA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "define test preprocessing function with regex library\n",
        "first, remove all english and numbers and special characters\n",
        "secondly, remove all punctuations\n",
        "finally, remove all urls\n",
        "'''\n",
        "\n",
        "import re\n",
        "def remove_english_number(text):\n",
        "  text = re.sub(\"[a-zA-Z0-9\\u200c\\n\\t_»«]+\", \"\",text)\n",
        "  text = re.sub(\"[^\\w\\s]\",'',text)\n",
        "  text = re.sub('https://.*','',text)\n",
        "  return text"
      ],
      "metadata": {
        "id": "cFhRASvef5Zp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "apply text preprocessing function to data\n",
        "'''\n",
        "\n",
        "for i in range(len(augmented_train[0])):\n",
        "  augmented_train[0][i] = remove_english_number(augmented_train[0][i])\n",
        "for i in range(len(augmented_test[0])):\n",
        "  augmented_test[0][i] = remove_english_number(augmented_test[0][i])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cEKYqE6IiMjr",
        "outputId": "0e8f62fe-0295-4827-cb4f-d4d699f5afc5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  after removing the cwd from sys.path.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "create vocabluary with corresponding word indices\n",
        "'''\n",
        "\n",
        "from collections import Counter\n",
        "all_text = ' '.join(augmented_train[0])\n",
        "all_text = all_text + ' '.join(augmented_test[0])\n",
        "# create a list of words\n",
        "words = all_text.split()\n",
        "# Count all the words using Counter Method\n",
        "count_words = Counter(words)\n",
        "total_words = len(words)\n",
        "sorted_words = count_words.most_common(total_words)\n",
        "vocab_to_int = {w:i+1 for i, (w,c) in enumerate(sorted_words)}"
      ],
      "metadata": {
        "id": "ckrhhHwzjHFM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "convert all text's words to integer form with above function\n",
        "'''\n",
        "\n",
        "text_int_train = []\n",
        "for text in augmented_train[0]:\n",
        "    r = [vocab_to_int[w] for w in text.split()]\n",
        "    text_int_train.append(r)\n",
        "\n",
        "text_int_test = []\n",
        "for text in augmented_test[0]:\n",
        "    r = [vocab_to_int[w] for w in text.split()]\n",
        "    text_int_test.append(r)"
      ],
      "metadata": {
        "id": "xc5oMdyPk-zM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "plot statistical features for trainset data\n",
        "'''\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "train_title_len = [len(x) for x in text_int_train]\n",
        "pd.Series(train_title_len).hist()\n",
        "plt.show()\n",
        "pd.Series(train_title_len).describe()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 421
        },
        "id": "1awI-_LimTXW",
        "outputId": "d680ba9f-e441-4790-b1e6-0abedfa674e8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAR6ElEQVR4nO3df6xc5X3n8fdnIUlZiAqU9MoL1ppqvbuickvSK6BK/rg0KjFktaRSFIFQMCkr9w+QEsnSrmmlpdsoEistyTZSFtVVrBApG5dVEsUCtKzr5qrKHyRASmMMpdwmjrDlYLU4JE5W0Tr73T/mMTt1rn3nXt+f53m/pNGc85znnHm+nuEz5z5zZkhVIUnqwz9Z6wFIklaPoS9JHTH0Jakjhr4kdcTQl6SOXLzWAzifq666qrZs2TJR3x//+MdceumlKzugNTLk2sD6NrIh1wYbt77nnnvu76vqHfNtW9ehv2XLFp599tmJ+s7OzjIzM7OyA1ojQ64NrG8jG3JtsHHrS/K9c21bcHonyeYkX0vyYpLDST7a2v8wybEkz7fbbWP7PJBkLsnLSd431r69tc0l2X2hhUmSFmeSM/3TwK6q+laStwPPJTnQtn2qqv7LeOck1wF3AL8K/DPgz5P8y7b5M8BvA0eBZ5Lsr6oXl6MQSdLCFgz9qjoOHG/LP0ryEnD1eXa5HdhXVT8FvptkDrihbZurqu8AJNnX+hr6krRKFjWnn2QL8E7gG8C7gfuT3A08y+ivgZOM3hCeHtvtKP//TeLVs9pvnOcxdgI7AaamppidnZ1obKdOnZq470Yz5NrA+jayIdcGw6xv4tBPchnwJeBjVfXDJI8AHweq3T8M/O6FDqiq9gB7AKanp2vSD1E26gcukxhybWB9G9mQa4Nh1jdR6Cd5C6PA/0JVfRmgql4b2/6nwONt9RiweWz3a1ob52mXJK2CSa7eCfBZ4KWq+uRY+6axbr8DvNCW9wN3JHlbkmuBrcA3gWeArUmuTfJWRh/27l+eMiRJk5jkTP/dwIeBQ0meb22/D9yZ5HpG0ztHgN8DqKrDSR5j9AHtaeC+qvoZQJL7gaeAi4C9VXV4GWuRJC1gkqt3vg5knk1PnmefTwCfmKf9yfPtJ0laWev6G7kb1ZbdTyzr8XZtO809Ex7zyEPvX9bHljQs/uCaJHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSMLhn6SzUm+luTFJIeTfLS1X5nkQJJX2v0VrT1JPp1kLsm3k7xr7Fg7Wv9XkuxYubIkSfOZ5Ez/NLCrqq4DbgLuS3IdsBs4WFVbgYNtHeBWYGu77QQegdGbBPAgcCNwA/DgmTcKSdLqWDD0q+p4VX2rLf8IeAm4GrgdeLR1exT4QFu+Hfh8jTwNXJ5kE/A+4EBVvV5VJ4EDwPZlrUaSdF4XL6Zzki3AO4FvAFNVdbxt+j4w1ZavBl4d2+1oaztX+9mPsZPRXwhMTU0xOzs70dhOnTo1cd+Vtmvb6WU93tQlkx9zvfwbLMZ6eu5WwpDrG3JtMMz6Jg79JJcBXwI+VlU/TPLmtqqqJLUcA6qqPcAegOnp6ZqZmZlov9nZWSbtu9Lu2f3Esh5v17bTPHxosqfqyF0zy/rYq2E9PXcrYcj1Dbk2GGZ9E129k+QtjAL/C1X15db8Wpu2od2faO3HgM1ju1/T2s7VLklaJZNcvRPgs8BLVfXJsU37gTNX4OwAvjrWfne7iucm4I02DfQUcEuSK9oHuLe0NknSKplkzuDdwIeBQ0meb22/DzwEPJbkXuB7wIfatieB24A54CfARwCq6vUkHweeaf3+qKpeX5YqJEkTWTD0q+rrQM6x+b3z9C/gvnMcay+wdzEDlCQtH7+RK0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0JakjC4Z+kr1JTiR5YaztD5McS/J8u902tu2BJHNJXk7yvrH27a1tLsnu5S9FkrSQSc70Pwdsn6f9U1V1fbs9CZDkOuAO4FfbPv8tyUVJLgI+A9wKXAfc2fpKklbRxQt1qKq/TLJlwuPdDuyrqp8C300yB9zQts1V1XcAkuxrfV9c9IglSUu2YOifx/1J7gaeBXZV1UngauDpsT5HWxvAq2e13zjfQZPsBHYCTE1NMTs7O9FgTp06NXHflbZr2+llPd7UJZMfc738GyzGenruVsKQ6xtybTDM+pYa+o8AHweq3T8M/O5yDKiq9gB7AKanp2tmZmai/WZnZ5m070q7Z/cTy3q8XdtO8/ChyZ6qI3fNLOtjr4b19NythCHXN+TaYJj1LSn0q+q1M8tJ/hR4vK0eAzaPdb2mtXGedknSKlnSJZtJNo2t/g5w5sqe/cAdSd6W5FpgK/BN4Blga5Jrk7yV0Ye9+5c+bEnSUix4pp/ki8AMcFWSo8CDwEyS6xlN7xwBfg+gqg4neYzRB7Sngfuq6mftOPcDTwEXAXur6vCyVyNJOq9Jrt65c57mz56n/yeAT8zT/iTw5KJGJ0laVn4jV5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOrJg6CfZm+REkhfG2q5MciDJK+3+itaeJJ9OMpfk20neNbbPjtb/lSQ7VqYcSdL5THKm/zlg+1ltu4GDVbUVONjWAW4FtrbbTuARGL1JAA8CNwI3AA+eeaOQJK2eBUO/qv4SeP2s5tuBR9vyo8AHxto/XyNPA5cn2QS8DzhQVa9X1UngAD//RiJJWmFLndOfqqrjbfn7wFRbvhp4dazf0dZ2rnZJ0iq6+EIPUFWVpJZjMABJdjKaGmJqaorZ2dmJ9jt16tTEfVfarm2nl/V4U5dMfsz18m+wGOvpuVsJQ65vyLXBMOtbaui/lmRTVR1v0zcnWvsxYPNYv2ta2zFg5qz22fkOXFV7gD0A09PTNTMzM1+3nzM7O8ukfVfaPbufWNbj7dp2mocPTfZUHblrZlkfezWsp+duJQy5viHXBsOsb6nTO/uBM1fg7AC+OtZ+d7uK5ybgjTYN9BRwS5Ir2ge4t7Q2SdIqWvD0MckXGZ2lX5XkKKOrcB4CHktyL/A94EOt+5PAbcAc8BPgIwBV9XqSjwPPtH5/VFVnfzgsSVphC4Z+Vd15jk3vnadvAfed4zh7gb2LGp0kaVn5jVxJ6sgFX72j9WXLMn+IPKkjD71/TR5X0uJ4pi9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqyMUXsnOSI8CPgJ8Bp6tqOsmVwJ8BW4AjwIeq6mSSAH8M3Ab8BLinqr51IY+/kC27n1jJw0vShrMcZ/o3V9X1VTXd1ncDB6tqK3CwrQPcCmxtt53AI8vw2JKkRViJ6Z3bgUfb8qPAB8baP18jTwOXJ9m0Ao8vSTqHVNXSd06+C5wECviTqtqT5AdVdXnbHuBkVV2e5HHgoar6ett2EPgPVfXsWcfcyegvAaampn5j3759E43l1KlTXHbZZf+o7dCxN5Zc23oydQm89r/XehTnt+3qX1zyvvM9d0My5PqGXBts3Ppuvvnm58ZmX/6RC5rTB95TVceS/DJwIMnfjG+sqkqyqHeVqtoD7AGYnp6umZmZifabnZ3l7L73DGROf9e20zx86EKfqpV15K6ZJe8733M3JEOub8i1wTDru6Dpnao61u5PAF8BbgBeOzNt0+5PtO7HgM1ju1/T2iRJq2TJoZ/k0iRvP7MM3AK8AOwHdrRuO4CvtuX9wN0ZuQl4o6qOL3nkkqRFu5A5gyngK6Npey4G/ntV/c8kzwCPJbkX+B7wodb/SUaXa84xumTzIxfw2JKkJVhy6FfVd4Bfn6f9H4D3ztNewH1LfTxJ0oXzG7mS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SerIkv/H6NK4LbufWPK+u7ad5p4l7n/kofcv+XGlHnmmL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTf3tGGdiG/+XOh/N0fbUSe6UtSR1Y99JNsT/Jykrkku1f78SWpZ6s6vZPkIuAzwG8DR4FnkuyvqhdXcxzScph0aulCfjp6Pk4r6UKs9pz+DcBcVX0HIMk+4HbA0JcmtJafY5xtud/QzmUt3+jW6t97pWpOVa3Iged9sOSDwPaq+ndt/cPAjVV1/1ifncDOtvqvgJcnPPxVwN8v43DXkyHXBta3kQ25Nti49f3zqnrHfBvW3dU7VbUH2LPY/ZI8W1XTKzCkNTfk2sD6NrIh1wbDrG+1P8g9BmweW7+mtUmSVsFqh/4zwNYk1yZ5K3AHsH+VxyBJ3VrV6Z2qOp3kfuAp4CJgb1UdXqbDL3pKaAMZcm1gfRvZkGuDAda3qh/kSpLWlt/IlaSOGPqS1JENH/pD+FmHJHuTnEjywljblUkOJHml3V/R2pPk063ebyd519qNfGFJNif5WpIXkxxO8tHWPpT6fiHJN5P8davvP7X2a5N8o9XxZ+3CBZK8ra3Pte1b1nL8k0hyUZK/SvJ4Wx9SbUeSHEryfJJnW9sgXpvnsqFDf+xnHW4FrgPuTHLd2o5qST4HbD+rbTdwsKq2AgfbOoxq3dpuO4FHVmmMS3Ua2FVV1wE3Afe152go9f0U+K2q+nXgemB7kpuA/wx8qqr+BXASuLf1vxc42do/1fqtdx8FXhpbH1JtADdX1fVj1+MP5bU5v6rasDfgN4GnxtYfAB5Y63EtsZYtwAtj6y8Dm9ryJuDltvwnwJ3z9dsIN+CrjH57aXD1Af8U+BZwI6NvcV7c2t98nTK6cu032/LFrV/WeuznqekaRsH3W8DjQIZSWxvnEeCqs9oG99ocv23oM33gauDVsfWjrW0IpqrqeFv+PjDVljdsze3P/XcC32BA9bXpj+eBE8AB4O+AH1TV6dZlvIY362vb3wB+aXVHvCj/Ffj3wP9t67/EcGoDKOB/JXmu/QQMDOi1OZ919zMM+nlVVUk29LW1SS4DvgR8rKp+mOTNbRu9vqr6GXB9ksuBrwD/eo2HtCyS/BvgRFU9l2RmrcezQt5TVceS/DJwIMnfjG/c6K/N+Wz0M/0h/6zDa0k2AbT7E619w9Wc5C2MAv8LVfXl1jyY+s6oqh8AX2M05XF5kjMnVeM1vFlf2/6LwD+s8lAn9W7g3yY5AuxjNMXzxwyjNgCq6li7P8HoDfsGBvjaHLfRQ3/IP+uwH9jRlncwmgs/0353u5LgJuCNsT9F152MTuk/C7xUVZ8c2zSU+t7RzvBJcgmjzyteYhT+H2zdzq7vTN0fBP6i2gTxelNVD1TVNVW1hdF/W39RVXcxgNoAklya5O1nloFbgBcYyGvznNb6Q4ULvQG3AX/LaB71D9Z6PEus4YvAceD/MJonvJfRXOhB4BXgz4ErW98wumLp74BDwPRaj3+B2t7DaN7028Dz7XbbgOr7NeCvWn0vAP+xtf8K8E1gDvgfwNta+y+09bm2/VfWuoYJ65wBHh9Sba2Ov263w2fyYyivzXPd/BkGSerIRp/ekSQtgqEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOvL/AAzjAeU9jYTdAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count    4190.000000\n",
              "mean      106.225537\n",
              "std        49.382846\n",
              "min         9.000000\n",
              "25%        77.000000\n",
              "50%        98.000000\n",
              "75%       124.000000\n",
              "max       556.000000\n",
              "dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "plot statistical features for trainset data\n",
        "'''\n",
        "\n",
        "test_title_len = [len(x) for x in text_int_test]\n",
        "pd.Series(test_title_len).hist()\n",
        "plt.show()\n",
        "pd.Series(test_title_len).describe()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 421
        },
        "id": "jSM9i5IwmyQN",
        "outputId": "46620539-958c-4029-9463-91ccf266a90d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAQ1UlEQVR4nO3dfYxldX3H8fdXVpQwyIJMN5uFulhgGsJEHjaUxofMSGsRLEtbQzDE7qY0mybaaFzTrjVpbdKmULOaNjG120JcGx+YIoQNxFa63dH4ByiL4IKIrDCkbtbdqAs6lNgO/faPe8bcvfNw78zce+756fuVTObc3z13zie/e+5nzz33YSMzkSSV5xXDDiBJWh0LXJIKZYFLUqEscEkqlAUuSYVaV+fG1q9fnxdccEGdm+zqxRdf5PTTTx92jAWamMtMvWtiLjP1rmm5Dh48+IPMHF1wRWbW9nPRRRdl0xw4cGDYERbVxFxm6l0Tc5mpd03LBTyci3Sqp1AkqVAWuCQVygKXpEJZ4JJUKAtckgplgUtSoSxwSSqUBS5JhbLAJalQtX6UXmXYvOv+Rcd3js+xfYnr+mXm1usG+velnyc9FXhEzAA/AV4G5jJzS0ScDdwJbAZmgBsz88RgYkqSOq3kFMpkZl6amVuqy7uA/Zl5IbC/uixJqslazoFvBfZWy3uBG9YeR5LUq8ge/lPjiHgWOAEk8I+ZuScins/M9dX1AZyYv9xx2x3ADoDR0dErpqam+pl/zWZnZxkZGRl2jAWGmevQkRcWHd9wGhx7abDbHt905orW9/7rnZl617Rck5OTB9vOfvxMrwW+KTOPRMQvAQ8Afwzsay/siDiRmWct93fGxsbyqaeeWnn6AZqenmZiYmLYMRYYZq7lXsTcfWiwr3uv9EVM77/emal3TcsVEYsWeE+nUDLzSPX7OHAPcCVwLCI2Vn98I3C8f3ElSd10LfCIOD0izphfBt4GPA7sA7ZVq20D7h1USEnSQr08H94A3NM6zc064LOZ+W8R8XVgKiJuAZ4DbhxcTElSp64FnpnPAG9YZPyHwNWDCCVJ6s6P0ktSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCWeCSVCgLXJIKZYFLUqEscEkqlAUuSYWywCWpUBa4JBXKApekQlngklQoC1ySCmWBS1KhLHBJKpQFLkmFssAlqVAWuCQVygKXpEJZ4JJUKAtckgplgUtSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCWeCSVKieCzwiTomIb0TEfdXl8yPioYg4HBF3RsSpg4spSeq0kiPw9wFPtl2+Dfh4Zl4AnABu6WcwSdLyeirwiDgXuA745+pyAG8F7qpW2QvcMIiAkqTFRWZ2XyniLuBvgDOADwLbgQero28i4jzgi5l5ySK33QHsABgdHb1iamqqb+H7YXZ2lpGRkWHHWGCYuQ4deWHR8Q2nwbGXag7TRb8yjW86c+1/pE0T9ysz9a5puSYnJw9m5pbO8XXdbhgR7wCOZ+bBiJhY6YYzcw+wB2BsbCwnJlb8JwZqenqapmWC4ebavuv+Rcd3js+x+1DXXaZW/co0c/PE2sO0aeJ+ZabeNTVXp172/DcC10fEtcCrgdcAfwesj4h1mTkHnAscGVxMSVKnrufAM/NDmXluZm4GbgL+MzNvBg4A76xW2wbcO7CUkqQF1vI+8D8FPhARh4HXArf3J5IkqRcrOnmYmdPAdLX8DHBl/yNJknrhJzElqVAWuCQVygKXpEJZ4JJUKAtckgplgUtSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCNevLnaUh2bzEd6Cv1s7xuSW/V73TzK3X9XXb+sXhEbgkFcoCl6RCWeCSVCgLXJIKZYFLUqEscEkqlAUuSYWywCWpUBa4JBXKApekQlngklQoC1ySCmWBS1KhLHBJKpQFLkmFssAlqVAWuCQVygKXpEJZ4JJUKAtckgrVtcAj4tUR8bWIeCwinoiIv6zGz4+IhyLicETcGRGnDj6uJGleL0fgPwXemplvAC4FromIq4DbgI9n5gXACeCWwcWUJHXqWuDZMltdfGX1k8Bbgbuq8b3ADQNJKElaVGRm95UiTgEOAhcAnwA+CjxYHX0TEecBX8zMSxa57Q5gB8Do6OgVU1NT/UvfB7Ozs4yMjAw7xgLDzHXoyAuLjm84DY69VHOYLpqYCVaWa3zTmYMNU2nivt7ETNC8XJOTkwczc0vn+LpebpyZLwOXRsR64B7gV3vdcGbuAfYAjI2N5cTERK83rcX09DRNywTDzbV91/2Lju8cn2P3oZ52mdo0MROsLNfMzRODDVNp4r7exEzQ3FydVvQulMx8HjgA/DqwPiLm99BzgSN9ziZJWkYv70IZrY68iYjTgN8EnqRV5O+sVtsG3DuokJKkhXp5jrcR2FudB38FMJWZ90XEt4DPR8RfAd8Abh9gTklSh64FnpnfBC5bZPwZ4MpBhJIkdecnMSWpUBa4JBXKApekQlngklQoC1ySCmWBS1KhmvcZZP3M5iU+0i5J4BG4JBXLApekQlngklQoC1ySCmWBS1KhLHBJKpQFLkmFssAlqVAWuCQVygKXpEJZ4JJUKAtckgplgUtSoSxwSSqUBS5JhfL7wKUhq+t733eOz7G9bVszt15Xy3Y1OB6BS1KhLHBJKpQFLkmFssAlqVAWuCQVygKXpEJZ4JJUKAtckgrVtcAj4ryIOBAR34qIJyLifdX42RHxQEQ8Xf0+a/BxJUnzejkCnwN2ZubFwFXAeyLiYmAXsD8zLwT2V5clSTXpWuCZeTQzH6mWfwI8CWwCtgJ7q9X2AjcMKqQkaaEVnQOPiM3AZcBDwIbMPFpd9X1gQ1+TSZKWFZnZ24oRI8CXgb/OzLsj4vnMXN92/YnMXHAePCJ2ADsARkdHr5iamupP8j6ZnZ1lZGRk2DEWmJ2d5dkXXh52jJNsOA2OvTTsFCdrYiZoZq7OTOObzhxemEqTH39NyjU5OXkwM7d0jvf0bYQR8UrgC8BnMvPuavhYRGzMzKMRsRE4vthtM3MPsAdgbGwsJyYmVpN/YKanp2laJmjl2v3VF4cd4yQ7x+fYfahZX2DZxEzQzFydmWZunhhemEqTH39NzNWpl3ehBHA78GRmfqztqn3Atmp5G3Bv/+NJkpbSyyHCG4F3A4ci4tFq7M+AW4GpiLgFeA64cTARJUmL6VrgmflVIJa4+ur+xpEk9cpPYkpSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCWeCSVCgLXJIKZYFLUqEscEkqlAUuSYWywCWpUBa4JBXKApekQlngklQoC1ySCmWBS1KhLHBJKpQFLkmFssAlqVAWuCQVygKXpEJZ4JJUKAtckgplgUtSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCWeCSVCgLXJIK1bXAI+KOiDgeEY+3jZ0dEQ9ExNPV77MGG1OS1KmXI/BPAdd0jO0C9mfmhcD+6rIkqUZdCzwzvwL8qGN4K7C3Wt4L3NDnXJKkLiIzu68UsRm4LzMvqS4/n5nrq+UATsxfXuS2O4AdAKOjo1dMTU31J3mfzM7OMjIyMuwYC8zOzvLsCy8PO8ZJNpwGx14adoqTNTETNDNXZ6bxTWcOL0ylyY+/JuWanJw8mJlbOsfXrfUPZ2ZGxJL/CmTmHmAPwNjYWE5MTKx1k301PT1N0zJBK9fur7447Bgn2Tk+x+5Da95l+qqJmaCZuTozzdw8MbwwlSY//pqYq9Nq34VyLCI2AlS/j/cvkiSpF6st8H3Atmp5G3Bvf+JIknrV9TleRHwOmADOiYjvAX8B3ApMRcQtwHPAjYMMKUn9sHnX/T2tt3N8ju09rtuLmVuv69vfate1wDPzXUtcdXWfs0iSVsBPYkpSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCWeCSVCgLXJIKZYFLUqEscEkqVLO+73IZvX6HwUp1+86DQX2HgSStlUfgklQoC1ySCmWBS1KhijkHPiyDOvfezc7xObx7JC3HI3BJKpQFLkmFssAlqVAWuCQVygKXpEJZ4JJUKAtckgplgUtSoSxwSSqUBS5JhbLAJalQFrgkFcoCl6RCWeCSVCgLXJIKZYFLUqEscEkq1JoKPCKuiYinIuJwROzqVyhJUnerLvCIOAX4BPB24GLgXRFxcb+CSZKWt5Yj8CuBw5n5TGb+D/B5YGt/YkmSuonMXN0NI94JXJOZf1hdfjfwa5n53o71dgA7qouXAI+vPu5AnAP8YNghFtHEXGbqXRNzmal3Tcv1uswc7Rwc+H97npl7gD0AEfFwZm4Z9DZXoomZoJm5zNS7JuYyU++amqvTWk6hHAHOa7t8bjUmSarBWgr868CFEXF+RJwK3ATs608sSVI3qz6FkplzEfFe4N+BU4A7MvOJLjfbs9rtDVATM0Ezc5mpd03MZabeNTXXSVb9IqYkabj8JKYkFcoCl6RC1VbgTfjYfUScFxEHIuJbEfFERLyvGv9IRByJiEern2trzjUTEYeqbT9cjZ0dEQ9ExNPV77NqzDPWNhePRsSPI+L9w5iniLgjIo5HxONtY4vOTbT8fbWPfTMiLq8x00cj4tvVdu+JiPXV+OaIeKltzj45iEzL5FryPouID1Vz9VRE/FaNme5syzMTEY9W47XM1TI9MNT9alUyc+A/tF7k/C7weuBU4DHg4jq23ZFjI3B5tXwG8B1aXwPwEeCDdedpyzUDnNMx9rfArmp5F3DbkLKdAnwfeN0w5gl4C3A58Hi3uQGuBb4IBHAV8FCNmd4GrKuWb2vLtLl9vSHM1aL3WbXfPwa8Cji/enyeUkemjut3A39e51wt0wND3a9W81PXEXgjPnafmUcz85Fq+SfAk8CmunP0aCuwt1reC9wwpBxXA9/NzOeGsfHM/Arwo47hpeZmK/DpbHkQWB8RG+vIlJlfysy56uKDtD4XUasl5mopW4HPZ+ZPM/NZ4DCtx2ltmSIigBuBz/V7u10yLdUDQ92vVqOuAt8E/Ffb5e8x5OKMiM3AZcBD1dB7q6dHd9R5uqKSwJci4mC0vnoAYENmHq2Wvw9sqDnTvJs4+QE2zHmat9TcNGU/+wNaR2zzzo+Ib0TElyPizUPIs9h91oS5ejNwLDOfbhurda46eqDp+9UCv5AvYkbECPAF4P2Z+WPgH4BfAS4FjtJ6WlenN2Xm5bS+2fE9EfGW9iuz9Tyu9vd7RusDWtcD/1oNDXueFhjW3CwlIj4MzAGfqYaOAr+cmZcBHwA+GxGvqTFS4+6zNu/i5IODWudqkR74mabtV0upq8Ab87H7iHglrTvtM5l5N0BmHsvMlzPz/4B/YgBPJZeTmUeq38eBe6rtH5t/mlb9Pl5npsrbgUcy81iVb6jz1GapuRnqfhYR24F3ADdXBUB1iuKH1fJBWueaL6or0zL32bDnah3wu8CdbVlrm6vFeoCG7lfLqavAG/Gx++qc2+3Ak5n5sbbx9vNZv0ON35gYEadHxBnzy7ReDHuc1vxsq1bbBtxbV6Y2Jx0hDXOeOiw1N/uA36/eNXAV8ELbU+KBiohrgD8Brs/M/24bH43Wd+cTEa8HLgSeqSNTtc2l7rN9wE0R8aqIOL/K9bW6cgG/AXw7M783P1DXXC3VAzRwv+qqrldLab2S+x1a/6p+eBiv2AJvovW06JvAo9XPtcC/AIeq8X3AxhozvZ7WuwEeA56YnxvgtcB+4GngP4Cza56r04EfAme2jdU+T7T+ATkK/C+tc4+3LDU3tN4l8IlqHzsEbKkx02Fa50nn96tPVuv+XnW/Pgo8Avx2zXO15H0GfLiaq6eAt9eVqRr/FPBHHevWMlfL9MBQ96vV/PhRekkq1C/ki5iS9PPAApekQlngklQoC1ySCmWBS1KhLHBJKpQFLkmF+n+k822GL/cvdwAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count    197.000000\n",
              "mean     101.091371\n",
              "std       38.807765\n",
              "min       10.000000\n",
              "25%       77.000000\n",
              "50%       99.000000\n",
              "75%      126.000000\n",
              "max      214.000000\n",
              "dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "remove very small or long texts in training dataset\n",
        "'''\n",
        "\n",
        "text_int_train = [ text_int_train[i] for i, l in enumerate(train_title_len) if l>0 ]\n",
        "encoded_labels_train = [ augmented_train[1][i] for i, l in enumerate(train_title_len) if l> 0 ]"
      ],
      "metadata": {
        "id": "dc9ovyGKm22K"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "remove very small or long texts in testing dataset\n",
        "'''\n",
        "\n",
        "text_int_test = [ text_int_test[i] for i, l in enumerate(test_title_len) if l>0 ]\n",
        "encoded_labels_test = [ augmented_test[1][i] for i, l in enumerate(test_title_len) if l> 0 ]"
      ],
      "metadata": {
        "id": "RVyvdFQGm2LA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "declare padding or cropping function for convert texts to fixed-size one\n",
        "'''\n",
        "\n",
        "import numpy as np\n",
        "def pad_features(reviews_int, seq_length):\n",
        "    ''' Return features of review_ints, where each review is padded with 0's or truncated to the input seq_length.\n",
        "    '''\n",
        "    features = np.zeros((len(reviews_int), seq_length), dtype = int)\n",
        "    for i, review in enumerate(reviews_int):\n",
        "        review_len = len(review)\n",
        "        if review_len <= seq_length:\n",
        "            zeroes = list(np.zeros(seq_length-review_len))\n",
        "            new = np.append(zeroes, review)\n",
        "        elif review_len > seq_length:\n",
        "            new = review[0:seq_length]\n",
        "        \n",
        "        features[i,:] = np.array(new)\n",
        "    \n",
        "    return features"
      ],
      "metadata": {
        "id": "wEJfCq40n4s3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "apply padding and cropping function to data\n",
        "'''\n",
        "\n",
        "text_int_train = pad_features(text_int_train,100)\n",
        "text_int_test = pad_features(text_int_test,100)"
      ],
      "metadata": {
        "id": "Y815zHSxsXf2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "convert python lists to numpy array for all news titles\n",
        "'''\n",
        "\n",
        "text_int_train = [np.array(text_int_train[i]) for i in range(len(text_int_train))]\n",
        "text_int_test = [np.array(text_int_test[i]) for i in range(len(text_int_test))]"
      ],
      "metadata": {
        "id": "KVPPqrcYrWx3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "convert train and test set to numpy arrays and create one-hot encoding\n",
        "format of labels\n",
        "'''\n",
        "\n",
        "train_x = np.array(text_int_train)\n",
        "train_y = np.array(encoded_labels_train)\n",
        "train_y = pd.get_dummies(encoded_labels_train).values\n",
        "\n",
        "test_x = np.array(text_int_test)\n",
        "test_y = np.array(encoded_labels_test)\n",
        "test_y = pd.get_dummies(encoded_labels_test).values"
      ],
      "metadata": {
        "id": "XTP8WNEzoIMc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "define keras model for training\n",
        "*convert all word vectors to fixed format: 400\n",
        "our model is embedding+dropout+LSTM+Dense+softmax\n",
        "we use categorical cross entropy loss function\n",
        "'''\n",
        "\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Embedding, LSTM, SpatialDropout1D\n",
        "from sklearn.model_selection import train_test_split\n",
        "from keras.utils.np_utils import to_categorical\n",
        "from keras.callbacks import EarlyStopping\n",
        "from keras.layers import Dropout\n",
        "\n",
        "vocab_size = len(vocab_to_int)+1 # +1 for the 0 padding\n",
        "embedding_dim = 400\n",
        "\n",
        "model = Sequential()\n",
        "model.add(Embedding(vocab_size, embedding_dim, input_length=100))\n",
        "model.add(SpatialDropout1D(0.2))\n",
        "model.add(LSTM(embedding_dim, dropout=0.2, recurrent_dropout=0.2))\n",
        "model.add(Dense(3, activation='softmax'))\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "print(model.summary())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4D7W7fxeknJk",
        "outputId": "c2112b55-eaf4-4fc7-db33-33ac36d136f6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding (Embedding)       (None, 100, 400)          6818000   \n",
            "                                                                 \n",
            " spatial_dropout1d (SpatialD  (None, 100, 400)         0         \n",
            " ropout1D)                                                       \n",
            "                                                                 \n",
            " lstm (LSTM)                 (None, 400)               1281600   \n",
            "                                                                 \n",
            " dense (Dense)               (None, 3)                 1203      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 8,100,803\n",
            "Trainable params: 8,100,803\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "we train the model in 5 epochs with 64 batch size\n",
        "\n",
        "we acquire about  76% accuracy for training phase and 52% accuracy for validation\n",
        "'''\n",
        "\n",
        "epochs = 5\n",
        "batch_size = 64\n",
        "\n",
        "history = model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size,validation_split=0.1,callbacks=[EarlyStopping(monitor='val_loss', patience=3, min_delta=0.0001)])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ScK7tqlklg2P",
        "outputId": "f08b6803-79e3-4958-f947-773278fc264c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/5\n",
            "59/59 [==============================] - 185s 3s/step - loss: 0.9998 - accuracy: 0.5243 - val_loss: 1.0345 - val_accuracy: 0.4964\n",
            "Epoch 2/5\n",
            "59/59 [==============================] - 168s 3s/step - loss: 0.7167 - accuracy: 0.7003 - val_loss: 1.0607 - val_accuracy: 0.5322\n",
            "Epoch 3/5\n",
            "59/59 [==============================] - 168s 3s/step - loss: 0.5555 - accuracy: 0.7552 - val_loss: 1.1059 - val_accuracy: 0.5251\n",
            "Epoch 4/5\n",
            "59/59 [==============================] - 167s 3s/step - loss: 0.5014 - accuracy: 0.7616 - val_loss: 1.1267 - val_accuracy: 0.5203\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "'''\n",
        "test trained model on testing dataset, we acquire about 55% accuracy on testing data \n",
        "'''\n",
        "\n",
        "accr = model.evaluate(test_x,test_y)\n",
        "print('Test set\\n  Loss: {:0.3f}\\n  Accuracy: {:0.3f}'.format(accr[0],accr[1]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7kRStQx1n2Dw",
        "outputId": "65de1915-b9b8-41fb-a5d9-2294fceaf019"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "7/7 [==============================] - 3s 384ms/step - loss: 1.0466 - accuracy: 0.5533\n",
            "Test set\n",
            "  Loss: 1.047\n",
            "  Accuracy: 0.553\n"
          ]
        }
      ]
    }
  ]
}