Word2Vec_NLP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-GtqrFbLgWz0",
        "outputId": "bef54264-5fc2-4ccf-b202-165c94b03d45"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package abc to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/abc.zip.\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 1
        }
      ],
      "source": [
        "import nltk, string, gensim\n",
        "from nltk.corpus import abc\n",
        "nltk.download('abc')\n",
        "nltk.download('punkt')\n",
        "nltk.download('stopwords')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('omw-1.4')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "abc.sents()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JimQ_-hxhIU9",
        "outputId": "2973a9da-69bf-4b47-9912-6f99b4dc67a0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['PM', 'denies', 'knowledge', 'of', 'AWB', 'kickbacks', 'The', 'Prime', 'Minister', 'has', 'denied', 'he', 'knew', 'AWB', 'was', 'paying', 'kickbacks', 'to', 'Iraq', 'despite', 'writing', 'to', 'the', 'wheat', 'exporter', 'asking', 'to', 'be', 'kept', 'fully', 'informed', 'on', 'Iraq', 'wheat', 'sales', '.'], ['Letters', 'from', 'John', 'Howard', 'and', 'Deputy', 'Prime', 'Minister', 'Mark', 'Vaile', 'to', 'AWB', 'have', 'been', 'released', 'by', 'the', 'Cole', 'inquiry', 'into', 'the', 'oil', 'for', 'food', 'program', '.'], ...]"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model= gensim.models.Word2Vec(abc.sents())"
      ],
      "metadata": {
        "id": "5q7AquOEggEf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data=model.wv.most_similar('science')\n",
        "print(data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h9-xKe-ghHEo",
        "outputId": "27331d4c-3d86-4147-93df-7f6a9db7a23c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[('law', 0.9415050745010376), ('agriculture', 0.931178867816925), ('general', 0.927341103553772), ('policy', 0.9263863563537598), ('media', 0.9183502197265625), ('practice', 0.9176878929138184), ('discussion', 0.9153490662574768), ('reservoir', 0.9141187071800232), ('Crean', 0.9128280878067017), ('tight', 0.9121520519256592)]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "json_file ='intents.json'\n",
        "with open('intents.json','r') as f:\n",
        "    data = json.load(f)"
      ],
      "metadata": {
        "id": "KS_RK0gWhqC4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KraOZznBjKot",
        "outputId": "675e48dc-97bc-477d-f59b-8c6c2363b502"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'tag': 'welcome',\n",
              "  'patterns': ['Hi',\n",
              "   'How are you',\n",
              "   'Is any one to talk?',\n",
              "   'Hello',\n",
              "   'hi are you available'],\n",
              "  'responses': ['Hello, thanks for contacting us',\n",
              "   'Good to see you here',\n",
              "   ' Hi there, how may I assist you?']},\n",
              " {'tag': 'goodbye',\n",
              "  'patterns': ['Bye', 'See you later', 'Goodbye', 'I will come back soon'],\n",
              "  'responses': ['See you later, thanks for visiting',\n",
              "   'have a great day ahead',\n",
              "   'Wish you Come back again soon.']},\n",
              " {'tag': 'thankful',\n",
              "  'patterns': ['Thanks for helping me',\n",
              "   'Thank your guidance',\n",
              "   \"That's helpful and kind from you\"],\n",
              "  'responses': ['Happy to help!',\n",
              "   'Any time!',\n",
              "   'My pleasure',\n",
              "   'It is my duty to help you']},\n",
              " {'tag': 'hoursopening',\n",
              "  'patterns': ['What hours are you open?',\n",
              "   'Tell your opening time?',\n",
              "   'When are you open?',\n",
              "   'Just your timing please'],\n",
              "  'responses': [\"We're open every day 8am-7pm\",\n",
              "   'Our office hours are 8am-7pm every day',\n",
              "   'We open office at 8 am and close at 7 pm']},\n",
              " {'tag': 'payments',\n",
              "  'patterns': ['Can I pay using credit card?',\n",
              "   ' Can I pay using Mastercard?',\n",
              "   ' Can I pay using cash only?'],\n",
              "  'responses': ['We accept VISA, Mastercard and credit card',\n",
              "   'We accept credit card, debit cards and cash. Please don’t worry']}]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "df = pd.DataFrame(data)\n",
        "df['patterns'] = df['patterns'].apply(', '.join) "
      ],
      "metadata": {
        "id": "2-RtR_z1jPq2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.corpus import stopwords\n",
        "from textblob import Word\n",
        "stop = stopwords.words('english')\n",
        "df['patterns'] = df['patterns'].apply(\n",
        "    lambda x:' '.join(x.lower() for x in x.split()))\n",
        "df['patterns']= df['patterns'].apply(\n",
        "    lambda x: ' '.join(x for x in x.split() if x not in string.punctuation))\n",
        "df['patterns']= df['patterns'].str.replace('[^\\w\\s]','')\n",
        "df['patterns']= df['patterns'].apply(\n",
        "    lambda x: ' '.join(x for x in x.split() if  not x.isdigit()))\n",
        "df['patterns'] = df['patterns'].apply(\n",
        "    lambda x:' '.join(x for x in x.split() if not x in stop))\n",
        "df['patterns'] = df['patterns'].apply(\n",
        "    lambda x: \" \".join([Word(word).lemmatize() for word in x.split()]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wmsqbV_Tkf6f",
        "outputId": "211dfcda-17c1-40c8-ed2c-8a5f90d9d60e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-12-a10f260ff0b5>:6: FutureWarning: The default value of regex will change from True to False in a future version.\n",
            "  df['patterns']= df['patterns'].str.replace('[^\\w\\s]','')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from gensim.models import Word2Vec"
      ],
      "metadata": {
        "id": "9SGm9bZqkg0L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#taking the outer list\n",
        "bigger_list=[]\n",
        "for i in df['patterns']:\n",
        "    li = i.split()\n",
        "    bigger_list.append(li)"
      ],
      "metadata": {
        "id": "vb6PE-rqnZHs"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#custom data is fed to machine for further processing\n",
        "model = Word2Vec(bigger_list, min_count=1,size=300,workers=4)\n",
        "print(model)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sw7R6889ooRd",
        "outputId": "a0e36d14-d808-4538-9f73-ae34faaccb71"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:gensim.models.base_any2vec:under 10 jobs per worker: consider setting a smaller `batch_words' for smoother alpha decay\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Word2Vec(vocab=32, size=300, alpha=0.025)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.save(\"word2vec.model\")\n",
        "model.save(\"model.bin\")"
      ],
      "metadata": {
        "id": "k-kjffQPo88A"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "new_model = Word2Vec.load('model.bin')"
      ],
      "metadata": {
        "id": "g2ocJ-ZJpsSu"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "vocab = list(new_model.wv.vocab)     # new_model.wv.key_to_index\n",
        "vocab;"
      ],
      "metadata": {
        "id": "S6zpN3SoqtiM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "similar_words = new_model.wv.most_similar('thanks')\t\n",
        "print(similar_words)\t"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MK2pzpwlrB7N",
        "outputId": "1bc69969-4be1-4562-a056-cf5190253c4e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[('pay', 0.09113314002752304), ('hour', 0.09038735926151276), ('thats', 0.08777609467506409), ('back', 0.07375020533800125), ('mastercard', 0.06709612905979156), ('using', 0.04830966144800186), ('guidance', 0.047691710293293), ('please', 0.04701085388660431), ('soon', 0.042033106088638306), ('thank', 0.04159191623330116)]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "similar_words = new_model.wv.most_similar('available')\t\n",
        "print(similar_words[:3])\t"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QiUvIn_0sFQH",
        "outputId": "f76b9b3c-ea47-4979-9e56-c65608912aed"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[('please', 0.09661215543746948), ('mastercard', 0.09072067588567734), ('hello', 0.08574282377958298)]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "similar_words = new_model.wv.most_similar('pay')\t\n",
        "print(similar_words[:3])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NlDjgQ-hsoR-",
        "outputId": "3739dcd7-7313-44ad-9111-c11d1ee2ab9a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[('come', 0.12683022022247314), ('cash', 0.11225173622369766), ('thank', 0.09228657186031342)]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dissimlar_words = model.wv.doesnt_match('see you later, thanks for visiting'.split())\n",
        "print(dissimlar_words)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7PxsCdM5tMPD",
        "outputId": "7bc83119-68bd-4dd4-c56d-828c8d6b3947"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:gensim.models.keyedvectors:vectors for words {'visiting', 'you', 'for', 'later,'} are not present in the model, ignoring these words\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "see\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install ktrain flask"
      ],
      "metadata": {
        "id": "rXZSDUhnwDgr"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}