IR-A1-Notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports and declerations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\hassa\\\\Desktop\\\\Development\\\\Uni Projects\\\\Information Retrieval\\\\A1'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pickle\n",
    "import os\n",
    "import string\n",
    "import re\n",
    "import pprint\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "import numpy as np\n",
    "import boolean\n",
    "import pyparsing\n",
    "import copy\n",
    "from pyparsing import Word, alphas, oneOf, operatorPrecedence, opAssoc\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "\n",
    "\n",
    "\n",
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Raw Vocabulary Storage:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem import PorterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Break words like Veterans.Before, West.In amendment.Change\n",
    "\n",
    "def split_words(vocabl):\n",
    "    new_vocab = set()\n",
    "    for word in vocabl:\n",
    "        if re.search('^[a-zA-Z]+[.][a-zA-Z]+$',word) is not None:\n",
    "            print(re.search('^[a-zA-Z]+[.][a-zA-Z]+$',word))\n",
    "            w1, w2 = word.split('.')\n",
    "#             print(w1)\n",
    "# #             print(w2)\n",
    "            new_vocab.add(w1)\n",
    "            new_vocab.add(w2)\n",
    "        elif re.search('^[a-zA-Z]+[?][a-zA-Z]+$',word) is not None:\n",
    "# #             print(re.search('^[a-zA-Z]+[.][a-zA-Z]+$',word))\n",
    "            w1, w2 = word.split('?')\n",
    "# #             print(w1)\n",
    "# #             print(w2)\n",
    "            new_vocab.add(w1)\n",
    "            new_vocab.add(w2)\n",
    "        elif re.search('^[a-zA-Z]+[,][a-zA-Z]+$',word) is not None:\n",
    "# #             print(re.search('^[a-zA-Z]+[.][a-zA-Z]+$',word))\n",
    "            w1, w2 = word.split(',')\n",
    "# #             print(w1)\n",
    "# #             print(w2)\n",
    "            new_vocab.add(w1)\n",
    "            new_vocab.add(w2)\n",
    "        else:\n",
    "            new_vocab.add(word)\n",
    "    return new_vocab\n",
    "\n",
    "# Remove Punctuation\n",
    "def remove_punctuation(word):\n",
    "    return word.translate(word.maketrans('','',string.punctuation))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PostingList(object):\n",
    "    def __init__(self):\n",
    "        self.total_count = 0\n",
    "        self.token = ''\n",
    "        self.occurrance = {\n",
    "#             'doc_id':0 = 'positions' : [],\n",
    "#             \n",
    "        }\n",
    "         \n",
    "    def __repr__(self):\n",
    "        \n",
    "        return f'total_cnt : {self.total_count} docs : [{self.occurrance.keys()}]'\n",
    "    \n",
    "    def addOccurrance(self, doc_id, position):\n",
    "        self.total_count += 1\n",
    "#         print(position)\n",
    "        if doc_id not in self.occurrance.keys():\n",
    "            self.occurrance[doc_id] = []\n",
    "        self.occurrance[doc_id].append(position)\n",
    "#         self.occurrance[doc_id]['position'].append(pos)\n",
    "    \n",
    "class InvertedIndex(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.index = {}\n",
    "        self.docs = {}\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.index.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = set()\n",
    "doc_contents = []\n",
    "inverted_index = InvertedIndex()\n",
    "printable = set(string.printable) \n",
    "# Printable characters are\n",
    "# 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\n",
    "# !\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~ \\t\\n\\r\\x0b\\x0c\n",
    "\n",
    "\n",
    "ps = PorterStemmer()\n",
    "\n",
    "stop_words = set()\n",
    "with open('Stopword-List.txt', 'r') as stop_word_file:\n",
    "    lines = stop_word_file.readlines()\n",
    "    for line in lines:\n",
    "        stop_words.add(line.split('\\n')[0])\n",
    "    stop_words.remove('')\n",
    "\n",
    "    \n",
    "    \n",
    "for file_number in range(0, 56):\n",
    "    with open(f'data/Trump Speechs/speech_{file_number}.txt', 'r') as file1:\n",
    "        lines = file1.readlines()\n",
    "#         print(f'File Number : speech_{file_number}.txt' )\n",
    "#         print(lines[0])\n",
    "        position = {'doc':file_number,'row':0, 'col':0, 'token_no':0}\n",
    "        \n",
    "#         {\n",
    "#             'total_count' : 0,\n",
    "#             'postings' : {\n",
    "#                 'count':0,\n",
    "#                 'doc_id':0,\n",
    "#                 'positions':[]\n",
    "#             }\n",
    "#         }\n",
    "        \n",
    "        for line_no,line in enumerate(lines):\n",
    "            doc_set = set()\n",
    "            # split words at . , whitespace ? ! : ;\n",
    "            position['row'] = line_no \n",
    "            position['col'] = 0\n",
    "            for word in re.split('[.\\s,?!:;-]', line):\n",
    "                position['col'] += len(word) + 1\n",
    "                position['token_no'] += 1\n",
    "                # Case Folding\n",
    "                word = word.lower()\n",
    "                \n",
    "                # Filter non-ASCII characters\n",
    "                word = ''.join(filter(lambda x: x in printable, word))\n",
    "                \n",
    "                # Remove Punctuations\n",
    "                word = remove_punctuation(word)\n",
    "                \n",
    "                if re.match('\\d+[A-Za-z]+',word):\n",
    "                    word = re.split('\\d+',word)[1]\n",
    "                if re.match('[A-Za-z]+\\d+',word):\n",
    "                    word = re.split('\\d+',word)[0]\n",
    "                \n",
    "                if len(word) == 0 or len(word) == 1 or word == '' or word == ' ':\n",
    "                    continue\n",
    "                if word in stop_words:\n",
    "                    continue\n",
    "                \n",
    "                \n",
    "                \n",
    "                word = ps.stem(word)\n",
    "                    \n",
    "                vocab.add(word)\n",
    "                \n",
    "                doc_set.add(word)\n",
    "                \n",
    "                if word in inverted_index.index.keys():\n",
    "                    \n",
    "                    inverted_index.index[word].addOccurrance(file_number, copy.deepcopy(position)) \n",
    "                else:\n",
    "                    plist = PostingList()\n",
    "                    inverted_index.index[word] = plist\n",
    "                    inverted_index.index[word].addOccurrance(file_number, copy.deepcopy(position))\n",
    "                    \n",
    "                \n",
    "                \n",
    "        \n",
    "            \n",
    "        \n",
    "        inverted_index.docs[file_number] = doc_set\n",
    "        doc_contents.append(doc_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4850"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "{16: [{'doc': 16, 'row': 1, 'col': 7055, 'token_no': 1308},\n",
       "  {'doc': 16, 'row': 1, 'col': 7326, 'token_no': 1361}],\n",
       " 17: [{'doc': 17, 'row': 1, 'col': 3434, 'token_no': 649}],\n",
       " 18: [{'doc': 18, 'row': 1, 'col': 1619, 'token_no': 315}],\n",
       " 19: [{'doc': 19, 'row': 1, 'col': 26500, 'token_no': 5472},\n",
       "  {'doc': 19, 'row': 1, 'col': 26542, 'token_no': 5480},\n",
       "  {'doc': 19, 'row': 1, 'col': 26703, 'token_no': 5513},\n",
       "  {'doc': 19, 'row': 1, 'col': 27855, 'token_no': 5757}],\n",
       " 20: [{'doc': 20, 'row': 1, 'col': 8800, 'token_no': 1692}],\n",
       " 21: [{'doc': 21, 'row': 1, 'col': 3743, 'token_no': 695}],\n",
       " 24: [{'doc': 24, 'row': 1, 'col': 2185, 'token_no': 428}],\n",
       " 25: [{'doc': 25, 'row': 1, 'col': 4633, 'token_no': 900}],\n",
       " 27: [{'doc': 27, 'row': 1, 'col': 2989, 'token_no': 577}],\n",
       " 33: [{'doc': 33, 'row': 1, 'col': 2072, 'token_no': 401}],\n",
       " 34: [{'doc': 34, 'row': 1, 'col': 6415, 'token_no': 1206}],\n",
       " 35: [{'doc': 35, 'row': 1, 'col': 2319, 'token_no': 437}],\n",
       " 36: [{'doc': 36, 'row': 1, 'col': 5575, 'token_no': 1065}],\n",
       " 39: [{'doc': 39, 'row': 1, 'col': 9830, 'token_no': 1895}],\n",
       " 40: [{'doc': 40, 'row': 1, 'col': 9078, 'token_no': 1730}],\n",
       " 42: [{'doc': 42, 'row': 1, 'col': 838, 'token_no': 159}],\n",
       " 43: [{'doc': 43, 'row': 1, 'col': 2200, 'token_no': 415}],\n",
       " 45: [{'doc': 45, 'row': 1, 'col': 1722, 'token_no': 318}],\n",
       " 46: [{'doc': 46, 'row': 1, 'col': 1431, 'token_no': 282}],\n",
       " 49: [{'doc': 49, 'row': 1, 'col': 7474, 'token_no': 1418}],\n",
       " 50: [{'doc': 50, 'row': 1, 'col': 4371, 'token_no': 804},\n",
       "  {'doc': 50, 'row': 1, 'col': 7276, 'token_no': 1366}],\n",
       " 51: [{'doc': 51, 'row': 1, 'col': 2973, 'token_no': 572}],\n",
       " 53: [{'doc': 53, 'row': 1, 'col': 917, 'token_no': 172}],\n",
       " 54: [{'doc': 54, 'row': 1, 'col': 2616, 'token_no': 493}]}"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# print('Total Vocabulary Size ')\n",
    "# print(len(vocab))\n",
    "# print('Total Number of Documents ')\n",
    "# print(len(doc_contents))\n",
    "# print(doc_contents[17])\n",
    "\n",
    "len(inverted_index)\n",
    "inverted_index.index[ps.stem('hammer')].occurrance\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Boolean Model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1. 0. 0. ... 0. 0. 0.]\n",
      " [1. 1. 0. ... 1. 1. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 1. 0.]\n",
      " [1. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "# print(sorted(list(vocab)))\n",
    "# for index,doc in enumerate(doc_contents):\n",
    "#     print('Vocab size of doc' + str(index))\n",
    "#     print(len(doc))\n",
    "\n",
    "vocab_list = sorted(list(vocab))\n",
    "\n",
    "term_doc_matrix_np = np.zeros((len(vocab), len(doc_contents)))\n",
    "\n",
    "for word_index, word in enumerate(vocab_list):\n",
    "    word_row = []\n",
    "    for doc_index, doc in enumerate(doc_contents):\n",
    "        if word in doc:\n",
    "            term_doc_matrix_np[word_index, doc_index] = 1\n",
    "        else:\n",
    "            term_doc_matrix_np[word_index, doc_index] = 0\n",
    "            \n",
    "print(term_doc_matrix_np)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "with open('pickled/vocab.p', 'ab') as vocab_file:\n",
    "    pickle.dump(vocab, vocab_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('pickled/vocab.p', 'rb') as vocab_file:\n",
    "    vocabf = pickle.load(vocab_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter your query : running\n",
      "[1. 1. 1. 1. 1. 1. 1. 0. 1. 1. 1. 1. 1. 0. 0. 0. 1. 1. 1. 1. 1. 1. 1. 0.\n",
      " 1. 1. 1. 1. 1. 0. 1. 0. 1. 1. 1. 1. 1. 1. 0. 1. 1. 1. 0. 0. 1. 1. 1. 1.\n",
      " 0. 0. 1. 1. 1. 1. 0. 0.]\n",
      "[[ 0]\n",
      " [ 1]\n",
      " [ 2]\n",
      " [ 3]\n",
      " [ 4]\n",
      " [ 5]\n",
      " [ 6]\n",
      " [ 8]\n",
      " [ 9]\n",
      " [10]\n",
      " [11]\n",
      " [12]\n",
      " [16]\n",
      " [17]\n",
      " [18]\n",
      " [19]\n",
      " [20]\n",
      " [21]\n",
      " [22]\n",
      " [24]\n",
      " [25]\n",
      " [26]\n",
      " [27]\n",
      " [28]\n",
      " [30]\n",
      " [32]\n",
      " [33]\n",
      " [34]\n",
      " [35]\n",
      " [36]\n",
      " [37]\n",
      " [39]\n",
      " [40]\n",
      " [41]\n",
      " [44]\n",
      " [45]\n",
      " [46]\n",
      " [47]\n",
      " [50]\n",
      " [51]\n",
      " [52]\n",
      " [53]]\n"
     ]
    }
   ],
   "source": [
    "query = input('Enter your query : ')\n",
    "query = ps.stem(query)\n",
    "query_actions = []\n",
    "query_wanted = []\n",
    "if query in vocab:\n",
    "    term_index = vocab_list.index(query)\n",
    "    term_row = term_doc_matrix_np[term_index]\n",
    "    print(term_row)\n",
    "    doc_ids = np.argwhere(term_row == 1)\n",
    "    print(doc_ids)\n",
    "\n",
    "else:\n",
    "    print(f'{query} not present in vocabulary')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<re.Match object; span=(0, 9), match='000dollar'>\n",
      "['', 'dollar']\n"
     ]
    }
   ],
   "source": [
    "test = '000dollar'\n",
    "x = re.match('\\d+[A-Za-z]+',test)\n",
    "print(x)\n",
    "x = re.split('\\d+',test)\n",
    "print(x)\n",
    "# For matching queries like\n",
    "# not hammer or pakistan\n",
    "# (magnum or not hammer) or not (polish and pakistan)\n",
    "x = re.match('(not)?\\s*(\\w+|(\\((not)?\\s*(\\w+)\\s+(and|or)\\s+(not)?\\s*(\\w+)\\)))\\s+(or|and)\\s+(not)?\\s*(\\w+|(\\((not)?\\s*(\\w+)\\s+(and|or)\\s+(not)?\\s*(\\w+)\\)))')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "ans = {'0', '1', '10', '11', '12', '16', '17', '18', '19', '2', '20', '21', '22', '24', '25', '26', '27', '28', '3', '30', '32', '33', '34', '35', '36', '37', '39', '4', '40', '41', '44', '45', '46', '47', '5', '50', '51', '52', '53', '6', '8', '9'}\n",
    "ans2 =set([str(x[0]) for x in doc_ids])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "42\n",
      "{'22', '1', '53', '25', '37', '8', '24', '10', '0', '39', '50', '17', '35', '40', '21', '33', '27', '52', '3', '36', '20', '16', '26', '45', '9', '30', '2', '5', '12', '19', '4', '32', '28', '46', '18', '47', '41', '34', '11', '51', '6', '44'}\n",
      "set()\n"
     ]
    }
   ],
   "source": [
    "print(len(ans))\n",
    "print((ans2))\n",
    "print(ans.difference(ans2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# References :\n",
    "\n",
    "http://www.pyregex.com/\n",
    "http://cs231n.github.io/python-numpy-tutorial/\n",
    "\n",
    "https://www.online-utility.org/text/analyzer.jsp\n",
    "\n",
    "https://stackoverflow.com/questions/2118261/parse-boolean-arithmetic-including-parentheses-with-regex\n",
    "\n",
    "https://regex101.com/r/M8z3U4/1\n",
    "\n",
    "https://iq.opengenus.org/porter-stemmer/\n",
    "\n",
    "https://unnikked.ga/how-to-build-a-boolean-expression-evaluator-518e9e068a65\n",
    "\n",
    "https://ruslanspivak.com/lsbasi-part7/\n",
    "\n",
    "https://github.com/gintas/django-picklefield"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 1 1 1 0 1 0 1 0 1 0 0 1 0 0 1 1 1 0 1 0 0 0 0 1 0 1 0 1 1 0 1 0 0 0 0 0\n",
      " 1 0 1 1 1 1 0 0 0 0 0 0 0 0 1 0 1 1 0]\n",
      "[[ 0]\n",
      " [ 1]\n",
      " [ 2]\n",
      " [ 3]\n",
      " [ 5]\n",
      " [ 7]\n",
      " [ 9]\n",
      " [12]\n",
      " [15]\n",
      " [16]\n",
      " [17]\n",
      " [19]\n",
      " [24]\n",
      " [26]\n",
      " [28]\n",
      " [29]\n",
      " [31]\n",
      " [37]\n",
      " [39]\n",
      " [40]\n",
      " [41]\n",
      " [42]\n",
      " [51]\n",
      " [53]\n",
      " [54]]\n",
      "25\n",
      "{'9', '19', '37', '53', '5', '42', '15', '2', '1', '26', '51', '24', '0', '39', '29', '31', '17', '41', '40', '28', '16', '12', '54', '3', '7'}\n",
      "set()\n"
     ]
    }
   ],
   "source": [
    "query_actions = []\n",
    "query_wanted = []\n",
    "term_index = vocab_list.index(ps.stem('actions'))\n",
    "term_row_actions = term_doc_matrix_np[term_index]\n",
    "\n",
    "query_actions = np.argwhere(term_row_actions == 1)\n",
    "term_index = vocab_list.index(ps.stem('wanted'))\n",
    "term_row_wanted = term_doc_matrix_np[term_index]\n",
    "\n",
    "query_wanted = np.argwhere(term_row_wanted == 1)\n",
    "\n",
    "and_query = np.array([1 if x == 1 and y == 1 else 0 for x,y in zip(term_row_actions, term_row_wanted)])\n",
    "and_doc_ids = np.argwhere(and_query == 1)\n",
    "print(and_query)\n",
    "print(and_doc_ids)\n",
    "\n",
    "ans = {'37', '3', '19', '1', '9', '40', '51', '16', '15', '12', '31', '41', '39', '0', '53', '26', '29', '17', '24', '54', '7', '2', '5', '28', '42'}\n",
    "ans2 =set([str(x[0]) for x in and_doc_ids])\n",
    "\n",
    "print(len(ans))\n",
    "print((ans2))\n",
    "print(ans.difference(ans2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['box']\n",
      "Current indexed Word : box\n",
      "box\n",
      "STATE\n",
      "['box']\n",
      "Outt\n",
      "[0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0]\n",
      "deque([])\n",
      "{4, 9, 44, 45, 46, 47, 18, 50, 53, 54, 23, 25}\n"
     ]
    }
   ],
   "source": [
    "from collections import deque \n",
    "\n",
    "line = 'box'\n",
    "boperators = ['and', 'or']\n",
    "uoperators = ['not']\n",
    "  \n",
    "stack = deque() \n",
    "        \n",
    "def clean_word(word):\n",
    "    # Case Folding\n",
    "    word = word.lower()\n",
    "     # Filter non-ASCII characters\n",
    "    word = ''.join(filter(lambda x: x in printable, word))\n",
    "#     print(word)\n",
    "    # Remove Punctuations\n",
    "    if word != '(' and word != ')':\n",
    "        word = remove_punctuation(word)\n",
    "#     print(word)\n",
    "    if re.match('\\d+[A-Za-z]+',word):\n",
    "        word = re.split('\\d+',word)[1]\n",
    "    if re.match('[A-Za-z]+\\d+',word):\n",
    "        word = re.split('\\d+',word)[0]\n",
    "#     print(word)\n",
    "    word = ps.stem(word)\n",
    "#     print(word)\n",
    "    return word\n",
    "query = (re.split('[.\\s,?!:;-]', line))\n",
    "\n",
    "\n",
    "def evaluate_expression(index, query, stack, state):\n",
    "    \n",
    "    print(\"Current indexed Word : \" + str(query[index]))\n",
    "    result_query = []\n",
    "    \n",
    "    if query[index] == '(':\n",
    "        bracket_term = query[index] \n",
    "        result, new_index = evaluate_expression(index+1, query, stack, state)\n",
    "        if new_index == -1:\n",
    "            state = []\n",
    "        for x in range(index, new_index+1):\n",
    "            state.pop(x)\n",
    "            \n",
    "        print(\"Result\")\n",
    "        print(result)\n",
    "        index = new_index\n",
    "        stack.append({'state':True,'data':result, 'query': 'not' + not_of_term})\n",
    "        result_query = result\n",
    "        \n",
    "        print('new_index ' + str(new_index))\n",
    "        print(f'This should be a ) = {query[new_index]}')\n",
    "        \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    if query[index] in uoperators:\n",
    "        not_of_term = query[index] \n",
    "        result, new_index = evaluate_expression(index+1, query, stack, state)\n",
    "        if new_index == -1:\n",
    "            state = []\n",
    "        for x in range(0, new_index-index+1):\n",
    "            state.pop(x)\n",
    "            \n",
    "        print(\"Result\")\n",
    "        print(result)\n",
    "        index = new_index\n",
    "        not_result = [0 if int(x)==1 else 1 for x in result]\n",
    "        stack.append({'state':True,'data':not_result, 'query': 'not' + not_of_term})\n",
    "        result_query = not_result\n",
    "        \n",
    "    print(query[index])\n",
    "    if len(state) == 0:\n",
    "        return result_query, index\n",
    "    print('STATE')\n",
    "    print(state)\n",
    "    \n",
    "    \n",
    "    if (query[index] not in boperators) and (query[index] not in uoperators):\n",
    "        query[index] = ps.stem(query[index])\n",
    "        if query[index] not in vocab_list:\n",
    "            print(f'{query[index]} is not in vocabulary of index')\n",
    "            return [], -1\n",
    "        term_index = vocab_list.index(query[index])\n",
    "        term_row = term_doc_matrix_np[term_index]\n",
    "        stack.append({'state':True,'data':term_row,'query':query[index]})\n",
    "        result_query = term_row\n",
    "        index += 1\n",
    "        \n",
    "    if index >= len(query):\n",
    "        return stack.pop()['data'], -1\n",
    "    if len(state) == 0:\n",
    "        return result_query, index\n",
    "    \n",
    "    elif query[index] in boperators:\n",
    "#         query2 = clean_word(next_word)\n",
    "        \n",
    "#         next_word = query[index+1]\n",
    "#         print(next_word)\n",
    "#         if query2 not in vocab_list:\n",
    "#             print(f'{query2} is not in vocabulary of index')\n",
    "#             return [], -1\n",
    "        \n",
    "        \n",
    "#         term_index2 = vocab_list.index(query2)\n",
    "#         term_row2  = term_doc_matrix_np[term_index2]\n",
    "        \n",
    "        \n",
    "        term_row2, new_index = evaluate_expression(index+1, query, stack, state)\n",
    "        if new_index == -1:\n",
    "            state = []\n",
    "        for x in range(0, new_index-index+1):\n",
    "            state.pop(x)\n",
    "        \n",
    "        \n",
    "        query1 =  stack.pop()\n",
    "        term_row1 = []\n",
    "        \n",
    "        if query1['state'] == False:\n",
    "            term_index1 = vocab_list.index(query1['data'])\n",
    "            term_row1 = term_doc_matrix_np[term_index1]\n",
    "        \n",
    "        else:\n",
    "            term_row1 = query1['data']\n",
    "        \n",
    "        print(term_row1)\n",
    "        \n",
    "        print(term_row2)\n",
    "        result_query = []\n",
    "        if query[index] == 'and':\n",
    "            and_query = np.array([1 if int(x) == 1 and int(y) == 1 else 0 for x,y in zip(term_row1, term_row2)])\n",
    "            print(and_query)\n",
    "            and_doc_ids = np.argwhere(and_query == 1)\n",
    "            query_ans =  set([x[0] for x in and_doc_ids])\n",
    "            print(query_ans)\n",
    "            # stack.append({'state':True,'data':and_query,'query':query2})\n",
    "            result_query = and_query\n",
    "            \n",
    "        elif query[index] == 'or':\n",
    "            or_query = np.array([1 if int(x) == 1 or int(y) == 1 else 0 for x,y in zip(term_row1, term_row2)])\n",
    "            print('OR')\n",
    "            print(or_query)\n",
    "            or_doc_ids = np.argwhere(or_query == 1)\n",
    "            query_ans =  set([x[0] for x in or_doc_ids])\n",
    "            print(query_ans)\n",
    "            # stack.append({'state':True,'data':or_query,'query':query2})\n",
    "            result_query = or_query\n",
    "        index = new_index\n",
    "        \n",
    "    return result_query, index\n",
    "\n",
    "\n",
    "# for index, word in enumerate(query):  \n",
    "#     word = clean_word(word)\n",
    "#     print(word)\n",
    "\n",
    "query = [clean_word(word) for word in query ]\n",
    "print(query)\n",
    "ans, index = evaluate_expression(0, query, stack, query)\n",
    "\n",
    "print(\"Outt\")\n",
    "print(list(ans))\n",
    "\n",
    "print(stack)\n",
    "final_doc_ids = np.argwhere(np.array(ans) == 1)\n",
    "final_ans =  set([x[0] for x in final_doc_ids])\n",
    "print(final_ans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'47', '22', '13', '27', '7', '38', '25', '42', '15', '4', '33', '6', '8', '34', '30', '21', '43', '55', '44', '50', '10', '29', '31', '11', '49', '18', '46', '23', '52', '45', '32', '36', '14', '54', '35', '20', '48'}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'9'}"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ansq ={'18', '46', '4', '45', '50', '9', '47', '23', '54', '44', '25'}\n",
    "ansi = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 39, 40, 41, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 54, 55}\n",
    "ansq2 = set([str(x) for x in ansi])\n",
    "print(ansq2)\n",
    "ansq.difference(ansq2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' SPI - Simple Pascal Interpreter '"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\" SPI - Simple Pascal Interpreter \"\"\"\n",
    "import re\n",
    "boperators = ['and', 'or']\n",
    "uoperators = ['not']\n",
    "\n",
    "# Token types\n",
    "#\n",
    "# EOF (end-of-file) token is used to indicate that\n",
    "# there is no more input left for lexical analysis\n",
    "INTEGER, PLUS, MINUS, MUL, DIV, LPAREN, RPAREN, EOF, TERM, AND, OR, NOT = (\n",
    "    'INTEGER', 'PLUS', 'MINUS', 'MUL', 'DIV', '(', ')', 'EOF', 'TERM', 'AND','OR', 'NOT'\n",
    ")\n",
    "\n",
    "\n",
    "class Token(object):\n",
    "    def __init__(self, type, value):\n",
    "        self.type = type\n",
    "        self.value = value\n",
    "        self.inverse = False\n",
    "        self.row = []\n",
    "\n",
    "    def __str__(self):\n",
    "        \"\"\"String representation of the class instance.\n",
    "\n",
    "        Examples:\n",
    "            Token(TERM, Hello)\n",
    "            Token(AND, '&')\n",
    "            Token(NOT, '!')\n",
    "        \"\"\"\n",
    "        return 'Token({type}, {value})'.format(\n",
    "            type=self.type,\n",
    "            value=repr(self.value)\n",
    "        )\n",
    "\n",
    "    def __repr__(self):\n",
    "        return self.__str__()\n",
    "\n",
    "\n",
    "class Lexer(object):\n",
    "    def __init__(self, text):\n",
    "        # client string input, e.g. \"hello | world & (why | are | you)\"\n",
    "        self.text = text\n",
    "        # self.pos is an index into self.text\n",
    "        self.pos = 0\n",
    "        self.current_char = self.text[self.pos]\n",
    "        \n",
    "\n",
    "    def error(self):\n",
    "        raise Exception('Invalid character')\n",
    "\n",
    "    def advance(self):\n",
    "        \"\"\"Advance the `pos` pointer and set the `current_char` variable.\"\"\"\n",
    "        self.pos += 1\n",
    "        if self.pos > len(self.text) - 1:\n",
    "            self.current_char = None  # Indicates end of input\n",
    "        else:\n",
    "            self.current_char = self.text[self.pos]\n",
    "\n",
    "    def skip_whitespace(self):\n",
    "        while self.current_char is not None and self.current_char.isspace():\n",
    "            self.advance()\n",
    "\n",
    "    def integer(self):\n",
    "        \"\"\"Return a (multidigit) integer consumed from the input.\"\"\"\n",
    "        result = ''\n",
    "        while self.current_char is not None and self.current_char.isdigit():\n",
    "            result += self.current_char\n",
    "            self.advance()\n",
    "        return int(result)\n",
    "    def word(self):\n",
    "        \"\"\"Return a (multidigit) integer consumed from the input.\"\"\"\n",
    "        result = ''\n",
    "        while self.current_char is not None and (self.current_char.isalpha() or self.current_char == '_'):\n",
    "            result += self.current_char\n",
    "            self.advance()\n",
    "        return str(result)\n",
    "\n",
    "    def get_next_token(self):\n",
    "        \"\"\"Lexical analyzer (also known as scanner or tokenizer)\n",
    "\n",
    "        This method is responsible for breaking a sentence\n",
    "        apart into tokens. One token at a time.\n",
    "        \"\"\"\n",
    "        while self.current_char is not None:\n",
    "\n",
    "            if self.current_char.isspace():\n",
    "                self.skip_whitespace()\n",
    "                continue\n",
    "\n",
    "            if self.current_char.isdigit():\n",
    "                return Token(INTEGER, self.integer())\n",
    "\n",
    "            \n",
    "            \n",
    "            if self.current_char == '&':\n",
    "#                 print('gotB' + self.current_char)\n",
    "                self.advance()\n",
    "                return Token(AND, 'AND')\n",
    "            \n",
    "            if self.current_char == '|':\n",
    "#                 print('gotB' + self.current_char)\n",
    "                self.advance()\n",
    "                return Token(OR, 'OR')\n",
    "            \n",
    "            if self.current_char == '!':\n",
    "#                 print('gotB' + self.current_char)\n",
    "                self.advance()\n",
    "                return Token(NOT,'NOT')\n",
    "            \n",
    "            if self.current_char == '+':\n",
    "                self.advance()\n",
    "                return Token(PLUS, '+')\n",
    "\n",
    "            if self.current_char == '-':\n",
    "                self.advance()\n",
    "                return Token(MINUS, '-')\n",
    "\n",
    "            if self.current_char == '*':\n",
    "                self.advance()\n",
    "                return Token(MUL, '*')\n",
    "\n",
    "            if self.current_char == '/':\n",
    "                self.advance()\n",
    "                return Token(DIV, '/')\n",
    "\n",
    "            if self.current_char == '(':\n",
    "                self.advance()\n",
    "                return Token(LPAREN, '(')\n",
    "\n",
    "            if self.current_char == ')':\n",
    "                self.advance()\n",
    "                return Token(RPAREN, ')')\n",
    "            \n",
    "            if self.current_char.isalpha():      \n",
    "#                 print('Got token  ' + self.current_char)\n",
    "                return Token(TERM, self.word())\n",
    "            \n",
    "            \n",
    "\n",
    "            self.error()\n",
    "\n",
    "        return Token(EOF, None)\n",
    "\n",
    "\n",
    "###############################################################################\n",
    "#                                                                             #\n",
    "#  PARSER                                                                     #\n",
    "#                                                                             #\n",
    "###############################################################################\n",
    "\n",
    "class AST(object):\n",
    "    pass\n",
    "\n",
    "\n",
    "class BinOp(AST):\n",
    "    def __init__(self, left, op, right):\n",
    "        self.left = left\n",
    "        self.token = self.op = op\n",
    "        self.right = right\n",
    "        self.inverse = False\n",
    "        self.row = []\n",
    "        self.value = ''\n",
    "        \n",
    "class Num(AST):\n",
    "    def __init__(self, token):\n",
    "        self.token = token\n",
    "        self.value = token.value\n",
    "        self.inverse = False\n",
    "        self.row = []\n",
    "\n",
    "class Parser(object):\n",
    "    def __init__(self, lexer):\n",
    "        self.lexer = lexer\n",
    "        # set current token to the first token taken from the input\n",
    "        self.current_token = self.lexer.get_next_token()\n",
    "\n",
    "    def error(self):\n",
    "        raise Exception('Invalid syntax')\n",
    "\n",
    "    def eat(self, token_type):\n",
    "        # compare the current token type with the passed token\n",
    "        # type and if they match then \"eat\" the current token\n",
    "        # and assign the next token to the self.current_token,\n",
    "        # otherwise raise an exception.\n",
    "        if self.current_token.type == token_type:\n",
    "            self.current_token = self.lexer.get_next_token()\n",
    "        else:\n",
    "            self.error()\n",
    "\n",
    "    def factor(self):\n",
    "        \"\"\"factor : INTEGER | LPAREN expr RPAREN\"\"\"\n",
    "        token = self.current_token\n",
    "        \n",
    "        if token.type == INTEGER:\n",
    "            self.eat(INTEGER)\n",
    "            return Num(token)\n",
    "        \n",
    "        elif token.type == TERM:\n",
    "            self.eat(TERM)\n",
    "            return Num(token)\n",
    "        \n",
    "        elif token.type == NOT:\n",
    "            self.eat(NOT)\n",
    "            node = self.expr()\n",
    "            node.inverse = True\n",
    "            return node\n",
    "            \n",
    "        \n",
    "        \n",
    "        elif token.type == LPAREN:\n",
    "            self.eat(LPAREN)\n",
    "            node = self.expr()\n",
    "            self.eat(RPAREN)\n",
    "            return node\n",
    "\n",
    "    def term(self):\n",
    "        \"\"\"term : factor ((MUL | DIV) factor)*\"\"\"\n",
    "        node = self.factor()\n",
    "\n",
    "        while self.current_token.type in (MUL, DIV, AND):\n",
    "            token = self.current_token\n",
    "            if token.type == MUL:\n",
    "                self.eat(MUL)\n",
    "            elif token.type == DIV:\n",
    "                self.eat(DIV)\n",
    "            elif token.type == AND:\n",
    "                self.eat(AND)\n",
    "        \n",
    "            \n",
    "\n",
    "            node = BinOp(left=node, op=token, right=self.factor())\n",
    "\n",
    "        return node\n",
    "\n",
    "    def expr(self):\n",
    "        \"\"\"\n",
    "        expr   : term ((PLUS | MINUS) term)*\n",
    "        term   : factor ((MUL | DIV) factor)*\n",
    "        factor : INTEGER | LPAREN expr RPAREN\n",
    "        \"\"\"\n",
    "        node = self.term()\n",
    "\n",
    "        while self.current_token.type in (PLUS, MINUS, OR):\n",
    "            token = self.current_token\n",
    "            if token.type == OR:\n",
    "                self.eat(OR)\n",
    "            elif token.type == PLUS:\n",
    "                self.eat(PLUS)\n",
    "            elif token.type == MINUS:\n",
    "                self.eat(MINUS)\n",
    "            node = BinOp(left=node, op=token, right=self.term())\n",
    "\n",
    "        return node\n",
    "\n",
    "    def parse(self):\n",
    "        return self.expr()\n",
    "\n",
    "\n",
    "###############################################################################\n",
    "#                                                                             #\n",
    "#  INTERPRETER                                                                #\n",
    "#                                                                             #\n",
    "###############################################################################\n",
    "\n",
    "class NodeVisitor(object):\n",
    "    def visit(self, node):\n",
    "        print('Checking Node Name')\n",
    "        \n",
    "        method_name = 'visit_' + type(node).__name__\n",
    "        visitor = getattr(self, method_name, self.generic_visit)\n",
    "        return visitor(node)\n",
    "\n",
    "    def generic_visit(self, node):\n",
    "        raise Exception('No visit_{} method'.format(type(node).__name__))\n",
    "\n",
    "\n",
    "class Interpreter(NodeVisitor):\n",
    "    def __init__(self, parser):\n",
    "        self.parser = parser\n",
    "\n",
    "    def visit_BinOp(self, node):\n",
    "        print('Bin OP : ' )\n",
    "        print(node.token)\n",
    "        print(node.value)\n",
    "        print(node.row)\n",
    "        print(node.inverse)\n",
    "        if node.op.type == PLUS:\n",
    "            return str(self.visit(node.left)) + '+' +  str(self.visit(node.right))\n",
    "        elif node.op.type == MINUS:\n",
    "            return str(self.visit(node.left)) + '-' +  str(self.visit(node.right))\n",
    "        elif node.op.type == MUL:\n",
    "            return str(self.visit(node.left)) + '*' +  str(self.visit(node.right))\n",
    "        elif node.op.type == DIV:\n",
    "            return str(self.visit(node.left)) + '/' +  str(self.visit(node.right))\n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        elif node.op.type == AND:\n",
    "            print('Node => ')\n",
    "            print(node)\n",
    "            left = self.visit(node.left)   \n",
    "            right = self.visit(node.right)\n",
    "            \n",
    "#             term_index_left = vocab_list.index(ps.stem(left.value))\n",
    "#             term_row_left = term_doc_matrix_np[term_index_left]\n",
    "            \n",
    "#             term_index_right = vocab_list.index(ps.stem(right.value))\n",
    "#             term_row_right = term_doc_matrix_np[term_index_right]\n",
    "            \n",
    "            if left.inverse == True:\n",
    "                left.value = '!' + str(left.value)\n",
    "                term_row_left = [1 if int(x)==0 else 0 for x in left.row]\n",
    "                left.inverse = False\n",
    "            \n",
    "            if right.inverse == True:\n",
    "                right.value = '!' + str(right.value)\n",
    "                term_row_right = [1 if int(x)==0 else 0 for x in left.right]\n",
    "                right.inverse = False\n",
    "            \n",
    "            node.row = [1 if int(x) == 1 and int(y) == 1 else 0 for x,y in zip(left.row, right.row)]\n",
    "            if node.inverse == True:\n",
    "                node.row = [1 if int(x)==0 else 0 for x in node.row]\n",
    "                node.inverse = False\n",
    "            \n",
    "            return node\n",
    "        \n",
    "        elif node.op.type == OR:\n",
    "            print('Node => ')\n",
    "            print(node)\n",
    "            left = self.visit(node.left)   \n",
    "            print((left.token))\n",
    "            print((left.row))\n",
    "            \n",
    "            right = self.visit(node.right)\n",
    "            \n",
    "#             term_index_left = vocab_list.index(ps.stem(left.value))\n",
    "#             term_row_left = term_doc_matrix_np[term_index_left]\n",
    "            \n",
    "#             term_index_right = vocab_list.index(ps.stem(right.value))\n",
    "#             term_row_right = term_doc_matrix_np[term_index_right]\n",
    "            \n",
    "            if left.inverse == True:\n",
    "                left.value = '!' + str(left.value)\n",
    "                term_row_left = [1 if int(x)==0 else 0 for x in left.row]\n",
    "                left.inverse = False\n",
    "            \n",
    "            if right.inverse == True:\n",
    "                right.value = '!' + str(right.value)\n",
    "                term_row_right = [1 if int(x)==0 else 0 for x in left.right]\n",
    "                right.inverse = False\n",
    "            \n",
    "            node.row = [1 if int(x) == 1 or int(y) == 1 else 0 for x,y in zip(left.row, right.row)]\n",
    "            \n",
    "            return node\n",
    "        \n",
    "\n",
    "    def visit_Num(self, node):\n",
    "        print('Num  : ' )\n",
    "        print(node.token)\n",
    "        print(node.value)\n",
    "        print(node.row)\n",
    "        print(node.inverse)\n",
    "        \n",
    "        node.value = node.value.split('_')[0]\n",
    "        if ps.stem(node.value) in vocab_list:\n",
    "            \n",
    "            term_index = vocab_list.index(ps.stem(node.value))\n",
    "            term_row = term_doc_matrix_np[term_index]\n",
    "        else:\n",
    "            term_row = [0 for x in range(0,len(term_doc_matrix_np[0]))]\n",
    "            print('Term Row')\n",
    "            print(term_row)\n",
    "            \n",
    "        node.row = term_row\n",
    "        if node.inverse == True:\n",
    "            node.row = [1 if int(x)==0 else 0 for x in term_row]\n",
    "            node.inverse = False\n",
    "        \n",
    "        return node\n",
    "\n",
    "    def interpret(self):\n",
    "        tree = self.parser.parse()\n",
    "        print(tree)\n",
    "        return self.visit(tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter your query : pakistan OR afghanistan OR aid\n",
      "pakistan | afghanistan | aid\n",
      "<__main__.BinOp object at 0x000002395E7EAD30>\n",
      "Checking Node Name\n",
      "Bin OP : \n",
      "Token(OR, 'OR')\n",
      "\n",
      "[]\n",
      "False\n",
      "Node => \n",
      "<__main__.BinOp object at 0x000002395E7EAD30>\n",
      "Checking Node Name\n",
      "Bin OP : \n",
      "Token(OR, 'OR')\n",
      "\n",
      "[]\n",
      "False\n",
      "Node => \n",
      "<__main__.BinOp object at 0x000002395E7EAF98>\n",
      "Checking Node Name\n",
      "Num  : \n",
      "Token(TERM, 'pakistan')\n",
      "pakistan\n",
      "[]\n",
      "False\n",
      "Token(TERM, 'pakistan')\n",
      "[0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.\n",
      " 0. 0. 0. 0. 0. 0. 0. 0.]\n",
      "Checking Node Name\n",
      "Num  : \n",
      "Token(TERM, 'afghanistan')\n",
      "afghanistan\n",
      "[]\n",
      "False\n",
      "Token(OR, 'OR')\n",
      "[0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "Checking Node Name\n",
      "Num  : \n",
      "Token(TERM, 'aid')\n",
      "aid\n",
      "[]\n",
      "False\n",
      "\n",
      "[0, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]\n",
      "{1, 3, 4, 37, 39, 40, 9, 41, 42, 16, 17, 18, 22, 29}\n"
     ]
    }
   ],
   "source": [
    "text = input('Enter your query : ')\n",
    "text = text.replace('and','&')\n",
    "text = text.replace('AND','&')\n",
    "text = text.replace('or','|')\n",
    "text = text.replace('OR','|')\n",
    "text = text.replace('NOT', '!')\n",
    "text = text.replace('not','!')\n",
    "print(text)\n",
    "\n",
    "lexer = Lexer(text)\n",
    "parser = Parser(lexer)\n",
    "interpreter = Interpreter(parser)\n",
    "result = interpreter.interpret()\n",
    "print(result.value)\n",
    "print(result.row)\n",
    "\n",
    "final_doc_ids = np.argwhere(np.array(result.row) == 1)\n",
    "final_ans =  set([x[0] for x in final_doc_ids])\n",
    "print(final_ans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'3', '39', '29', '42', '17', '37', '9', '1', '41', '4', '22', '18', '40', '16'}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ansq ={'29', '16', '4', '22', '37', '40', '42', '18', '1', '17', '41', '39', '9', '3'}\n",
    "# ansi = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 39, 40, 41, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 54, 55}\n",
    "ansq2 = set([str(x) for x in final_ans])\n",
    "print(ansq2)\n",
    "ansq.difference(ansq2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "planeset\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'planeset'"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b= clean_word('plane_set')\n",
    "print(b)\n",
    "ps.stem(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "def intersect_posting(p1, p2):\n",
    "    if isinstance(p1, set) and isinstance(p2, set):\n",
    "        return p1.intersection(p2)\n",
    "    elif isinstance(p1, set):\n",
    "        return p1.intersection(p2.occurrance.keys())\n",
    "    elif isinstance(p2, set):\n",
    "        return p2.intersection(p1.occurrance.keys())\n",
    "    pn = PostingList()\n",
    "    pn.token = f'{p1.token} & {p2.token}'\n",
    "    for pn_keys in (p1.occurrance.keys() & p2.occurrance.keys()) :\n",
    "        pn.addOccurrance(pn_keys, p1.occurrance[pn_keys])\n",
    "        pn.addOccurrance(pn_keys, p2.occurrance[pn_keys])\n",
    "    return pn\n",
    "\n",
    "\n",
    "def union_posting(p1, p2):\n",
    "    if isinstance(p1, set) and isinstance(p2, set):\n",
    "        return p1.union(p2)\n",
    "    elif isinstance(p1, set):\n",
    "        return p1.union(p2.occurrance.keys())\n",
    "    elif isinstance(p2, set):\n",
    "        return p2.union(p1.occurance.keys())\n",
    "    \n",
    "    pn = PostingList()\n",
    "    pn.token = f'{p1.token} | {p2.token}'\n",
    "    for pn1_keys in p1.occurrance.keys() :\n",
    "        pn.addOccurrance(pn1_keys, p1.occurrance[pn1_keys])\n",
    "    for pn2_keys in p2.occurrance.keys() :\n",
    "        pn.addOccurrance(pn2_keys, p2.occurrance[pn2_keys])\n",
    "    \n",
    "    return pn\n",
    "def inverse_posting(inverted_index,p):\n",
    "    if isinstance(p, set) :\n",
    "        return set(inverted_index.docs.keys()).difference(p)\n",
    "    \n",
    "    return inverted_index.docs.keys() - p.occurrance.keys()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_ans = inverse_posting(inverted_index, inverted_index.index[ps.stem('hammer')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'1', '37', '52', '31', '3', '14', '8', '22', '38', '4', '15', '41', '55', '10', '44', '0', '29', '2', '23', '48', '11', '12', '5', '7', '9', '30', '13', '32', '47', '28', '6', '26'}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ansq ={'31', '28', '37', '30', '7', '10', '14', '1', '6', '41', '15', '11', '29', '26', '52', '13', '32', '44', '4', '8', '22', '38', '48', '0', '47', '2', '23', '9', '3', '5', '12', '55'}\n",
    "# ansi = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 39, 40, 41, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 54, 55}\n",
    "ansq2 = set([str(x) for x in final_ans])\n",
    "print(ansq2)\n",
    "ansq.difference(ansq2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "def positional_intersect(p1, p2, k):\n",
    "    ip = intersect_posting(p1, p2)\n",
    "    lip = sorted(list(ip))\n",
    "    ans = []\n",
    "    for doc in lip:\n",
    "#         print(type(p1))\n",
    "        positions1 = p1.occurrance[doc]\n",
    "        positions2 = p2.occurrance[doc]\n",
    "        index_p2 = 0\n",
    "        index_p1 = 0\n",
    "        for pos1 in positions1:\n",
    "            for pos2 in positions2:\n",
    "                if pos2['token_no'] -  pos1['token_no'] <= k and pos2['token_no'] -  pos1['token_no'] > 0:\n",
    "                    ans.append({'doc':doc, 'pos1':  pos1, 'pos2':pos2})\n",
    "            \n",
    "        \n",
    "        \n",
    "    return ans\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'40', '16', '24', '39', '12', '27', '20', '51', '26', '14', '8', '46', '45'}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ans = positional_intersect(inverted_index.index[ps.stem('keep')], inverted_index.index[ps.stem('out')], 3)\n",
    "final_ans = set(map(lambda pos: pos['doc'],ans))\n",
    "ansq ={'20', '24', '39', '40', '51'}\n",
    "# ansi = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 39, 40, 41, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 54, 55}\n",
    "ansq2 = set([str(x) for x in final_ans])\n",
    "print(ansq2)\n",
    "ansq.difference(ansq2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "total_cnt : 687 docs : [dict_keys([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55])]"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[{'doc': 2, 'row': 1, 'col': 7683, 'token_no': 1508},\n",
       " {'doc': 2, 'row': 1, 'col': 8050, 'token_no': 1576},\n",
       " {'doc': 2, 'row': 1, 'col': 8178, 'token_no': 1599},\n",
       " {'doc': 2, 'row': 1, 'col': 8375, 'token_no': 1632},\n",
       " {'doc': 2, 'row': 1, 'col': 8425, 'token_no': 1642},\n",
       " {'doc': 2, 'row': 1, 'col': 15804, 'token_no': 3106},\n",
       " {'doc': 2, 'row': 1, 'col': 16159, 'token_no': 3172}]"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inverted_index.index[ps.stem('hillary')]\n",
    "inverted_index.index[ps.stem('clinton')].occurrance[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'doc': 1, 'row': 1, 'col': 11216, 'token_no': 2225}]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[{'doc': 1, 'row': 1, 'col': 11224, 'token_no': 2226}]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inverted_index.index[ps.stem('hillary')].occurrance[1]\n",
    "inverted_index.index[ps.stem('clinton')].occurrance[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' SPI - Simple Pascal Interpreter '"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\" SPI - Simple Pascal Interpreter \"\"\"\n",
    "import re\n",
    "boperators = ['and', 'or']\n",
    "uoperators = ['not']\n",
    "\n",
    "# Token types\n",
    "#\n",
    "# EOF (end-of-file) token is used to indicate that\n",
    "# there is no more input left for lexical analysis\n",
    "LPAREN, RPAREN, EOF, TERM, AND, OR, NOT = (\n",
    "    '(', ')', 'EOF', 'TERM', 'AND','OR', 'NOT'\n",
    ")\n",
    "\n",
    "\n",
    "class Token(object):\n",
    "    def __init__(self, type, value):\n",
    "        self.type = type\n",
    "        self.value = value\n",
    "        self.inverse = False\n",
    "        self.row = []\n",
    "\n",
    "    def __str__(self):\n",
    "        \"\"\"String representation of the class instance.\n",
    "\n",
    "        Examples:\n",
    "            Token(TERM, Hello)\n",
    "            Token(AND, '&')\n",
    "            Token(NOT, '!')\n",
    "        \"\"\"\n",
    "        return 'Token({type}, {value})'.format(\n",
    "            type=self.type,\n",
    "            value=repr(self.value)\n",
    "        )\n",
    "\n",
    "    def __repr__(self):\n",
    "        return self.__str__()\n",
    "\n",
    "\n",
    "class Lexer(object):\n",
    "    def __init__(self, text):\n",
    "        # client string input, e.g. \"hello | world & (why | are | you)\"\n",
    "        self.text = text\n",
    "        # self.pos is an index into self.text\n",
    "        self.pos = 0\n",
    "        self.current_char = self.text[self.pos]\n",
    "        \n",
    "\n",
    "    def error(self):\n",
    "        raise Exception('Invalid character')\n",
    "\n",
    "    def advance(self):\n",
    "        \"\"\"Advance the `pos` pointer and set the `current_char` variable.\"\"\"\n",
    "        self.pos += 1\n",
    "        if self.pos > len(self.text) - 1:\n",
    "            self.current_char = None  # Indicates end of input\n",
    "        else:\n",
    "            self.current_char = self.text[self.pos]\n",
    "\n",
    "    def skip_whitespace(self):\n",
    "        while self.current_char is not None and self.current_char.isspace():\n",
    "            self.advance()\n",
    "\n",
    "    def integer(self):\n",
    "        \"\"\"Return a (multidigit) integer consumed from the input.\"\"\"\n",
    "        result = ''\n",
    "        while self.current_char is not None and self.current_char.isdigit():\n",
    "            result += self.current_char\n",
    "            self.advance()\n",
    "        return int(result)\n",
    "    def word(self):\n",
    "        \"\"\"Return a (multidigit) integer consumed from the input.\"\"\"\n",
    "        result = ''\n",
    "        while self.current_char is not None and (self.current_char.isalpha() or self.current_char == '_'):\n",
    "            result += self.current_char\n",
    "            self.advance()\n",
    "        return str(result)\n",
    "\n",
    "    def get_next_token(self):\n",
    "        \"\"\"Lexical analyzer (also known as scanner or tokenizer)\n",
    "\n",
    "        This method is responsible for breaking a sentence\n",
    "        apart into tokens. One token at a time.\n",
    "        \"\"\"\n",
    "        while self.current_char is not None:\n",
    "\n",
    "            if self.current_char.isspace():\n",
    "                self.skip_whitespace()\n",
    "                continue\n",
    "\n",
    "            if self.current_char.isdigit():\n",
    "                return Token(INTEGER, self.integer())\n",
    "\n",
    "            \n",
    "            \n",
    "            if self.current_char == '&':\n",
    "#                 print('gotB' + self.current_char)\n",
    "                self.advance()\n",
    "                return Token(AND, 'AND')\n",
    "            \n",
    "            if self.current_char == '|':\n",
    "#                 print('gotB' + self.current_char)\n",
    "                self.advance()\n",
    "                return Token(OR, 'OR')\n",
    "            \n",
    "            if self.current_char == '!':\n",
    "#                 print('gotB' + self.current_char)\n",
    "                self.advance()\n",
    "                return Token(NOT,'NOT')\n",
    "            \n",
    "\n",
    "            if self.current_char == '(':\n",
    "                self.advance()\n",
    "                return Token(LPAREN, '(')\n",
    "\n",
    "            if self.current_char == ')':\n",
    "                self.advance()\n",
    "                return Token(RPAREN, ')')\n",
    "            \n",
    "            if self.current_char.isalpha():      \n",
    "#                 print('Got token  ' + self.current_char)\n",
    "                return Token(TERM, self.word())\n",
    "            \n",
    "            \n",
    "\n",
    "            self.error()\n",
    "\n",
    "        return Token(EOF, None)\n",
    "\n",
    "\n",
    "###############################################################################\n",
    "#                                                                             #\n",
    "#  PARSER                                                                     #\n",
    "#                                                                             #\n",
    "###############################################################################\n",
    "\n",
    "class AST(object):\n",
    "    pass\n",
    "\n",
    "\n",
    "class BinOp(AST):\n",
    "    def __init__(self, left, op, right):\n",
    "        self.left = left\n",
    "        self.token = self.op = op\n",
    "        self.right = right\n",
    "        self.inverse = False\n",
    "        self.row = []\n",
    "        self.value = ''\n",
    "        \n",
    "class Num(AST):\n",
    "    def __init__(self, token):\n",
    "        self.token = token\n",
    "        self.value = token.value\n",
    "        self.inverse = False\n",
    "        self.row = []\n",
    "\n",
    "class Parser(object):\n",
    "    def __init__(self, lexer):\n",
    "        self.lexer = lexer\n",
    "        # set current token to the first token taken from the input\n",
    "        self.current_token = self.lexer.get_next_token()\n",
    "\n",
    "    def error(self):\n",
    "        raise Exception('Invalid syntax')\n",
    "\n",
    "    def eat(self, token_type):\n",
    "        # compare the current token type with the passed token\n",
    "        # type and if they match then \"eat\" the current token\n",
    "        # and assign the next token to the self.current_token,\n",
    "        # otherwise raise an exception.\n",
    "        if self.current_token.type == token_type:\n",
    "            self.current_token = self.lexer.get_next_token()\n",
    "        else:\n",
    "            self.error()\n",
    "\n",
    "    def factor(self):\n",
    "        \"\"\"factor : INTEGER | LPAREN expr RPAREN\"\"\"\n",
    "        token = self.current_token\n",
    "        \n",
    "        \n",
    "        if token.type == TERM:\n",
    "            self.eat(TERM)\n",
    "            return Num(token)\n",
    "        \n",
    "        elif token.type == NOT:\n",
    "            self.eat(NOT)\n",
    "            node = self.expr()\n",
    "            node.inverse = True\n",
    "            return node\n",
    "            \n",
    "        \n",
    "        \n",
    "        elif token.type == LPAREN:\n",
    "            self.eat(LPAREN)\n",
    "            node = self.expr()\n",
    "            self.eat(RPAREN)\n",
    "            return node\n",
    "\n",
    "    def term(self):\n",
    "        \"\"\"term : factor ((MUL | DIV) factor)*\"\"\"\n",
    "        node = self.factor()\n",
    "\n",
    "        while self.current_token.type in (AND,):\n",
    "            token = self.current_token\n",
    "         \n",
    "            if token.type == AND:\n",
    "                self.eat(AND)\n",
    "        \n",
    "            \n",
    "\n",
    "            node = BinOp(left=node, op=token, right=self.factor())\n",
    "\n",
    "        return node\n",
    "\n",
    "    def expr(self):\n",
    "        \"\"\"\n",
    "        expr   : term ((PLUS | MINUS) term)*\n",
    "        term   : factor ((MUL | DIV) factor)*\n",
    "        factor : INTEGER | LPAREN expr RPAREN\n",
    "        \"\"\"\n",
    "        node = self.term()\n",
    "\n",
    "        while self.current_token.type in (OR,):\n",
    "            token = self.current_token\n",
    "            if token.type == OR:\n",
    "                self.eat(OR)\n",
    "            node = BinOp(left=node, op=token, right=self.term())\n",
    "\n",
    "        return node\n",
    "\n",
    "    def parse(self):\n",
    "        return self.expr()\n",
    "\n",
    "\n",
    "class NodeVisitor(object):\n",
    "    def visit(self, node):\n",
    "        print('Checking Node Name')\n",
    "        \n",
    "        method_name = 'visit_' + type(node).__name__\n",
    "        visitor = getattr(self, method_name, self.generic_visit)\n",
    "        return visitor(node)\n",
    "\n",
    "    def generic_visit(self, node):\n",
    "        raise Exception('No visit_{} method'.format(type(node).__name__))\n",
    "\n",
    "\n",
    "class Interpreter(NodeVisitor):\n",
    "    def __init__(self, parser, index, ps):\n",
    "        self.parser = parser\n",
    "        self.index = index\n",
    "        self.ps = ps\n",
    "\n",
    "    def visit_BinOp(self, node):\n",
    "        print('Bin OP : ' )\n",
    "        print(node.token)\n",
    "        print(node.value)\n",
    "        print(node.row)\n",
    "        print(node.inverse)\n",
    "        if node.op.type == AND:\n",
    "            print('Node => ')\n",
    "            print(node)\n",
    "            left = self.visit(node.left)   \n",
    "            right = self.visit(node.right)\n",
    "            \n",
    "#             term_index_left = vocab_list.index(ps.stem(left.value))\n",
    "#             term_row_left = term_doc_matrix_np[term_index_left]\n",
    "            \n",
    "#             term_index_right = vocab_list.index(ps.stem(right.value))\n",
    "#             term_row_right = term_doc_matrix_np[term_index_right]\n",
    "            \n",
    "            if left.inverse == True:\n",
    "                left.value = '!' + str(left.value)\n",
    "                term_row_left = self.index.index[self.ps.stem(left.row)]\n",
    "                left.inverse = False\n",
    "            \n",
    "            if right.inverse == True:\n",
    "                right.value = '!' + str(right.value)\n",
    "                term_row_right = self.index.index[self.ps.stem(right.row)]\n",
    "                right.inverse = False\n",
    "            \n",
    "            node.row = intersect_posting(left.row, right.row)\n",
    "            if node.inverse == True:\n",
    "                node.row = self.index.index[self.ps.stem(node.row)]\n",
    "                node.inverse = False\n",
    "            \n",
    "            return node\n",
    "        \n",
    "        elif node.op.type == OR:\n",
    "            print('Node => ')\n",
    "            print(node)\n",
    "            left = self.visit(node.left)   \n",
    "            right = self.visit(node.right)\n",
    "            \n",
    "#             term_index_left = vocab_list.index(ps.stem(left.value))\n",
    "#             term_row_left = term_doc_matrix_np[term_index_left]\n",
    "            \n",
    "#             term_index_right = vocab_list.index(ps.stem(right.value))\n",
    "#             term_row_right = term_doc_matrix_np[term_index_right]\n",
    "            \n",
    "            if left.inverse == True:\n",
    "                left.value = '!' + str(left.value)\n",
    "                term_row_left = self.index.index[self.ps.stem(left.row)]\n",
    "                left.inverse = False\n",
    "            \n",
    "            if right.inverse == True:\n",
    "                right.value = '!' + str(right.value)\n",
    "                term_row_right = self.index.index[self.ps.stem(right.row)]\n",
    "                right.inverse = False\n",
    "            \n",
    "            node.row = union_posting(left.row, right.row)\n",
    "            if node.inverse == True:\n",
    "                node.row = self.index.index[self.ps.stem(node.row)]\n",
    "                node.inverse = False\n",
    "            \n",
    "            return node\n",
    "        \n",
    "\n",
    "    def visit_Num(self, node):\n",
    "        print('Num  : ' )\n",
    "        print(node.token)\n",
    "        print(node.value)\n",
    "        print(node.row)\n",
    "        print(node.inverse)\n",
    "        \n",
    "        node.value = node.value.split('_')[0]\n",
    "        if self.ps.stem(node.value) in self.index.index.keys():\n",
    "            \n",
    "            term_docs = self.index.index[self.ps.stem(node.value)]\n",
    "            \n",
    "        else:\n",
    "            term_docs = {}\n",
    "            print('Term Row')\n",
    "            print(term_docs)\n",
    "            \n",
    "        node.row = term_docs\n",
    "        if node.inverse == True:\n",
    "            node.row = inverse_posting(node.row)\n",
    "            node.inverse = False\n",
    "        \n",
    "        return node\n",
    "\n",
    "    def interpret(self):\n",
    "        tree = self.parser.parse()\n",
    "        print(tree)\n",
    "        return self.visit(tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter your query : united or plane\n",
      "united | plane\n",
      "<__main__.BinOp object at 0x000001B6003C5358>\n",
      "Checking Node Name\n",
      "Bin OP : \n",
      "Token(OR, 'OR')\n",
      "\n",
      "[]\n",
      "False\n",
      "Node => \n",
      "<__main__.BinOp object at 0x000001B6003C5358>\n",
      "Checking Node Name\n",
      "Num  : \n",
      "Token(TERM, 'united')\n",
      "united\n",
      "[]\n",
      "False\n",
      "Checking Node Name\n",
      "Num  : \n",
      "Token(TERM, 'plane')\n",
      "plane\n",
      "[]\n",
      "False\n",
      "\n",
      "total_cnt : 68 docs : [dict_keys([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 16, 17, 18, 19, 20, 21, 22, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 54, 55])]\n"
     ]
    }
   ],
   "source": [
    "text = input('Enter your query : ')\n",
    "text = text.replace('and','&')\n",
    "text = text.replace('AND','&')\n",
    "text = text.replace('or','|')\n",
    "text = text.replace('OR','|')\n",
    "text = text.replace('NOT', '!')\n",
    "text = text.replace('not','!')\n",
    "print(text)\n",
    "\n",
    "lexer = Lexer(text)\n",
    "parser = Parser(lexer)\n",
    "interpreter = Interpreter(parser, inverted_index, ps)\n",
    "result = interpreter.interpret()\n",
    "print(result.value)\n",
    "print(result.row)\n",
    "\n",
    "# final_doc_ids = np.argwhere(np.array(result.row) == 1)\n",
    "# final_ans =  set([x[0] for x in result.row])\n",
    "# print(final_ans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{4: [[{'doc': 4, 'row': 1, 'col': 7000, 'token_no': 1321}, {'doc': 4, 'row': 1, 'col': 13488, 'token_no': 2508}], [[{'doc': 4, 'row': 1, 'col': 10044, 'token_no': 1894}]]], 6: [[{'doc': 6, 'row': 1, 'col': 13926, 'token_no': 3070}], [[{'doc': 6, 'row': 1, 'col': 20719, 'token_no': 4526}]]], 43: [[{'doc': 43, 'row': 1, 'col': 8730, 'token_no': 1589}], [[{'doc': 43, 'row': 1, 'col': 2692, 'token_no': 500}]]], 44: [[{'doc': 44, 'row': 1, 'col': 7302, 'token_no': 1388}, {'doc': 44, 'row': 1, 'col': 7365, 'token_no': 1401}], [[{'doc': 44, 'row': 1, 'col': 4742, 'token_no': 897}]]], 45: [[{'doc': 45, 'row': 1, 'col': 10864, 'token_no': 1973}], [[{'doc': 45, 'row': 1, 'col': 1748, 'token_no': 325}]]], 46: [[{'doc': 46, 'row': 1, 'col': 6288, 'token_no': 1195}, {'doc': 46, 'row': 1, 'col': 7615, 'token_no': 1451}], [[{'doc': 46, 'row': 1, 'col': 1457, 'token_no': 289}]]], 47: [[{'doc': 47, 'row': 1, 'col': 10166, 'token_no': 1850}], [[{'doc': 47, 'row': 1, 'col': 1432, 'token_no': 269}]]], 18: [[{'doc': 18, 'row': 1, 'col': 2619, 'token_no': 505}, {'doc': 18, 'row': 1, 'col': 13385, 'token_no': 2545}], [[{'doc': 18, 'row': 1, 'col': 7599, 'token_no': 1485}], [{'doc': 18, 'row': 1, 'col': 3819, 'token_no': 759}]]], 51: [[{'doc': 51, 'row': 1, 'col': 8910, 'token_no': 1644}], [[{'doc': 51, 'row': 1, 'col': 7199, 'token_no': 1330}]]], 50: [[{'doc': 50, 'row': 1, 'col': 7915, 'token_no': 1483}], [[{'doc': 50, 'row': 1, 'col': 4399, 'token_no': 811}]]], 53: [[{'doc': 53, 'row': 1, 'col': 241, 'token_no': 53}], [[{'doc': 53, 'row': 1, 'col': 983, 'token_no': 188}]]], 54: [[{'doc': 54, 'row': 1, 'col': 1817, 'token_no': 346}, {'doc': 54, 'row': 1, 'col': 10737, 'token_no': 1994}], [[{'doc': 54, 'row': 1, 'col': 6139, 'token_no': 1158}], [{'doc': 54, 'row': 1, 'col': 2693, 'token_no': 511}]]]}\n",
      "{'47', '50', '53', '6', '54', '51', '18', '43', '46', '45', '4', '44'}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ans = result.row\n",
    "final_ans = ans\n",
    "print(ans.occurrance)\n",
    "final_ans = set(map(lambda pos: pos,ans.occurrance.keys()))\n",
    "ansq ={'18', '46', '54', '43', '4', '45', '50', '53', '47', '6', '51', '44'}\n",
    "# ansi = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 39, 40, 41, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 54, 55}\n",
    "ansq2 = set([str(x) for x in final_ans])\n",
    "print(ansq2)\n",
    "ansq.difference(ansq2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{4: [[{'doc': 4, 'row': 1, 'col': 7000, 'token_no': 1321},\n",
       "   {'doc': 4, 'row': 1, 'col': 13488, 'token_no': 2508}],\n",
       "  [[{'doc': 4, 'row': 1, 'col': 10044, 'token_no': 1894}]]],\n",
       " 6: [[{'doc': 6, 'row': 1, 'col': 13926, 'token_no': 3070}],\n",
       "  [[{'doc': 6, 'row': 1, 'col': 20719, 'token_no': 4526}]]],\n",
       " 43: [[{'doc': 43, 'row': 1, 'col': 8730, 'token_no': 1589}],\n",
       "  [[{'doc': 43, 'row': 1, 'col': 2692, 'token_no': 500}]]],\n",
       " 44: [[{'doc': 44, 'row': 1, 'col': 7302, 'token_no': 1388},\n",
       "   {'doc': 44, 'row': 1, 'col': 7365, 'token_no': 1401}],\n",
       "  [[{'doc': 44, 'row': 1, 'col': 4742, 'token_no': 897}]]],\n",
       " 45: [[{'doc': 45, 'row': 1, 'col': 10864, 'token_no': 1973}],\n",
       "  [[{'doc': 45, 'row': 1, 'col': 1748, 'token_no': 325}]]],\n",
       " 46: [[{'doc': 46, 'row': 1, 'col': 6288, 'token_no': 1195},\n",
       "   {'doc': 46, 'row': 1, 'col': 7615, 'token_no': 1451}],\n",
       "  [[{'doc': 46, 'row': 1, 'col': 1457, 'token_no': 289}]]],\n",
       " 47: [[{'doc': 47, 'row': 1, 'col': 10166, 'token_no': 1850}],\n",
       "  [[{'doc': 47, 'row': 1, 'col': 1432, 'token_no': 269}]]],\n",
       " 18: [[{'doc': 18, 'row': 1, 'col': 2619, 'token_no': 505},\n",
       "   {'doc': 18, 'row': 1, 'col': 13385, 'token_no': 2545}],\n",
       "  [[{'doc': 18, 'row': 1, 'col': 7599, 'token_no': 1485}],\n",
       "   [{'doc': 18, 'row': 1, 'col': 3819, 'token_no': 759}]]],\n",
       " 51: [[{'doc': 51, 'row': 1, 'col': 8910, 'token_no': 1644}],\n",
       "  [[{'doc': 51, 'row': 1, 'col': 7199, 'token_no': 1330}]]],\n",
       " 50: [[{'doc': 50, 'row': 1, 'col': 7915, 'token_no': 1483}],\n",
       "  [[{'doc': 50, 'row': 1, 'col': 4399, 'token_no': 811}]]],\n",
       " 53: [[{'doc': 53, 'row': 1, 'col': 241, 'token_no': 53}],\n",
       "  [[{'doc': 53, 'row': 1, 'col': 983, 'token_no': 188}]]],\n",
       " 54: [[{'doc': 54, 'row': 1, 'col': 1817, 'token_no': 346},\n",
       "   {'doc': 54, 'row': 1, 'col': 10737, 'token_no': 1994}],\n",
       "  [[{'doc': 54, 'row': 1, 'col': 6139, 'token_no': 1158}],\n",
       "   [{'doc': 54, 'row': 1, 'col': 2693, 'token_no': 511}]]]}"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ans.occurrance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}