heap index inital

Author: orgh0

invertedindex.py

@@ -1,3 +1,4 @@
+import os
 import re
 import argparse
 from collections import *
@@ -6,8 +7,10 @@
 from utils import get_context, update_words
 
 index_categories = ['title', 'text', 'category', 'infobox']
+final_index_files = list()
 
 pagination_val  = 10
+page_tags = open("./data/title_tags.txt")
 
 def make_index(iter_doc):
     num_pages = 0
@@ -48,6 +51,7 @@ def make_index(iter_doc):
                 try:
                     text = text.lower()
                     title = text+"\n"
+                    page_tags.write(title)
                     update_words(word_dict["title"], text, istext=True)
                 except:
                     pass
@@ -64,10 +68,8 @@ def make_index(iter_doc):
 
                 if num_pages % pagination_val == 0:
                     for key, value in index_dict.items():
-                        # print(key)
-                        # print(value)
                         file = "./output/"+key[0:2]+str(output_file_num)+".txt"
-                        o = open(file, "w")
+                        o = open(titlefile, "w")
                         for word in sorted(value):
                             index = ",".join(value[word])
                             index = word + "-" + index + "\n"
@@ -77,11 +79,32 @@ def make_index(iter_doc):
                     for key, val in index_dict.items():
                         index_dict[key].clear()
             elem.clear()
+    return output_file_num
+
+def make_heaped_index(numer_of_files):
+    for category in index_categories:
+        cat_inp = list()
+        cat_fp = open("output/" + category + ".txt", "w")
+        final_index_files.append(cat_fp)
+        for i in range(numer_of_files):
+            temp_file = "output/" + cat[0:2] + str(i) + ".txt"
+            if os.stat(temp_file).st_size != 0:
+                temp_file_fp = open(temp_file, "r")
+                cat_inp.append(temp_file_fp)
+            else:
+                pass
+        if(len(cat_inp) == 0):
+            break           
+            
+
+
+
 
 
 def main(data, output_dep_folder):
     iter_doc = get_context(data)
-    make_index(iter_doc)
+    num = make_index(iter_doc)
+    make_heaped_index(num)
 
 if __name__ == '__main__':