color modified

Author: shubham1172

knowledge_graph.py

@@ -37,9 +37,7 @@ def create_knowledge_graph(frame):
     :return: knowledge dict and plotted image
     """
 
-    knowledge = {"scene": "", "classes": {}}
-    # add scene
-    knowledge["scene"] = SceneClassifier.predict(frame)
+    knowledge = {"scene": SceneClassifier.predict(frame), "classes": {}}
 
     boxes, classes = ObjectDetector.predict(frame)
     class_count = Counter(classes)
@@ -54,7 +52,7 @@ def create_knowledge_graph(frame):
         l, t, r, b = list(map(int, box))
         crop = frame[t:b, l:r]
         text = TextDetector.detect(crop)
-        color = ColorDetector.predict(crop)
+        color = ColorDetector.find_color(crop)
         index = len(knowledge["classes"][classes[i]].get("objects", {}))
         if index == 0:
             knowledge["classes"][classes[i]]["objects"] = {}

modules/answer_generator.py

@@ -3,14 +3,16 @@
 
 
 class AnswerGenerator:
-    def __init__(self):
+    def __init__(self, verbose=False):
         """
         create a model from pre-trained weights
         """
-        print("START")
+        if verbose:
+            print("Loading model. This might take a while...")
         with nostderrout():
             self.model = build_model(configs.squad.squad)
-        print("END")
+        if verbose:
+            print("Loaded model.")
 
     def predict(self, comprehension, question):
         """

modules/color_detection.py

@@ -1,58 +1,75 @@
-from sklearn.cluster import KMeans
+import cv2
 import numpy as np
-import webcolors
 
 
 class ColorDetector:
 
+    colors = {"black": [0, 0, 0],
+              "white": [255, 255, 255],
+              "gray": [102, 102, 102],
+              "maroon": [140, 0, 0],
+              "red": [255, 0, 0],
+              "orange": [255, 85, 0],
+              "yellow": [255, 170, 0],
+              "green": [0, 255, 0],
+              "aqua": [0, 255, 242],
+              "blue": [0, 85, 255],
+              "purple": [102, 0, 255],
+              "pink": [255, 0, 255]}
+
     @staticmethod
-    def find_closest_color(requested_colour):
+    def nearest_color(color):
         """
-        finds the closest color in the sixteen HTML4 color
-        :param requested_colour: RGB value of color
-        :return: RGB value of color closest among HTML4 colors
+        find the nearest color
+        :param color: (R, G, B)
+        :return: nearest (R', G', B')
         """
-        min_colours = {}
-        for key, name in webcolors.css3_hex_to_names.items():
-            r_c, g_c, b_c = webcolors.hex_to_rgb(key)
-            rd = (r_c - requested_colour[0]) ** 2
-            gd = (g_c - requested_colour[1]) ** 2
-            bd = (b_c - requested_colour[2]) ** 2
-            min_colours[(rd + gd + bd)] = name
-        return min_colours[min(min_colours.keys())]
+        wrapper = np.uint8([[list(color)]])
+        hsv_wrapper = cv2.cvtColor(wrapper, cv2.COLOR_RGB2HSV)
+        hsv = hsv_wrapper[0, 0] + [10, 0, 0]
+        h, s, v = hsv
+        if v < 30:
+            return ColorDetector.colors["black"]
+        elif v < 80:
+            return ColorDetector.colors["gray"]
+        elif v > 190 and s < 27:
+            return ColorDetector.colors["white"]
+        elif s < 54 and v < 185:
+            return ColorDetector.colors["gray"]
+        elif h < 18:
+            if v < 150:
+                return ColorDetector.colors["maroon"]
+            else:
+                return ColorDetector.colors["red"]
+        elif h < 26:
+            return ColorDetector.colors["orange"]
+        elif h < 34:
+            return ColorDetector.colors["yellow"]
+        elif h < 73:
+            return ColorDetector.colors["green"]
+        elif h < 102:
+            return ColorDetector.colors["aqua"]
+        elif h < 127:
+            return ColorDetector.colors["blue"]
+        elif h < 149:
+            return ColorDetector.colors["purple"]
+        elif h < 175:
+            return ColorDetector.colors["pink"]
+        else:
+            return ColorDetector.colors["red"]
 
     @staticmethod
-    def get_colour_name(requested_colour):
-        """
-        get color name of passed RGB value
-        :param requested_colour: RGB value of a color
-        :return: named color of the RGB value
-        """
-        try:
-            closest_name = webcolors.rgb_to_name(requested_colour)
-        except ValueError:
-            closest_name = ColorDetector.find_closest_color(requested_colour)
-        return closest_name
+    def approximate_image(image):
+        return np.asarray([[ColorDetector.nearest_color(pixel) for pixel in row] for row in image])
 
     @staticmethod
-    def predict(frame):
-        """
-        predicts primary color in the frame
-        :param frame: input image as numpy array
-        :return: name of the color
-        """
-        height, width, dim = frame.shape
-        resized = np.resize(frame, (width * height, dim))
-        # cluster
-        k_means = KMeans(16)
-        labels = k_means.fit_predict(resized)
-        palette = k_means.cluster_centers_
-        # create new image
-        new_image = np.reshape(palette[labels], (width, height, palette.shape[1]))
-        new_image = new_image.astype(np.uint8)
-        # find dominant color
-        unique, counts = np.unique(new_image.reshape(-1, new_image.shape[2]), return_counts=True, axis=0)
-        sort_ix = np.argsort(counts)
-        sort_ix = sort_ix[-1]
-        
-        return ColorDetector.get_colour_name(tuple(unique[sort_ix]))
+    def find_color(image):
+        image = ColorDetector.approximate_image(image)
+        (values, counts) = np.unique(image.reshape(-1, image.shape[2]), return_counts=True, axis=0)
+        index = np.argmax(counts)
+        color = values[index]
+        for (key, value) in ColorDetector.colors.items():
+            if (value == color).all():
+                return key
+        return "black"
+

modules/paragraph_generator.py

@@ -1,6 +1,7 @@
 import inflect
 from num2words import num2words
 
+
 class ParagraphGenerator:
     def __init__(self):
         self.ie = inflect.engine()
@@ -35,6 +36,36 @@ def describe_class_count(self, class_, count):
             self.ie.plural(class_, count=count)
         )
 
+    def describe_object_text(self, classname, id, text, single=False):
+        """
+        describe the text on the object
+        Example:
+            The first board says hello.
+            Hello is written on the first stop sign.
+        :param classname: object to describe
+        :param id: identifier of the object
+        :param text: text on the object
+        :return: sentence describing text
+        """
+        num2word = num2words(id, to='ordinal') + ' '
+        if single:
+            num2word = ""
+
+        description = ""
+
+        description += "The %s%s says %s. " % (
+            num2word,
+            classname,
+            text
+        )
+        description += "%s is written on the %s%s. " % (
+            text,
+            num2word,
+            classname
+        )
+
+        return description
+
     def describe_object_color(self, classname, id, color, single=False):
         """
         describe the color of the object
@@ -45,6 +76,7 @@ def describe_object_color(self, classname, id, color, single=False):
         :param classname: object to describe
         :param id: identifier of the object
         :param color: color of the object
+        :param single: if object is occurring once, do not mention index
         :return: sentence describing color
         """
         num2word = num2words(id, to='ordinal') + ' '
@@ -62,36 +94,6 @@ def describe_object_color(self, classname, id, color, single=False):
             color
         )
 
-        def describe_object_text(self, classname, id, text, single=False):
-            """
-            describe the text on the object
-            Example:
-                The first board says hello.
-                Hello is written on the first stop sign.
-            :param classname: object to describe
-            :param id: identifier of the object
-            :param text: text on the object
-            :return: sentence describing text
-            """
-            num2word = num2words(id, to='ordinal') + ' '
-            if single:
-                num2word = ""
-
-            description = ""
-
-            description +=  "The %s%s says %s. " % (
-                num2word,
-                classname,
-                text
-            )
-            description +=  "%s is written on the %s%s. " % (
-                text,
-                num2word,
-                classname
-            )
-
-            return description
-
     def describe_scene(self, scene):
         """
         describe the scene

modules/text_detection.py

@@ -1,4 +1,5 @@
 import cv2
+import re
 import numpy as np
 import pytesseract
 from imutils.object_detection import non_max_suppression
@@ -86,11 +87,10 @@ def get_stitched_text(detected):
         :param detected: array of [text, bounding box]
         :return: sentence of words
         """
-        # TODO: complete this function
         sentence = ""
         for text, _ in detected:
-            sentence += text.strip() + " "
-        return sentence
+            sentence += " ".join(re.findall("[a-zA-Z]+", text)).strip() + " "
+        return sentence.strip()
 
     @staticmethod
     def detect(frame):

run.py

@@ -14,15 +14,17 @@
 
 knowledge, frame = knowledge_graph.create_knowledge_graph(image)
 pprint(knowledge)
-
-paragraph_generator = ParagraphGenerator()
-paragraph = paragraph_generator.generate(knowledge)
-
-answer_generator = AnswerGenerator()
-print(paragraph)
-
-while True:
-    question = input().strip()
-    if question == "":
-        break
-    print(answer_generator.predict(paragraph, question))
+#
+# paragraph_generator = ParagraphGenerator()
+# paragraph = paragraph_generator.generate(knowledge)
+#
+# print(paragraph)
+# answer_generator = AnswerGenerator(verbose=True)
+#
+# while True:
+#     print("[QUESTION] ", end="")
+#     question = input().strip()
+#     if question == "":
+#         break
+#     answer = answer_generator.predict(paragraph, question)[0]
+#     print("[ANSWER] %s" % answer)