-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathAR-Keyboard.py
208 lines (186 loc) · 6.82 KB
/
AR-Keyboard.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
import cv2
import pickle
import numpy as npy
import pyautogui as gui
with open('range.pickle','rb') as f:
t = pickle.load(f)
cam = cv2.VideoCapture(0)
hsv_lower = npy.array([t[0],t[1],t[2]])
hsv_upper = npy.array([t[3],t[4],t[5]])
width = cam.get(cv2.CAP_PROP_FRAME_WIDTH)
height = cam.get(cv2.CAP_PROP_FRAME_HEIGHT)
max_keys_in_row = 10
key_width = int(width/max_keys_in_row)
s = ''
def format_to_file():
global s
f = open('ouput.txt','w+')
formatted = ''
for a in s:
if a=='~':
formatted=formatted[:-1]
if a=='|':
formatted+='\n'
else:
formatted+=a
print(formatted)
f.write(formatted)
f.flush()
f.close()
def get_keys():
"""
this function is used to design the keyboard.
it returns the 4 parameter that are needed to design the keys.
they are key label, top right corner coordinate, bottom left corner coordinate, and center coordinate
"""
row0_key_width = key_width * 10
row1_key_width = key_width * 10 # width of first row of keys
row2_key_width = key_width * 10 # width of second row
row3_key_width = key_width * 10 # width of third row
row4_key_width = key_width * 7 # width of spacebar
row_keys = [] # stores the keys along with its 2 corner coordinates and the center coordinate
# for the first row
x1, y1 = 0, int((height - key_width * 5) / 2) # 4 is due to the fact that we will have 4 rows. y1 is set such that the whole keyboard has equal margin on both top and bottom
x2, y2 = key_width + x1, key_width + y1
c1, c2 = x1, y1 # copying x1, x2, y1 and y2
c = 0
keys = "1234567890"
for i in range(len(keys)):
row_keys.append([keys[c], (x1, y1), (x2, y2), (int((x2+x1)/2) - 5, int((y2+y1)/2) + 10)])
x1 += key_width
x2 += key_width
c += 1
x1, y1 = c1, c2
x1, y1 = int((row0_key_width - row1_key_width) / 2) + x1, y1 + key_width # 4 is due to the fact that we will have 4 rows. y1 is set such that the whole keyboard has equal margin on both top and bottom
x2, y2 = key_width + x1, key_width + y1
c1, c2 = x1, y1 # copying x1, x2, y1 and y2
c = 0
keys = "qwertyuiop"
for i in range(len(keys)):
row_keys.append([keys[c], (x1, y1), (x2, y2), (int((x2+x1)/2) - 5, int((y2+y1)/2) + 10)])
x1 += key_width
x2 += key_width
c += 1
x1, y1 = c1, c2
# for second row
x1, y1 = int((row1_key_width - row2_key_width) / 2) + x1, y1 + key_width # x1 is set such that it leaves equal margin on both left and right side
x2, y2 = key_width + x1, key_width + y1
c1, c2 = x1, y1
c = 0
keys = "asdfghjkl"
for i in range(len(keys)):
row_keys.append([keys[c], (x1, y1), (x2, y2), (int((x2+x1)/2) - 5, int((y2+y1)/2) + 10)])
x1 += key_width
x2 += key_width
c += 1
x1, y1 = c1, c2
# for third row
x1, y1 = int((row2_key_width - row3_key_width) / 2) + x1, y1 + key_width
x2, y2 = key_width + x1, key_width + y1
c1, c2 = x1, y1
c = 0
keys = "zxcvbnm~|$"
for i in range(len(keys)):
row_keys.append([keys[c], (x1, y1), (x2, y2), (int((x2+x1)/2) - 5, int((y2+y1)/2) + 10)])
x1 += key_width
x2 += key_width
c += 1
x1, y1 = c1, c2
# for the space bar
x1, y1 = int((row3_key_width - row4_key_width) / 2) + x1, y1 + key_width
x2, y2 = 5 * key_width + x1, key_width + y1
c1, c2 = x1, y1
c = 0
keys = " "
for i in range(len(keys)):
row_keys.append([keys[c], (x1, y1), (x2, y2), (int((x2+x1)/2) - 5, int((y2+y1)/2) + 10)])
x1 += key_width
x2 += key_width
c += 1
x1, y1 = c1, c2
return row_keys
def do_keypress(img, center, row_keys_points):
global s
# this fuction presses a key and marks the pressed key with blue color
for row in row_keys_points:
arr1 = list(npy.int0(npy.array(center) >= npy.array(row[1]))) # center of the contour has greater value than the top left corner point of a key
arr2 = list(npy.int0(npy.array(center) <= npy.array(row[2]))) # center of the contour has less value than the bottom right corner point of a key
if arr1 == [1, 1] and arr2 == [1, 1]:
if(row[0][0]=='$'):
format_to_file()
exit(0)
gui.press(row[0])
print(row[0][0])
s+=row[0][0]
cv2.fillConvexPoly(img, npy.array([npy.array(row[1]), \
npy.array([row[1][0], row[2][1]]), \
npy.array(row[2]), \
npy.array([row[2][0], row[1][1]])]), \
(255, 0, 0))
return img
def main():
row_keys_points = get_keys()
new_area, old_area = 0, 0
c, c2 = 0, 0 # c stores the number of iterations for calculating the difference b/w present area and previous area
# c2 stores the number of iterations for calculating the difference b/w present center and previous center
flag_keypress = False # if a key is pressed then this flag is True
while True:
img = cam.read()[1]
img = cv2.flip(img, 1)
imgHSV = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
mask = cv2.inRange(imgHSV, hsv_lower, hsv_upper)
blur = cv2.medianBlur(mask, 15)
blur = cv2.GaussianBlur(blur , (5,5), 0)
thresh = cv2.threshold(blur, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)[1]
contours = cv2.findContours(thresh.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)[1]
if len(contours) > 0:
cnt = max(contours, key = cv2.contourArea)
if cv2.contourArea(cnt) > 350:
# draw a rectangle and a center
rect = cv2.minAreaRect(cnt)
center = list(rect[0])
box = cv2.boxPoints(rect)
box = npy.int0(box)
cv2.circle(img, tuple(npy.int0(center)), 2, (0, 255, 0), 2)
cv2.drawContours(img,[box],0,(0,0,255),2)
# calculation of difference of area and center
new_area = cv2.contourArea(cnt)
new_center = npy.int0(center)
if c == 0:
old_area = new_area
c += 1
diff_area = 0
if c > 3: # after every 3rd iteration difference of area is calculated
diff_area = new_area - old_area
c = 0
if c2 == 0:
old_center = new_center
c2 += 1
diff_center = npy.array([0, 0])
if c2 > 5: # after every 5th iteration difference of center is claculated
diff_center = new_center - old_center
c2 = 0
# setting some thresholds
center_threshold = 10
area_threshold = 200
if abs(diff_center[0]) < center_threshold or abs(diff_center[1]) < center_threshold:
if diff_area > area_threshold and flag_keypress == False:
# time.sleep
img = do_keypress(img, new_center, row_keys_points)
flag_keypress = True
elif diff_area < -(area_threshold) and flag_keypress == True:
flag_keypress = False
else:
flag_keypress = False
else:
flag_keypress = False
# displaying the keyboard
for key in row_keys_points:
cv2.putText(img, key[0], key[3], cv2.FONT_HERSHEY_DUPLEX, 1, (0, 0, 0))
cv2.rectangle(img, key[1], key[2], (0, 0, 0), thickness = 3)
cv2.imshow("img", img)
if cv2.waitKey(1) == ord('q'):
break
cam.release()
cv2.destroyAllWindows()
main()