17
17
18
18
using namespace cldnn ;
19
19
20
+ namespace {
21
+ void eliminate_pad_for_onednn_impl (program& p, program_node& node) {
22
+ // Padded offsets aren't supported by onednn kernels
23
+ bool has_paddings = false ;
24
+ bool use_onednn = false ;
25
+ for (size_t idx = 0 ; idx < node.get_dependencies ().size (); idx++) {
26
+ const auto & input = node.get_dependency (idx);
27
+ if (!input.is_in_data_flow () || input.is_constant ())
28
+ continue ;
29
+ if (input.get_output_layout ().data_padding ) {
30
+ has_paddings = true ;
31
+ break ;
32
+ }
33
+ }
34
+
35
+ if (has_paddings) {
36
+ // oneDNN doesn't support padded memory, so we check that onednn impl can be used with dropped paddings
37
+ use_onednn = test_no_input_pad<bool >(node, [](const program_node& node) {
38
+ return node.type ()->has_impl_for (node, impl_types::onednn);
39
+ });
40
+ }
41
+
42
+ if (use_onednn) {
43
+ for (size_t idx = 0 ; idx < node.get_dependencies ().size (); idx++) {
44
+ auto node_and_port = node.get_dependency_with_port (idx);
45
+ auto & input = *node_and_port.first ;
46
+ auto port = node_and_port.second ;
47
+ if (!input.is_in_data_flow () || input.is_constant ())
48
+ continue ;
49
+
50
+ auto & in_layout = input.get_output_layout (false , port);
51
+ auto & in_padding = in_layout.data_padding ;
52
+ if (static_cast <bool >(in_padding)) {
53
+ bool spatial_padding = false ;
54
+ for (size_t i = 0 ; i < in_layout.get_spatial_rank (); ++i) {
55
+ spatial_padding |= (in_padding._lower_size [2 + i] != 0 );
56
+ }
57
+ for (size_t i = 0 ; i < in_layout.get_spatial_rank (); ++i) {
58
+ spatial_padding |= (in_padding._upper_size [2 + i] != 0 );
59
+ }
60
+
61
+ bool feature_padding = false ;
62
+ feature_padding |= (in_padding._lower_size [1 ] != 0 );
63
+ feature_padding |= (in_padding._upper_size [1 ] != 0 );
64
+
65
+ bool batch_padding = false ;
66
+ batch_padding |= (in_padding._lower_size [0 ] != 0 );
67
+ batch_padding |= (in_padding._upper_size [0 ] != 0 );
68
+
69
+ if (batch_padding && !feature_padding && !spatial_padding) {
70
+ batch_padding = false ;
71
+ }
72
+
73
+ if (spatial_padding || batch_padding) {
74
+ cldnn::layout layout_wo_padding = in_layout;
75
+ layout_wo_padding.data_padding = cldnn::padding{};
76
+ layout_wo_padding.data_padding ._lower_size [1 ] = in_layout.data_padding ._lower_size [1 ];
77
+ layout_wo_padding.data_padding ._upper_size [1 ] = in_layout.data_padding ._upper_size [1 ];
78
+ if (input.is_type <reorder>()) {
79
+ input.set_output_padding (padding ());
80
+ input.set_output_layout (layout_wo_padding, false , port);
81
+ } else {
82
+ auto new_reorder = std::make_shared<reorder>(input.id () + " _padding_reorder_" + node.id (), input.id (), layout_wo_padding);
83
+ auto & new_reorder_node = p.get_or_create (new_reorder);
84
+ p.add_intermediate (new_reorder_node, node, idx);
85
+ new_reorder_node.recalc_output_layouts (false );
86
+ }
87
+ } else {
88
+ return ;
89
+ }
90
+ }
91
+ }
92
+
93
+ return ;
94
+ }
95
+ }
96
+ } // namespace
97
+
20
98
/*
21
99
This pass checks if data formats (layouts) of output/input in hidden layers match.
22
100
If not than required reorder is added to the network.
@@ -50,6 +128,36 @@ void add_required_reorders::add_reorder(program& p, program_node* node, program_
50
128
throw std::runtime_error (" Internal Error: container index out of range exception." );
51
129
}
52
130
p.add_intermediate (new_reorder_node, *usr, idx);
131
+ new_reorder_node.recalc_output_layouts (false );
132
+ }
133
+
134
+ bool add_required_reorders::test_format (cldnn::program_node& node, format requested_format) {
135
+ for (size_t i = 0 ; i < node.get_outputs_count (); i++) {
136
+ auto out_layout = node.get_output_layout (false , i);
137
+ out_layout.format = requested_format;
138
+ node.set_output_layout (out_layout, false , i);
139
+ }
140
+
141
+ for (size_t i = 0 ; i < node.get_dependencies ().size (); i++) {
142
+ const auto & dep_with_port = node.get_dependency_with_port (i);
143
+ auto & dep = dep_with_port.first ;
144
+
145
+ auto current_format = dep->get_output_layout (false , dep_with_port.second ).format ;
146
+
147
+ if (format::is_weights_format (current_format))
148
+ continue ;
149
+
150
+ if (dep->is_type <reorder>()) {
151
+ auto & port = dep_with_port.second ;
152
+ auto new_layout = dep->get_output_layout (false , port);
153
+ new_layout.format = requested_format;
154
+ dep->set_output_layout (new_layout, false , port);
155
+ } else if (current_format != requested_format) {
156
+ add_reorder (node.get_program (), dep_with_port.first , &node, true );
157
+ }
158
+ }
159
+
160
+ return node.type ()->has_impl_for (node, impl_types::any, shape_types::any);
53
161
}
54
162
55
163
void add_required_reorders::run (program& p) {
@@ -153,57 +261,10 @@ void add_required_reorders::run(program& p) {
153
261
}
154
262
}
155
263
156
- if (usr->type ()->does_an_implementation_exist (*usr)) {
157
- if (usr->get_preferred_impl_type () != impl_types::onednn) {
158
- continue ;
159
- } else {
160
- // oneDNN doesn't support padded memory, so add reorder directly if needed
161
- for (size_t idx = 0 ; idx < usr->get_dependencies ().size (); idx++) {
162
- auto & input = usr->get_dependency (idx);
163
- if (!input.is_in_data_flow () || input.is_constant ())
164
- continue ;
165
-
166
- auto & in_layout = input.get_output_layout ();
167
- auto & in_padding = in_layout.data_padding ;
168
- if (static_cast <bool >(in_padding)) {
169
- bool spatial_padding = false ;
170
- for (size_t i = 0 ; i < in_layout.get_spatial_rank (); ++i) {
171
- spatial_padding |= (in_padding._lower_size [2 + i] != 0 );
172
- }
173
- for (size_t i = 0 ; i < in_layout.get_spatial_rank (); ++i) {
174
- spatial_padding |= (in_padding._upper_size [2 + i] != 0 );
175
- }
176
-
177
- bool feature_padding = false ;
178
- feature_padding |= (in_padding._lower_size [1 ] != 0 );
179
- feature_padding |= (in_padding._upper_size [1 ] != 0 );
180
-
181
- bool batch_padding = false ;
182
- batch_padding |= (in_padding._lower_size [0 ] != 0 );
183
- batch_padding |= (in_padding._upper_size [0 ] != 0 );
184
-
185
- if (batch_padding && !feature_padding && !spatial_padding) {
186
- batch_padding = false ;
187
- }
188
-
189
- if (spatial_padding || batch_padding) {
190
- cldnn::layout layout_padding = input.get_output_layout ();
191
- cldnn::layout layout_wo_padding = input.get_output_layout ();
192
- layout_wo_padding.data_padding = cldnn::padding{};
193
- layout_wo_padding.data_padding ._lower_size [1 ] = layout_padding.data_padding ._lower_size [1 ];
194
- layout_wo_padding.data_padding ._upper_size [1 ] = layout_padding.data_padding ._upper_size [1 ];
195
- auto new_reorder = std::make_shared<reorder>(input.id () + " _padding_reorder_" + usr->id (), input.id (), layout_wo_padding);
196
- auto & new_reorder_node = p.get_or_create (new_reorder);
197
- p.add_intermediate (new_reorder_node, *usr, idx);
198
- new_reorder_node.recalc_output_layouts (false );
199
- } else {
200
- continue ;
201
- }
202
- }
203
- }
204
- continue ;
205
- }
206
- }
264
+ eliminate_pad_for_onednn_impl (p, *usr);
265
+
266
+ if (usr->type ()->has_impl_for (*usr))
267
+ continue ;
207
268
208
269
bool correct_layout_selected = false ;
209
270
bool weights_data = (usr->is_type <convolution>() || usr->is_type <deconvolution>() || usr->is_type <fully_connected>());
@@ -221,19 +282,11 @@ void add_required_reorders::run(program& p) {
221
282
original_layout.data_type ,
222
283
node.first ->get_output_layout ().format );
223
284
usr->set_output_layout (current_layout, false );
224
- if (usr->type ()->does_possible_implementation_exist (*usr)) {
285
+ if (usr->type ()->has_impl_for (*usr)) {
225
286
correct_layout_selected = true ;
226
287
break ;
227
288
}
228
289
}
229
-
230
- OPENVINO_ASSERT (correct_layout_selected,
231
- " [GPU] No layout format available for " , usr->id (), " , impl_type: " , usr->get_preferred_impl_type (),
232
- " (format: " , original_layout.format .to_string (),
233
- " , data_type: " , ov::element::Type (original_layout.data_type ), " ) " ,
234
- " compatible with " , node.first ->id (),
235
- " (format: " , node.first ->get_output_layout ().format .to_string (),
236
- " , data_type: " , ov::element::Type (node.first ->get_output_layout ().data_type ), " )" );
237
290
}
238
291
}
239
292
@@ -254,53 +307,23 @@ void add_required_reorders::run(program& p) {
254
307
preferred_layout_formats.push_back (cldnn::format::byxf);
255
308
}
256
309
257
- if (original_layout.is_dynamic () && usr->type ()->does_dynamic_implementation_exist (*usr)) {
310
+ if (original_layout.is_dynamic () && usr->type ()->has_impl_for (*usr, shape_types::dynamic_shape )) {
258
311
correct_layout_selected = true ;
259
312
}
260
313
261
- if (usr->get_preferred_impl_type () == impl_types::onednn) {
262
- usr->set_preferred_impl_type (impl_types::ocl);
263
- usr->set_output_layout (original_layout, false );
264
- if (usr->type ()->does_possible_implementation_exist (*usr)) {
265
- correct_layout_selected = true ;
266
- }
267
- }
268
-
269
314
if (!correct_layout_selected) {
270
315
for (auto new_layout_format : preferred_layout_formats) {
271
- layout current_layout (original_layout.get_partial_shape (), original_layout.data_type , new_layout_format);
272
- usr->set_output_layout (current_layout, false );
273
- if (usr->type ()->does_possible_implementation_exist (*usr)) {
316
+ if (test_format (*usr, new_layout_format)) {
274
317
correct_layout_selected = true ;
275
318
break ;
276
319
}
277
320
}
278
321
}
279
322
}
280
323
281
- // layout is selected now add required reorders
282
- auto dep_itr = usr->get_dependencies ().begin ();
283
- while (dep_itr != usr->get_dependencies ().end ()) {
284
- auto node = *dep_itr++;
285
- // do not add a reorder if usr or node are reorders or does not belong to data_flow
286
- if (!usr->is_type <reorder>() && node.first ->is_in_data_flow ()) {
287
- if (usr->is_type <convert_color>()) {
288
- auto reorder_prim = node.first ->as <reorder>().get_primitive ();
289
- if (reorder_prim->has_surface_input ())
290
- continue ;
291
- }
292
-
293
- if (usr->get_output_layout () != node.first ->get_output_layout ()) {
294
- // Preserve original data type to prevent Convolution input data type from changing
295
- // in the following sequence: Node(U8, unsupported format) -> Conv(FP16, bfyx).
296
- // Without this condition, inserted reorder will change Conv's input to FP16, instead of
297
- // expected U8 format.
298
- bool keep_original_dt = false ;
299
- if (usr->is_type <convolution>())
300
- keep_original_dt = true ;
301
- add_reorder (p, node.first , usr, keep_original_dt);
302
- }
303
- }
304
- }
324
+ OPENVINO_ASSERT (correct_layout_selected,
325
+ " [GPU] No layout format available for " , usr->id (), " , impl_type: " , usr->get_preferred_impl_type (),
326
+ " (format: " , original_layout.format .to_string (),
327
+ " , data_type: " , ov::element::Type (original_layout.data_type ), " ) " );
305
328
}
306
329
}
0 commit comments