Skip to content

Commit 3a99145

Browse files
[GPU] Extended impls factory and onednn impls update (#25680)
### Details: - Changed impl registration and selection logic. All impls now are registered in prioritized list along with validation function and optional custom callback. - Added impls factory interface which extends existing impl creation capability with validation method + query for preferred formats. - Move onednn-specific logic from layout optimizer to onednn-based impls. - Move onednn descriptors creation to cpp file to avoid further usage in other components - Removed unused eltwise_onednn impl files
1 parent 73d2a44 commit 3a99145

File tree

135 files changed

+4505
-2554
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

135 files changed

+4505
-2554
lines changed

src/plugins/intel_gpu/include/intel_gpu/runtime/utils.hpp

+13
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,19 @@ template <typename T>
192192
inline bool one_of(const T& val, const std::vector<T>& vec) {
193193
return std::any_of(vec.begin(), vec.end(), [&val](const T& v) { return v == val; });
194194
}
195+
template <typename T, typename... T1>
196+
inline bool one_of(const T& val, T1... args) {
197+
return one_of(val, std::vector<T>{args...});
198+
}
199+
200+
template <typename T, typename P>
201+
constexpr bool everyone_is(T val, P item) {
202+
return val == item;
203+
}
204+
template <typename T, typename P, typename... Args>
205+
constexpr bool everyone_is(T val, P item, Args... item_others) {
206+
return val == item && everyone_is(val, item_others...);
207+
}
195208

196209
// Helpers to get string for types that have operator<< defined
197210
template <typename T>

src/plugins/intel_gpu/src/graph/graph_optimizer/add_required_reorders.cpp

+119-96
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,84 @@
1717

1818
using namespace cldnn;
1919

20+
namespace {
21+
void eliminate_pad_for_onednn_impl(program& p, program_node& node) {
22+
// Padded offsets aren't supported by onednn kernels
23+
bool has_paddings = false;
24+
bool use_onednn = false;
25+
for (size_t idx = 0; idx < node.get_dependencies().size(); idx++) {
26+
const auto& input = node.get_dependency(idx);
27+
if (!input.is_in_data_flow() || input.is_constant())
28+
continue;
29+
if (input.get_output_layout().data_padding) {
30+
has_paddings = true;
31+
break;
32+
}
33+
}
34+
35+
if (has_paddings) {
36+
// oneDNN doesn't support padded memory, so we check that onednn impl can be used with dropped paddings
37+
use_onednn = test_no_input_pad<bool>(node, [](const program_node& node) {
38+
return node.type()->has_impl_for(node, impl_types::onednn);
39+
});
40+
}
41+
42+
if (use_onednn) {
43+
for (size_t idx = 0; idx < node.get_dependencies().size(); idx++) {
44+
auto node_and_port = node.get_dependency_with_port(idx);
45+
auto& input = *node_and_port.first;
46+
auto port = node_and_port.second;
47+
if (!input.is_in_data_flow() || input.is_constant())
48+
continue;
49+
50+
auto& in_layout = input.get_output_layout(false, port);
51+
auto& in_padding = in_layout.data_padding;
52+
if (static_cast<bool>(in_padding)) {
53+
bool spatial_padding = false;
54+
for (size_t i = 0; i < in_layout.get_spatial_rank(); ++i) {
55+
spatial_padding |= (in_padding._lower_size[2 + i] != 0);
56+
}
57+
for (size_t i = 0; i < in_layout.get_spatial_rank(); ++i) {
58+
spatial_padding |= (in_padding._upper_size[2 + i] != 0);
59+
}
60+
61+
bool feature_padding = false;
62+
feature_padding |= (in_padding._lower_size[1] != 0);
63+
feature_padding |= (in_padding._upper_size[1] != 0);
64+
65+
bool batch_padding = false;
66+
batch_padding |= (in_padding._lower_size[0] != 0);
67+
batch_padding |= (in_padding._upper_size[0] != 0);
68+
69+
if (batch_padding && !feature_padding && !spatial_padding) {
70+
batch_padding = false;
71+
}
72+
73+
if (spatial_padding || batch_padding) {
74+
cldnn::layout layout_wo_padding = in_layout;
75+
layout_wo_padding.data_padding = cldnn::padding{};
76+
layout_wo_padding.data_padding._lower_size[1] = in_layout.data_padding._lower_size[1];
77+
layout_wo_padding.data_padding._upper_size[1] = in_layout.data_padding._upper_size[1];
78+
if (input.is_type<reorder>()) {
79+
input.set_output_padding(padding());
80+
input.set_output_layout(layout_wo_padding, false, port);
81+
} else {
82+
auto new_reorder = std::make_shared<reorder>(input.id() + "_padding_reorder_" + node.id(), input.id(), layout_wo_padding);
83+
auto& new_reorder_node = p.get_or_create(new_reorder);
84+
p.add_intermediate(new_reorder_node, node, idx);
85+
new_reorder_node.recalc_output_layouts(false);
86+
}
87+
} else {
88+
return;
89+
}
90+
}
91+
}
92+
93+
return;
94+
}
95+
}
96+
} // namespace
97+
2098
/*
2199
This pass checks if data formats (layouts) of output/input in hidden layers match.
22100
If not than required reorder is added to the network.
@@ -50,6 +128,36 @@ void add_required_reorders::add_reorder(program& p, program_node* node, program_
50128
throw std::runtime_error("Internal Error: container index out of range exception.");
51129
}
52130
p.add_intermediate(new_reorder_node, *usr, idx);
131+
new_reorder_node.recalc_output_layouts(false);
132+
}
133+
134+
bool add_required_reorders::test_format(cldnn::program_node& node, format requested_format) {
135+
for (size_t i = 0; i < node.get_outputs_count(); i++) {
136+
auto out_layout = node.get_output_layout(false, i);
137+
out_layout.format = requested_format;
138+
node.set_output_layout(out_layout, false, i);
139+
}
140+
141+
for (size_t i = 0; i < node.get_dependencies().size(); i++) {
142+
const auto& dep_with_port = node.get_dependency_with_port(i);
143+
auto& dep = dep_with_port.first;
144+
145+
auto current_format = dep->get_output_layout(false, dep_with_port.second).format;
146+
147+
if (format::is_weights_format(current_format))
148+
continue;
149+
150+
if (dep->is_type<reorder>()) {
151+
auto& port = dep_with_port.second;
152+
auto new_layout = dep->get_output_layout(false, port);
153+
new_layout.format = requested_format;
154+
dep->set_output_layout(new_layout, false, port);
155+
} else if (current_format != requested_format) {
156+
add_reorder(node.get_program(), dep_with_port.first, &node, true);
157+
}
158+
}
159+
160+
return node.type()->has_impl_for(node, impl_types::any, shape_types::any);
53161
}
54162

55163
void add_required_reorders::run(program& p) {
@@ -153,57 +261,10 @@ void add_required_reorders::run(program& p) {
153261
}
154262
}
155263

156-
if (usr->type()->does_an_implementation_exist(*usr)) {
157-
if (usr->get_preferred_impl_type() != impl_types::onednn) {
158-
continue;
159-
} else {
160-
// oneDNN doesn't support padded memory, so add reorder directly if needed
161-
for (size_t idx = 0; idx < usr->get_dependencies().size(); idx++) {
162-
auto& input = usr->get_dependency(idx);
163-
if (!input.is_in_data_flow() || input.is_constant())
164-
continue;
165-
166-
auto& in_layout = input.get_output_layout();
167-
auto& in_padding = in_layout.data_padding;
168-
if (static_cast<bool>(in_padding)) {
169-
bool spatial_padding = false;
170-
for (size_t i = 0; i < in_layout.get_spatial_rank(); ++i) {
171-
spatial_padding |= (in_padding._lower_size[2 + i] != 0);
172-
}
173-
for (size_t i = 0; i < in_layout.get_spatial_rank(); ++i) {
174-
spatial_padding |= (in_padding._upper_size[2 + i] != 0);
175-
}
176-
177-
bool feature_padding = false;
178-
feature_padding |= (in_padding._lower_size[1] != 0);
179-
feature_padding |= (in_padding._upper_size[1] != 0);
180-
181-
bool batch_padding = false;
182-
batch_padding |= (in_padding._lower_size[0] != 0);
183-
batch_padding |= (in_padding._upper_size[0] != 0);
184-
185-
if (batch_padding && !feature_padding && !spatial_padding) {
186-
batch_padding = false;
187-
}
188-
189-
if (spatial_padding || batch_padding) {
190-
cldnn::layout layout_padding = input.get_output_layout();
191-
cldnn::layout layout_wo_padding = input.get_output_layout();
192-
layout_wo_padding.data_padding = cldnn::padding{};
193-
layout_wo_padding.data_padding._lower_size[1] = layout_padding.data_padding._lower_size[1];
194-
layout_wo_padding.data_padding._upper_size[1] = layout_padding.data_padding._upper_size[1];
195-
auto new_reorder = std::make_shared<reorder>(input.id() + "_padding_reorder_" + usr->id(), input.id(), layout_wo_padding);
196-
auto& new_reorder_node = p.get_or_create(new_reorder);
197-
p.add_intermediate(new_reorder_node, *usr, idx);
198-
new_reorder_node.recalc_output_layouts(false);
199-
} else {
200-
continue;
201-
}
202-
}
203-
}
204-
continue;
205-
}
206-
}
264+
eliminate_pad_for_onednn_impl(p, *usr);
265+
266+
if (usr->type()->has_impl_for(*usr))
267+
continue;
207268

208269
bool correct_layout_selected = false;
209270
bool weights_data = (usr->is_type<convolution>() || usr->is_type<deconvolution>() || usr->is_type<fully_connected>());
@@ -221,19 +282,11 @@ void add_required_reorders::run(program& p) {
221282
original_layout.data_type,
222283
node.first->get_output_layout().format);
223284
usr->set_output_layout(current_layout, false);
224-
if (usr->type()->does_possible_implementation_exist(*usr)) {
285+
if (usr->type()->has_impl_for(*usr)) {
225286
correct_layout_selected = true;
226287
break;
227288
}
228289
}
229-
230-
OPENVINO_ASSERT(correct_layout_selected,
231-
"[GPU] No layout format available for ", usr->id(), ", impl_type: ", usr->get_preferred_impl_type(),
232-
" (format: ", original_layout.format.to_string(),
233-
", data_type: ", ov::element::Type(original_layout.data_type), ") ",
234-
"compatible with ", node.first->id(),
235-
" (format: ", node.first->get_output_layout().format.to_string(),
236-
", data_type: ", ov::element::Type(node.first->get_output_layout().data_type), ")");
237290
}
238291
}
239292

@@ -254,53 +307,23 @@ void add_required_reorders::run(program& p) {
254307
preferred_layout_formats.push_back(cldnn::format::byxf);
255308
}
256309

257-
if (original_layout.is_dynamic() && usr->type()->does_dynamic_implementation_exist(*usr)) {
310+
if (original_layout.is_dynamic() && usr->type()->has_impl_for(*usr, shape_types::dynamic_shape)) {
258311
correct_layout_selected = true;
259312
}
260313

261-
if (usr->get_preferred_impl_type() == impl_types::onednn) {
262-
usr->set_preferred_impl_type(impl_types::ocl);
263-
usr->set_output_layout(original_layout, false);
264-
if (usr->type()->does_possible_implementation_exist(*usr)) {
265-
correct_layout_selected = true;
266-
}
267-
}
268-
269314
if (!correct_layout_selected) {
270315
for (auto new_layout_format : preferred_layout_formats) {
271-
layout current_layout(original_layout.get_partial_shape(), original_layout.data_type, new_layout_format);
272-
usr->set_output_layout(current_layout, false);
273-
if (usr->type()->does_possible_implementation_exist(*usr)) {
316+
if (test_format(*usr, new_layout_format)) {
274317
correct_layout_selected = true;
275318
break;
276319
}
277320
}
278321
}
279322
}
280323

281-
// layout is selected now add required reorders
282-
auto dep_itr = usr->get_dependencies().begin();
283-
while (dep_itr != usr->get_dependencies().end()) {
284-
auto node = *dep_itr++;
285-
// do not add a reorder if usr or node are reorders or does not belong to data_flow
286-
if (!usr->is_type<reorder>() && node.first->is_in_data_flow()) {
287-
if (usr->is_type<convert_color>()) {
288-
auto reorder_prim = node.first->as<reorder>().get_primitive();
289-
if (reorder_prim->has_surface_input())
290-
continue;
291-
}
292-
293-
if (usr->get_output_layout() != node.first->get_output_layout()) {
294-
// Preserve original data type to prevent Convolution input data type from changing
295-
// in the following sequence: Node(U8, unsupported format) -> Conv(FP16, bfyx).
296-
// Without this condition, inserted reorder will change Conv's input to FP16, instead of
297-
// expected U8 format.
298-
bool keep_original_dt = false;
299-
if (usr->is_type<convolution>())
300-
keep_original_dt = true;
301-
add_reorder(p, node.first, usr, keep_original_dt);
302-
}
303-
}
304-
}
324+
OPENVINO_ASSERT(correct_layout_selected,
325+
"[GPU] No layout format available for ", usr->id(), ", impl_type: ", usr->get_preferred_impl_type(),
326+
" (format: ", original_layout.format.to_string(),
327+
", data_type: ", ov::element::Type(original_layout.data_type), ") ");
305328
}
306329
}

0 commit comments

Comments
 (0)