[FORK][FEATURE] Migrate legacy post ops and zero points on runtime data pointers

ONEDNN 3.2 migration squashed commits:
    - fix depthwise nwc conv
    - Fix deconv 3D post OPs segment fault issue. (openvinotoolkit#130)
    - fix in avx2 conv+fakequant post ops +nxc last channel wrong result (openvinotoolkit#128)
    - Luwei/fix deconv 3d postops bug (openvinotoolkit#136)
        -- Fix the deconv fused with depthwise issue in cpuFuncTests
        -- Switch to use jit_uni_depthwise_injector API.
        -- Fix potential conflicts in registers and YMM.
        -- Update with optimization.
    - fix legacyOps with stock src_zero_point in jit_avx512_core_amx
    - Fix incorrect offset to rsp
    - Preserve bf16emu scratch register when conflict with legacy post ops
    - fix per-OC legacyPostOps for jit_avx512_dw_conv_fwd_kernel_bf16
    - Fix segment fault caused by dest scale.

ONEDNN 3.5 migration squshed commmits:
[FIX][FORK][FEATURE] Introduced Depthwise and Quantization post ops

[FORK] [FIX] Fix legacy zero point issue on AVX2 + AVX512.
[FEATURE]Migrate legacy post ops and zero points on runtime data pointers

[FORK][FIX] jit_uni_dw_conv_row_f32: fixed post ops start idx
[FEATURE] Migrate legacy post ops and zero points on runtime data pointers

[Fix] Update the attr checking caused by forked onednn.

[FORK][FIX][x64] Add proper post op checks to gemm_conv is split
particially and squashed.

[FORK][FIX] jit_uni_dw_conv_kernel_f32: fixed register conflict

[FIX] SPlit the removed unused variables

[FIX] fix avx512 bf16 dw stack pointer

[ARM] Fixed legacy post-ops changes for ARM target

Author: luweizhou2016

include/oneapi/dnnl/dnnl.h

@@ -24,6 +24,7 @@
 #include "oneapi/dnnl/dnnl_config.h"
 #include "oneapi/dnnl/dnnl_types.h"
 #include "oneapi/dnnl/dnnl_version.h"
+#include <stdbool.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -519,19 +520,13 @@ dnnl_status_t DNNL_API dnnl_primitive_attr_get_output_compensations(
         const_dnnl_primitive_attr_t attr, int *count, int *mask, const int32_t **compensations);
 
 dnnl_status_t DNNL_API dnnl_primitive_attr_set_output_compensations(
-        dnnl_primitive_attr_t attr, int count, int mask, const int32_t *compensations);
-
-dnnl_status_t DNNL_API dnnl_primitive_attr_get_input_zero_points(
-        const_dnnl_primitive_attr_t attr, int *count, int *mask, const uint8_t **zero_points);
+        dnnl_primitive_attr_t attr, int count, int mask);
 
 dnnl_status_t DNNL_API dnnl_primitive_attr_set_input_zero_points(
-        dnnl_primitive_attr_t attr, int count, int mask, const uint8_t *zero_points);
-
-dnnl_status_t DNNL_API dnnl_primitive_attr_get_weights_zero_points(
-        const_dnnl_primitive_attr_t attr, int *count, int *mask, const float **zero_points);
+        dnnl_primitive_attr_t attr, int count, int mask);
 
 dnnl_status_t DNNL_API dnnl_primitive_attr_set_weights_zero_points(
-        dnnl_primitive_attr_t attr, int count, int mask, const float *zero_points);
+        dnnl_primitive_attr_t attr, int count, int mask);
 
 /// Returns primitive attributes post-ops.
 ///
@@ -739,8 +734,7 @@ dnnl_status_t DNNL_API dnnl_post_ops_get_params_dw(
 ///
 /// The kind of this post operation is #dnnl_convolution.
 dnnl_status_t DNNL_API dnnl_post_ops_append_dw_conv(
-        dnnl_post_ops_t post_ops, int in_h, int in_w, int ker_h, int ker_w, int str_h, int str_w, dnnl_data_type_t in_dt,
-        const float* weights_data, const float* biases_data);
+        dnnl_post_ops_t post_ops, int in_h, int in_w, int ker_h, int ker_w, int str_h, int str_w, dnnl_data_type_t in_dt);
 
 /// Appends a binary post-op.
 ///
@@ -822,14 +816,13 @@ dnnl_status_t DNNL_API dnnl_post_ops_get_params_prelu(
         const_dnnl_post_ops_t post_ops, int index, int *mask);
 
 dnnl_status_t DNNL_API dnnl_post_ops_append_depthwise(
-        dnnl_post_ops_t post_ops, dnnl_alg_kind_t alg,
-        const float* weights_data, const float* biases_data);
+        dnnl_post_ops_t post_ops, dnnl_alg_kind_t alg, size_t offset_size, const size_t* offset);
 
 dnnl_status_t DNNL_API dnnl_post_ops_append_quantization(
         dnnl_post_ops_t post_ops, dnnl_alg_kind_t alg,
-        const void* crop_low, const void* crop_high,
-        const void* input_scale, const void* input_shift,
-        const void* output_scale, const void* output_shift);
+        size_t per_channel_size, const bool* per_channel,
+        size_t all_default_size, const bool* all_default,
+        size_t offset_size, const size_t* offset);
 
 dnnl_status_t DNNL_API dnnl_post_ops_append_binarization(
         dnnl_post_ops_t post_ops, dnnl_alg_kind_t alg, const float* weights_data, const float* output_mask);

include/oneapi/dnnl/dnnl.hpp

@@ -29,6 +29,7 @@
 #include <memory>
 #include <string>
 #include <vector>
+#include <array>
 #include <unordered_map>
 
 #include "oneapi/dnnl/dnnl.h"
@@ -148,6 +149,10 @@ struct primitive : public handle<dnnl_primitive_t> {
         layer_normalization = dnnl_layer_normalization,
         /// A group normalization primitive
         group_normalization = dnnl_group_normalization,
+
+        depthwise = dnnl_depthwise,
+        quantization = dnnl_quantization,
+        binarization = dnnl_binarization,
     };
 
     using handle::handle;
@@ -168,7 +173,7 @@ struct primitive : public handle<dnnl_primitive_t> {
             const std::vector<uint8_t> &cache_blob);
 
     /// Constructs a primitive from a primitive descriptor.
-    ///
+    ///src/common/deconvolution_pd.hpp
     /// @param pd Primitive descriptor.
     primitive(const primitive_desc &pd);
 
@@ -3864,10 +3869,9 @@ struct post_ops : public handle<dnnl_post_ops_t> {
                 "could not append a binary post-op");
     }
 
-    void append_dw_conv(int in_h, int in_w, int ker_h, int ker_w, int str_h, int str_w, dnnl_data_type_t in_dt,
-                        const float* weights_data, const float* biases_data) {
+    void append_dw_conv(int in_h, int in_w, int ker_h, int ker_w, int str_h, int str_w, dnnl_data_type_t in_dt) {
         error::wrap_c_api(dnnl_post_ops_append_dw_conv(get(),
-                                                       in_h, in_w, ker_h, ker_w, str_h, str_w, in_dt, weights_data, biases_data),
+                                                       in_h, in_w, ker_h, ker_w, str_h, str_w, in_dt),
                           "could not append dw conv");
     }
 
@@ -3956,19 +3960,15 @@ struct post_ops : public handle<dnnl_post_ops_t> {
                 "could not get parameters of a binary post-op");
     }
 
-    void append_depthwise(algorithm alg, const float* weights_data,
-            const float* biases_data) {
-        error::wrap_c_api(dnnl_post_ops_append_depthwise(get(),
-                    convert_to_c(alg), weights_data, biases_data),
+    void append_depthwise(algorithm alg, const std::array<size_t, 2>& offset) {
+        error::wrap_c_api(dnnl_post_ops_append_depthwise(get(), convert_to_c(alg), offset.size(), offset.data()),
                 "could not append depthwise");
     }
 
-    void append_quantization(algorithm alg,
-            const void* crop_low, const void* crop_high,
-            const void* input_scale, const void* input_shift,
-            const void* output_scale, const void* output_shift) {
-        error::wrap_c_api(dnnl_post_ops_append_quantization(get(), convert_to_c(alg), crop_low, crop_high,
-                input_scale, input_shift, output_scale, output_shift),
+    void append_quantization(algorithm alg, const std::array<bool, 6>& per_channel, const std::array<bool, 6>& all_default,
+                             const std::array<size_t, 6>& offset) {
+        error::wrap_c_api(dnnl_post_ops_append_quantization(get(), convert_to_c(alg), per_channel.size(), per_channel.data(),
+                all_default.size(), all_default.data(), offset.size(), offset.data()),
                           "could not append quantization");
     }
 
@@ -4226,66 +4226,21 @@ struct primitive_attr : public handle<dnnl_primitive_attr_t> {
                 "could not set zero points primitive attribute");
     }
 
-    void get_output_compensations(int &mask, std::vector<int32_t> &compensations) const
+    void set_output_compensations(dnnl_dim_t count, int mask)
     {
-        int count, c_mask;
-        const int32_t *c_compensations;
-        error::wrap_c_api(dnnl_primitive_attr_get_output_compensations(get(),
-                    &count, &c_mask, &c_compensations),
-                "could not get int output compensations");
-        compensations.resize(count);
-
-        mask = c_mask;
-        for (int c = 0; c < count; ++c)
-            compensations[c] = c_compensations[c];
-    }
-
-    void set_output_compensations(int mask, const std::vector<int32_t> &compensations)
-    {
-        error::wrap_c_api(dnnl_primitive_attr_set_output_compensations(get(),
-                    (int)compensations.size(), mask, &compensations[0]),
+        error::wrap_c_api(dnnl_primitive_attr_set_output_compensations(get(), count, mask),
                 "could not set int output compensations");
     }
 
-    void get_input_zero_points(int &mask, std::vector<uint8_t> &zero_points) const
+    void set_input_zero_points(dnnl_dim_t count, int mask)
     {
-        int count, c_mask;
-        const uint8_t *c_zero_points;
-        error::wrap_c_api(dnnl_primitive_attr_get_input_zero_points(get(),
-                    &count, &c_mask, &c_zero_points),
-                "could not get int input zero_points");
-        zero_points.resize(count);
-
-        mask = c_mask;
-        for (int c = 0; c < count; ++c)
-            zero_points[c] = c_zero_points[c];
-    }
-
-    void set_input_zero_points(int mask, const std::vector<uint8_t> &zero_points)
-    {
-        error::wrap_c_api(dnnl_primitive_attr_set_input_zero_points(get(),
-                    (int)zero_points.size(), mask, &zero_points[0]),
+        error::wrap_c_api(dnnl_primitive_attr_set_input_zero_points(get(), count, mask),
                 "could not set int input zero_points");
     }
 
-    void get_weights_zero_points(int &mask, std::vector<int8_t> &zero_points) const
-    {
-        int count, c_mask;
-        const float *c_zero_points;
-        error::wrap_c_api(dnnl_primitive_attr_get_weights_zero_points(get(),
-                    &count, &c_mask, &c_zero_points),
-                "could not get int weights zero_points");
-        zero_points.resize(count);
-
-        mask = c_mask;
-        for (int c = 0; c < count; ++c)
-            zero_points[c] = c_zero_points[c];
-    }
-
-    void set_weights_zero_points(int mask, const std::vector<float> &zero_points)
+    void set_weights_zero_points(dnnl_dim_t count, int mask)
     {
-        error::wrap_c_api(dnnl_primitive_attr_set_weights_zero_points(get(),
-                    (int)zero_points.size(), mask, &zero_points[0]),
+        error::wrap_c_api(dnnl_primitive_attr_set_weights_zero_points(get(), count, mask),
                 "could not set int weights zero_points");
     }
 

src/common/convolution.cpp

@@ -171,7 +171,10 @@ status_t conv_attr_check(const convolution_desc_t &desc, const engine_t *engine,
         if (is_int8)
             fwd_attr_mask |= smask_t::scales_runtime
                     | smask_t::zero_points_runtime
-                    | smask_t::zero_points_runtime_data_type;
+                    | smask_t::zero_points_runtime_data_type
+                    | smask_t::input_zero_points
+                    | smask_t::output_compensations
+                    | smask_t::weights_zero_points;
 
         VCHECK_CONV_UNIMPL(attr->has_default_values(fwd_attr_mask, dst_dt),
                 VERBOSE_UNSUPPORTED_ATTR);
@@ -208,17 +211,17 @@ status_t conv_attr_check(const convolution_desc_t &desc, const engine_t *engine,
             const auto &po = attr->post_ops_;
             using namespace primitive_kind;
             VCHECK_CONV_UNIMPL(po.has_default_values({binary, eltwise, prelu,
-                                       sum, convolution}),
+                                       sum, convolution, depthwise, quantization}),
                     VERBOSE_UNSUPPORTED_POSTOP);
 
             // Check sum
             VCHECK_CONV_UNIMPL(po.check_sum_consistency(dst_dt, is_int8, true),
                     VERBOSE_UNSUPPORTED_POSTOP);
         }
-    } else {
-        auto bwd_attr_mask = smask_t::fpmath_mode;
-        VCHECK_CONV_UNIMPL(attr->has_default_values(bwd_attr_mask),
-                VERBOSE_UNSUPPORTED_ATTR);
+    // } else {
+    //     auto bwd_attr_mask = smask_t::fpmath_mode;
+    //     VCHECK_CONV_UNIMPL(attr->has_default_values(bwd_attr_mask),
+    //             VERBOSE_UNSUPPORTED_ATTR);
     }
 
     return status::success;

src/common/convolution_pd.hpp

@@ -298,9 +298,8 @@ struct convolution_fwd_pd_t : public convolution_pd_t {
     }
 
     int n_inputs() const override {
-        // todo: [antonvor] uncomment when new behavior of dw convolution fusing from oneDNN 1.6 will be supported
-        return 2 + with_bias() /* + attr_post_op_dw_inputs() */ + n_binary_po_inputs()
-                + n_prelu_po_inputs();
+        return 2 + with_bias() + attr_post_op_dw_inputs() + n_binary_po_inputs()
+                + n_prelu_po_inputs() + n_depthwise_po_inputs() + n_quantization_po_inputs();
     }
 
     int n_outputs() const override { return 1; }
@@ -330,8 +329,7 @@ struct convolution_fwd_pd_t : public convolution_pd_t {
         const auto &po = attr_.post_ops_;
         int conv = po.find(primitive_kind::convolution);
         if (conv == -1) return 0;
-        return po.entry_[conv].depthwise_conv.bias_dt == data_type::undef ? 1
-                                                                          : 2;
+        return 2;
     }
 };
 
@@ -379,7 +377,9 @@ struct convolution_bwd_data_pd_t : public convolution_pd_t {
         return &glob_zero_md;
     }
 
-    int n_inputs() const override { return 2 + with_bias(); }
+    int n_inputs() const override {
+        return 2 + with_bias() + n_depthwise_po_inputs() + n_quantization_po_inputs();
+    }
     int n_outputs() const override { return 1; }
 
     virtual bool support_bias() const { return false; }

src/common/deconvolution_pd.hpp

@@ -245,7 +245,7 @@ struct deconvolution_fwd_pd_t : public deconvolution_pd_t {
     }
 
     int n_inputs() const override {
-        return 2 + with_bias() + n_prelu_po_inputs() + n_binary_po_inputs();
+        return 2 + with_bias() + n_prelu_po_inputs() + n_binary_po_inputs() + n_depthwise_po_inputs() + n_quantization_po_inputs();
     }
     int n_outputs() const override { return 1; }
 

src/common/inner_product.cpp

@@ -125,7 +125,7 @@ status_t ip_attr_check(const inner_product_desc_t &desc, const engine_t *engine,
             is_int8 = is_int8
                     || utils::one_of(dst_dt, data_type::s8, data_type::u8,
                             data_type::s32);
-        if (is_int8) fwd_attr_mask |= smask_t::scales_runtime;
+        if (is_int8) fwd_attr_mask |= smask_t::scales_runtime | smask_t::zero_points_runtime;
 
         VCHECK_IP_UNIMPL(attr->has_default_values(fwd_attr_mask, dst_dt),
                 VERBOSE_UNSUPPORTED_ATTR);

src/common/pooling.cpp

@@ -151,7 +151,7 @@ status_t pooling_attr_check(const pooling_desc_t &desc, const engine_t *engine,
         if (!attr->post_ops_.has_default_values()) {
             const auto &po = attr->post_ops_;
             using namespace primitive_kind;
-            VCHECK_POOLING_IMPL(po.has_default_values({binary, eltwise}),
+            VCHECK_POOLING_IMPL(po.has_default_values({binary, eltwise, quantization}),
                     VERBOSE_UNSUPPORTED_POSTOP);
         }
     } else {

src/common/pooling_pd.hpp

@@ -215,7 +215,7 @@ struct pooling_fwd_pd_t : public pooling_pd_t {
                                                          : &glob_zero_md;
     }
 
-    int n_inputs() const override { return 1 + n_binary_po_inputs(); }
+    int n_inputs() const override { return 1 + n_binary_po_inputs() + n_depthwise_po_inputs() + n_quantization_po_inputs(); }
     int n_outputs() const override {
         return 1 + (!types::is_zero_md(workspace_md()));
     }