[FORK][FEATURE] Migrate legacy post ops and zero points on runtime data pointers

* Fix the deconv fused with depthwise issue in cpuFuncTests

* Switch to use jit_uni_depthwise_injector API.

* Fix potential conflicts in registers and YMM.

* Update with optimization.

* ONEDNN 3.2 migration squashed commits:

- Preserve bf16emu scratch register when conflict with legacy post ops
- fix per-OC legacyPostOps for jit_avx512_dw_conv_fwd_kernel_bf16
- gemm convolution: fixed bias offset
- Fix onednn gemm JIT/gemm  kernel scale.
- Luwei/fix deconv 3d postops bug (#136)
- Fix deconv 3D post OPs segment fault issue. (#130)
- Fix incorrect offset to rsp
- fix depthwise nwc conv
- fix legacyOps with stock src_zero_point in jit_avx512_core_amx
- fix in avx2 conv+fakequant post ops +nxc last channel wrong result (#128)

Author: dmitry-gorokhov

include/oneapi/dnnl/dnnl.h

@@ -24,6 +24,7 @@
 #include "oneapi/dnnl/dnnl_config.h"
 #include "oneapi/dnnl/dnnl_types.h"
 #include "oneapi/dnnl/dnnl_version.h"
+#include <stdbool.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -354,23 +355,14 @@ dnnl_status_t DNNL_API dnnl_primitive_attr_set_scales_mask(
 dnnl_status_t DNNL_API dnnl_primitive_attr_set_zero_points_mask(
         dnnl_primitive_attr_t attr, int arg, int mask);
 
-dnnl_status_t DNNL_API dnnl_primitive_attr_get_output_compensations(
-        const_dnnl_primitive_attr_t attr, int *count, int *mask, const int32_t **compensations);
-
 dnnl_status_t DNNL_API dnnl_primitive_attr_set_output_compensations(
-        dnnl_primitive_attr_t attr, int count, int mask, const int32_t *compensations);
-
-dnnl_status_t DNNL_API dnnl_primitive_attr_get_input_zero_points(
-        const_dnnl_primitive_attr_t attr, int *count, int *mask, const uint8_t **zero_points);
+        dnnl_primitive_attr_t attr, int count, int mask);
 
 dnnl_status_t DNNL_API dnnl_primitive_attr_set_input_zero_points(
-        dnnl_primitive_attr_t attr, int count, int mask, const uint8_t *zero_points);
-
-dnnl_status_t DNNL_API dnnl_primitive_attr_get_weights_zero_points(
-        const_dnnl_primitive_attr_t attr, int *count, int *mask, const float **zero_points);
+        dnnl_primitive_attr_t attr, int count, int mask);
 
 dnnl_status_t DNNL_API dnnl_primitive_attr_set_weights_zero_points(
-        dnnl_primitive_attr_t attr, int count, int mask, const float *zero_points);
+        dnnl_primitive_attr_t attr, int count, int mask);
 
 /// Returns primitive attributes post-ops.
 ///
@@ -578,8 +570,7 @@ dnnl_status_t DNNL_API dnnl_post_ops_get_params_dw(
 ///
 /// The kind of this post operation is #dnnl_convolution.
 dnnl_status_t DNNL_API dnnl_post_ops_append_dw_conv(
-        dnnl_post_ops_t post_ops, int in_h, int in_w, int ker_h, int ker_w, int str_h, int str_w, dnnl_data_type_t in_dt,
-        const float* weights_data, const float* biases_data);
+        dnnl_post_ops_t post_ops, int in_h, int in_w, int ker_h, int ker_w, int str_h, int str_w, dnnl_data_type_t in_dt);
 
 /// Appends a binary post-op.
 ///
@@ -661,14 +652,13 @@ dnnl_status_t DNNL_API dnnl_post_ops_get_params_prelu(
         const_dnnl_post_ops_t post_ops, int index, int *mask);
 
 dnnl_status_t DNNL_API dnnl_post_ops_append_depthwise(
-        dnnl_post_ops_t post_ops, dnnl_alg_kind_t alg,
-        const float* weights_data, const float* biases_data);
+        dnnl_post_ops_t post_ops, dnnl_alg_kind_t alg, size_t offset_size, const size_t* offset);
 
 dnnl_status_t DNNL_API dnnl_post_ops_append_quantization(
         dnnl_post_ops_t post_ops, dnnl_alg_kind_t alg,
-        const void* crop_low, const void* crop_high,
-        const void* input_scale, const void* input_shift,
-        const void* output_scale, const void* output_shift);
+        size_t per_channel_size, const bool* per_channel,
+        size_t all_default_size, const bool* all_default,
+        size_t offset_size, const size_t* offset);
 
 dnnl_status_t DNNL_API dnnl_post_ops_append_binarization(
         dnnl_post_ops_t post_ops, dnnl_alg_kind_t alg, const float* weights_data, const float* output_mask);

include/oneapi/dnnl/dnnl.hpp

@@ -29,6 +29,7 @@
 #include <memory>
 #include <string>
 #include <vector>
+#include <array>
 #include <unordered_map>
 
 #include "oneapi/dnnl/dnnl.h"
@@ -146,6 +147,10 @@ struct primitive : public handle<dnnl_primitive_t> {
         softmax = dnnl_softmax,
         /// A layer normalization primitive.
         layer_normalization = dnnl_layer_normalization,
+
+        depthwise = dnnl_depthwise,
+        quantization = dnnl_quantization,
+        binarization = dnnl_binarization,
     };
 
     using handle::handle;
@@ -166,7 +171,7 @@ struct primitive : public handle<dnnl_primitive_t> {
             const std::vector<uint8_t> &cache_blob);
 
     /// Constructs a primitive from a primitive descriptor.
-    ///
+    ///src/common/deconvolution_pd.hpp
     /// @param pd Primitive descriptor.
     primitive(const primitive_desc &pd);
 
@@ -3594,10 +3599,9 @@ struct post_ops : public handle<dnnl_post_ops_t> {
                 "could not append a binary post-op");
     }
 
-    void append_dw_conv(int in_h, int in_w, int ker_h, int ker_w, int str_h, int str_w, dnnl_data_type_t in_dt,
-                        const float* weights_data, const float* biases_data) {
+    void append_dw_conv(int in_h, int in_w, int ker_h, int ker_w, int str_h, int str_w, dnnl_data_type_t in_dt) {
         error::wrap_c_api(dnnl_post_ops_append_dw_conv(get(),
-                                                       in_h, in_w, ker_h, ker_w, str_h, str_w, in_dt, weights_data, biases_data),
+                                                       in_h, in_w, ker_h, ker_w, str_h, str_w, in_dt),
                           "could not append dw conv");
     }
 
@@ -3686,19 +3690,15 @@ struct post_ops : public handle<dnnl_post_ops_t> {
                 "could not get parameters of a binary post-op");
     }
 
-    void append_depthwise(algorithm alg, const float* weights_data,
-            const float* biases_data) {
-        error::wrap_c_api(dnnl_post_ops_append_depthwise(get(),
-                    convert_to_c(alg), weights_data, biases_data),
+    void append_depthwise(algorithm alg, const std::array<size_t, 2>& offset) {
+        error::wrap_c_api(dnnl_post_ops_append_depthwise(get(), convert_to_c(alg), offset.size(), offset.data()),
                 "could not append depthwise");
     }
 
-    void append_quantization(algorithm alg,
-            const void* crop_low, const void* crop_high,
-            const void* input_scale, const void* input_shift,
-            const void* output_scale, const void* output_shift) {
-        error::wrap_c_api(dnnl_post_ops_append_quantization(get(), convert_to_c(alg), crop_low, crop_high,
-                input_scale, input_shift, output_scale, output_shift),
+    void append_quantization(algorithm alg, const std::array<bool, 6>& per_channel, const std::array<bool, 6>& all_default,
+                             const std::array<size_t, 6>& offset) {
+        error::wrap_c_api(dnnl_post_ops_append_quantization(get(), convert_to_c(alg), per_channel.size(), per_channel.data(),
+                all_default.size(), all_default.data(), offset.size(), offset.data()),
                           "could not append quantization");
     }
 
@@ -3811,66 +3811,21 @@ struct primitive_attr : public handle<dnnl_primitive_attr_t> {
                 "could not set zero points primitive attribute");
     }
 
-    void get_output_compensations(int &mask, std::vector<int32_t> &compensations) const
+    void set_output_compensations(dnnl_dim_t count, int mask)
     {
-        int count, c_mask;
-        const int32_t *c_compensations;
-        error::wrap_c_api(dnnl_primitive_attr_get_output_compensations(get(),
-                    &count, &c_mask, &c_compensations),
-                "could not get int output compensations");
-        compensations.resize(count);
-
-        mask = c_mask;
-        for (int c = 0; c < count; ++c)
-            compensations[c] = c_compensations[c];
-    }
-
-    void set_output_compensations(int mask, const std::vector<int32_t> &compensations)
-    {
-        error::wrap_c_api(dnnl_primitive_attr_set_output_compensations(get(),
-                    (int)compensations.size(), mask, &compensations[0]),
+        error::wrap_c_api(dnnl_primitive_attr_set_output_compensations(get(), count, mask),
                 "could not set int output compensations");
     }
 
-    void get_input_zero_points(int &mask, std::vector<uint8_t> &zero_points) const
+    void set_input_zero_points(dnnl_dim_t count, int mask)
     {
-        int count, c_mask;
-        const uint8_t *c_zero_points;
-        error::wrap_c_api(dnnl_primitive_attr_get_input_zero_points(get(),
-                    &count, &c_mask, &c_zero_points),
-                "could not get int input zero_points");
-        zero_points.resize(count);
-
-        mask = c_mask;
-        for (int c = 0; c < count; ++c)
-            zero_points[c] = c_zero_points[c];
-    }
-
-    void set_input_zero_points(int mask, const std::vector<uint8_t> &zero_points)
-    {
-        error::wrap_c_api(dnnl_primitive_attr_set_input_zero_points(get(),
-                    (int)zero_points.size(), mask, &zero_points[0]),
+        error::wrap_c_api(dnnl_primitive_attr_set_input_zero_points(get(), count, mask),
                 "could not set int input zero_points");
     }
 
-    void get_weights_zero_points(int &mask, std::vector<int8_t> &zero_points) const
-    {
-        int count, c_mask;
-        const float *c_zero_points;
-        error::wrap_c_api(dnnl_primitive_attr_get_weights_zero_points(get(),
-                    &count, &c_mask, &c_zero_points),
-                "could not get int weights zero_points");
-        zero_points.resize(count);
-
-        mask = c_mask;
-        for (int c = 0; c < count; ++c)
-            zero_points[c] = c_zero_points[c];
-    }
-
-    void set_weights_zero_points(int mask, const std::vector<float> &zero_points)
+    void set_weights_zero_points(dnnl_dim_t count, int mask)
     {
-        error::wrap_c_api(dnnl_primitive_attr_set_weights_zero_points(get(),
-                    (int)zero_points.size(), mask, &zero_points[0]),
+        error::wrap_c_api(dnnl_primitive_attr_set_weights_zero_points(get(), count, mask),
                 "could not set int weights zero_points");
     }
 

src/common/convolution_pd.hpp

@@ -285,9 +285,8 @@ struct convolution_fwd_pd_t : public convolution_pd_t {
     }
 
     int n_inputs() const override {
-        // todo: [antonvor] uncomment when new behavior of dw convolution fusing from oneDNN 1.6 will be supported
-        return 2 + with_bias() /* + attr_post_op_dw_inputs() */ + n_binary_po_inputs()
-                + n_prelu_po_inputs();
+        return 2 + with_bias() + attr_post_op_dw_inputs() + n_binary_po_inputs()
+                + n_prelu_po_inputs() + n_depthwise_po_inputs() + n_quantization_po_inputs();
     }
 
     int n_outputs() const override { return 1; }
@@ -317,8 +316,7 @@ struct convolution_fwd_pd_t : public convolution_pd_t {
         const auto &po = attr_.post_ops_;
         int conv = po.find(primitive_kind::convolution);
         if (conv == -1) return 0;
-        return po.entry_[conv].depthwise_conv.bias_dt == data_type::undef ? 1
-                                                                          : 2;
+        return 2;
     }
 };
 
@@ -366,7 +364,9 @@ struct convolution_bwd_data_pd_t : public convolution_pd_t {
         return &glob_zero_md;
     }
 
-    int n_inputs() const override { return 2 + with_bias(); }
+    int n_inputs() const override {
+        return 2 + with_bias() + n_depthwise_po_inputs() + n_quantization_po_inputs();
+    }
     int n_outputs() const override { return 1; }
 
     virtual bool support_bias() const { return false; }

src/common/deconvolution_pd.hpp

@@ -216,7 +216,7 @@ struct deconvolution_fwd_pd_t : public deconvolution_pd_t {
     }
 
     int n_inputs() const override {
-        return 2 + with_bias() + n_prelu_po_inputs() + n_binary_po_inputs();
+        return 2 + with_bias() + n_prelu_po_inputs() + n_binary_po_inputs() + n_depthwise_po_inputs() + n_quantization_po_inputs();
     }
     int n_outputs() const override { return 1; }
 

src/common/pooling_pd.hpp

@@ -202,7 +202,7 @@ struct pooling_fwd_pd_t : public pooling_pd_t {
                                                          : &glob_zero_md;
     }
 
-    int n_inputs() const override { return 1 + n_binary_po_inputs(); }
+    int n_inputs() const override { return 1 + n_binary_po_inputs() + n_depthwise_po_inputs() + n_quantization_po_inputs(); }
     int n_outputs() const override {
         return 1 + (!types::is_zero_md(workspace_md()));
     }