[FORK][FEATURE] Added support of hsigmoid, round_half_to_even, round_half_away_from_zero elementwise algorithms

Note: hsigmoid implementation probably can be removed since
there is hardsigmoid implementation

Author: a-sidorova

include/oneapi/dnnl/dnnl.hpp

@@ -406,6 +406,12 @@ enum class algorithm {
     eltwise_hardswish = dnnl_eltwise_hardswish,
     /// Elementwise: hardsigmoid
     eltwise_hardsigmoid = dnnl_eltwise_hardsigmoid,
+    /// Elementwise: hsigmoid
+    eltwise_hsigmoid = dnnl_eltwise_hsigmoid,
+    /// Elementwise: round_half_to_even
+    eltwise_round_half_to_even = dnnl_eltwise_round_half_to_even,
+    /// Elementwise: round_half_away_from_zero
+    eltwise_round_half_away_from_zero = dnnl_eltwise_round_half_away_from_zero,
     /// Elementwise: rectified linar unit (ReLU) (dst for backward)
     eltwise_relu_use_dst_for_bwd = dnnl_eltwise_relu_use_dst_for_bwd,
     /// Elementwise: hyperbolic tangent non-linearity (tanh) (dst for backward)

include/oneapi/dnnl/dnnl_types.h

@@ -2088,6 +2088,12 @@ typedef enum {
     dnnl_eltwise_mish,
     /// Eltwise: hardswish
     dnnl_eltwise_hardswish,
+    /// Eltwise: hsigmoid
+    dnnl_eltwise_hsigmoid,
+    /// Eltwise: round_half_to_even
+    dnnl_eltwise_round_half_to_even,
+    /// Eltwise: round_half_away_from_zero
+    dnnl_eltwise_round_half_away_from_zero,
     /// Eltwise: ReLU (dst for backward)
     dnnl_eltwise_relu_use_dst_for_bwd = 0x100,
     /// Eltwise: hyperbolic tangent non-linearity (tanh) (dst for backward)

src/common/c_types_map.hpp

@@ -90,6 +90,9 @@ const alg_kind_t eltwise_gelu_tanh = dnnl_eltwise_gelu_tanh;
 const alg_kind_t eltwise_gelu_erf = dnnl_eltwise_gelu_erf;
 const alg_kind_t eltwise_hardswish = dnnl_eltwise_hardswish;
 const alg_kind_t eltwise_hardsigmoid = dnnl_eltwise_hardsigmoid;
+const alg_kind_t eltwise_hsigmoid = dnnl_eltwise_hsigmoid;
+const alg_kind_t eltwise_round_half_to_even = dnnl_eltwise_round_half_to_even;
+const alg_kind_t eltwise_round_half_away_from_zero = dnnl_eltwise_round_half_away_from_zero;
 const alg_kind_t eltwise_relu_use_dst_for_bwd
         = dnnl_eltwise_relu_use_dst_for_bwd;
 const alg_kind_t eltwise_tanh_use_dst_for_bwd

src/common/dnnl_debug_autogenerated.cpp

@@ -1789,6 +1789,9 @@ const char *dnnl_alg_kind2str(dnnl_alg_kind_t v) {
     if (v == dnnl_eltwise_round) return "eltwise_round";
     if (v == dnnl_eltwise_mish) return "eltwise_mish";
     if (v == dnnl_eltwise_hardswish) return "eltwise_hardswish";
+    if (v == dnnl_eltwise_hsigmoid) return "eltwise_hsigmoid";
+    if (v == dnnl_eltwise_round_half_to_even) return "eltwise_round_half_to_even";
+    if (v == dnnl_eltwise_round_half_away_from_zero) return "eltwise_round_half_away_from_zero";
     if (v == dnnl_eltwise_relu_use_dst_for_bwd) return "eltwise_relu_use_dst_for_bwd";
     if (v == dnnl_eltwise_tanh_use_dst_for_bwd) return "eltwise_tanh_use_dst_for_bwd";
     if (v == dnnl_eltwise_elu_use_dst_for_bwd) return "eltwise_elu_use_dst_for_bwd";

src/common/eltwise.cpp

@@ -57,7 +57,8 @@ status_t eltwise_desc_init(eltwise_desc_t *eltwise_desc, prop_kind_t prop_kind,
     VCHECK_ELTWISE(
             IMPLICATION(!is_fwd, !any_null(diff_src_desc, diff_dst_desc)),
             VERBOSE_NULL_ARG);
-    VCHECK_ELTWISE(IMPLICATION(alg_kind == eltwise_round, is_fwd),
+    VCHECK_ELTWISE(IMPLICATION(one_of(alg_kind, eltwise_round, eltwise_hsigmoid,
+                        eltwise_round_half_away_from_zero, eltwise_round_half_to_even), is_fwd),
             VERBOSE_BAD_PROPKIND);
     VCHECK_ELTWISE(
             IMPLICATION(is_fwd, !memory_desc_wrapper(src_desc).format_any()),

src/common/eltwise_pd.hpp

@@ -158,7 +158,7 @@ struct eltwise_fwd_pd_t : public eltwise_pd_t {
         return one_of(alg, eltwise_relu, eltwise_tanh, eltwise_elu,
                        eltwise_square, eltwise_abs, eltwise_sqrt, eltwise_swish,
                        eltwise_gelu_tanh, eltwise_gelu_erf, eltwise_round,
-                       eltwise_hardswish)
+                       eltwise_hardswish, eltwise_round_half_away_from_zero, eltwise_round_half_to_even)
                 || one_of(alg, eltwise_relu_use_dst_for_bwd,
                         eltwise_tanh_use_dst_for_bwd,
                         eltwise_elu_use_dst_for_bwd,

src/common/math_utils.hpp

@@ -414,6 +414,31 @@ inline U hardswish_bwd(T dd, T s, A alpha, A beta) {
     return v <= 0.f ? 0.f : v >= 1.f ? dd : dd * w;
 }
 
+template <typename T,
+        typename U = typename utils::remove_reference<T>::type>
+inline U hsigmoid_fwd(T s) {
+    float v = s + 3.0f;
+    v = v > 0.0f ? v : 0.0f;
+    v = v < 6.0f ? v : 6.0f;
+    return (U)(v / 6.0f);
+}
+
+template <typename T,
+        typename U = typename utils::remove_reference<T>::type>
+inline U round_half_to_even_fwd(T s) {
+    float r = ::roundf((float)s);
+    float d = (float)s - r;
+    float remainder = ::fmodf(r, 2.0f);
+    return ((d != 0.5f) && (d != -0.5f)) || (remainder == 0.0f) ? (U)r :
+           (U)((float)s + d);
+}
+
+template <typename T,
+        typename U = typename utils::remove_reference<T>::type>
+inline U round_half_away_from_zero_fwd(T s) {
+    return (U)(::roundf((float)s));
+}
+
 inline bool is_eltwise_ok(
         data_type_t src_dt, alg_kind_t alg, float alpha, float beta) {
     using namespace alg_kind;
@@ -426,7 +451,8 @@ inline bool is_eltwise_ok(
                       eltwise_exp, eltwise_gelu_tanh, eltwise_hardsigmoid,
                       eltwise_hardswish, eltwise_swish, eltwise_log,
                       eltwise_clip, eltwise_clip_v2, eltwise_pow,
-                      eltwise_gelu_erf, eltwise_round)
+                      eltwise_gelu_erf, eltwise_round,
+                      eltwise_hsigmoid, eltwise_round_half_away_from_zero, eltwise_round_half_to_even)
             && IMPLICATION(
                     one_of(alg, eltwise_clip, eltwise_clip_v2), beta >= alpha)
             && IMPLICATION(alg == eltwise_round, src_dt == dnnl_f32)

src/cpu/primitive_attr_postops.cpp

@@ -69,6 +69,9 @@ float compute_eltwise_scalar_fwd(
         case eltwise_mish: d = mish_fwd(s); break;
         case eltwise_hardsigmoid: d = hardsigmoid_fwd(s, alpha, beta); break;
         case eltwise_hardswish: d = hardswish_fwd(s, alpha, beta); break;
+        case eltwise_hsigmoid: d = hsigmoid_fwd(s); break;
+        case eltwise_round_half_away_from_zero: d = round_half_away_from_zero_fwd(s); break;
+        case eltwise_round_half_to_even: d = round_half_to_even_fwd(s); break;
         case eltwise_relu_use_dst_for_bwd: d = relu_fwd(s, alpha); break;
         case eltwise_tanh_use_dst_for_bwd: d = tanh_fwd(s); break;
         case eltwise_elu_use_dst_for_bwd: d = elu_fwd(s, alpha); break;
@@ -155,11 +158,12 @@ ref_eltwise_scalar_fwd_t::ref_eltwise_scalar_fwd_t(
             eltwise_soft_relu, eltwise_mish, eltwise_logistic, eltwise_exp,
             eltwise_gelu_tanh, eltwise_swish, eltwise_log, eltwise_clip,
             eltwise_clip_v2, eltwise_pow, eltwise_gelu_erf, eltwise_round,
-            eltwise_hardsigmoid, eltwise_hardswish,
-            eltwise_relu_use_dst_for_bwd, eltwise_tanh_use_dst_for_bwd,
-            eltwise_elu_use_dst_for_bwd, eltwise_sqrt_use_dst_for_bwd,
-            eltwise_logistic_use_dst_for_bwd, eltwise_exp_use_dst_for_bwd,
-            eltwise_clip_v2_use_dst_for_bwd));
+            eltwise_hardswish, eltwise_hardsigmoid,
+            eltwise_hsigmoid, eltwise_round_half_away_from_zero, eltwise_round_half_to_even,
+            eltwise_relu_use_dst_for_bwd,
+            eltwise_tanh_use_dst_for_bwd, eltwise_elu_use_dst_for_bwd,
+            eltwise_sqrt_use_dst_for_bwd, eltwise_logistic_use_dst_for_bwd,
+            eltwise_exp_use_dst_for_bwd, eltwise_clip_v2_use_dst_for_bwd));
 }
 
 ref_eltwise_scalar_fwd_t::ref_eltwise_scalar_fwd_t(

src/cpu/x64/injectors/jit_uni_eltwise_injector.cpp

@@ -41,6 +41,7 @@ bool is_alg_supported(alg_kind_t alg) {
             eltwise_gelu_tanh, eltwise_hardsigmoid, eltwise_hardswish,
             eltwise_swish, eltwise_log, eltwise_clip, eltwise_clip_v2,
             eltwise_pow, eltwise_gelu_erf, eltwise_round,
+            eltwise_hsigmoid, eltwise_round_half_away_from_zero, eltwise_round_half_to_even,
             eltwise_relu_use_dst_for_bwd, eltwise_tanh_use_dst_for_bwd,
             eltwise_elu_use_dst_for_bwd, eltwise_sqrt_use_dst_for_bwd,
             eltwise_logistic_use_dst_for_bwd, eltwise_exp_use_dst_for_bwd,
@@ -1833,6 +1834,49 @@ size_t jit_uni_eltwise_injector<isa, Wmm>::op_vecs_count(
     return ret;
 }
 
+template <cpu_isa_t isa, typename Wmm>
+void jit_uni_eltwise_injector<isa, Wmm>::hsigmoid_compute_vector_fwd(
+        const Vmm &vmm_src) {
+    // x + 3
+    h->uni_vaddps(vmm_src, vmm_src, table_val(hsigmoid, 0));
+    // relu6(x + 3)
+    h->uni_vmaxps(vmm_src, vmm_src, table_val(zero));
+    h->uni_vminps(vmm_src, vmm_src, table_val(hsigmoid, 1));
+    // relu6(x + 3) / 6
+    h->uni_vmulps(vmm_src, vmm_src, table_val(hsigmoid, 2));
+}
+
+template <cpu_isa_t isa, typename Wmm>
+void jit_uni_eltwise_injector<isa, Wmm>::round_half_to_even_compute_vector_fwd(
+        const Vmm &vmm_src) {
+    h->uni_vroundps(vmm_src, vmm_src, _op_near);
+}
+
+template <cpu_isa_t isa, typename Wmm>
+void jit_uni_eltwise_injector<isa, Wmm>::round_half_away_from_zero_compute_vector_fwd(
+        const Vmm &vmm_src) {
+    // create a mask of negative numbers for later returning sign
+    compute_cmp_mask(vmm_src, table_val(zero), _cmp_lt_os);
+
+    // round half away from zero for positive numbers
+    h->uni_vandps(vmm_src, vmm_src, table_val(positive_mask));
+    h->uni_vaddps(vmm_src, vmm_src, table_val(half));
+    h->uni_vroundps(vmm_src, vmm_src, _op_floor);
+
+    // return a sign for negative numbers using the mask
+    if (isa == sse41) {
+        h->movups(vmm_aux(1), vmm_src);
+        h->mulps(vmm_aux(1), table_val(minus_one));
+        h->blendvps(vmm_src, vmm_aux(1));
+    } else if (isa == avx2) {
+        h->vmulps(vmm_aux(1), vmm_src, table_val(minus_one));
+        h->vblendvps(vmm_src, vmm_src, vmm_aux(1), vmm_mask_);
+    } else if (isa == avx512_core) {
+        h->vmulps(vmm_aux(1), vmm_src, table_val(minus_one));
+        h->vblendmps(vmm_src | k_mask_, vmm_src, vmm_aux(1));
+    }
+}
+
 template <cpu_isa_t isa, typename Wmm>
 size_t jit_uni_eltwise_injector<isa, Wmm>::aux_vecs_count(
         alg_kind_t alg, bool is_fwd, float alpha) {
@@ -1873,6 +1917,9 @@ size_t jit_uni_eltwise_injector<isa, Wmm>::aux_vecs_count(
             case eltwise_round: n_vmms = 0; break;
             case eltwise_hardswish: n_vmms = 1; break;
             case eltwise_hardsigmoid: n_vmms = 0; break;
+            case eltwise_hsigmoid: n_vmms = 0; break;
+            case eltwise_round_half_to_even: n_vmms = 0; break;
+            case eltwise_round_half_away_from_zero: n_vmms = 2; break;
             default: assert(!"unsupported eltwise algorithm");
         }
     } else {
@@ -2042,6 +2089,9 @@ void jit_uni_eltwise_injector<isa, Wmm>::compute_body(
                 case eltwise_hardsigmoid:
                     hardsigmoid_compute_vector_fwd(Vmm(idx));
                     break;
+                case eltwise_hsigmoid: hsigmoid_compute_vector_fwd(Vmm(idx)); break;
+                case eltwise_round_half_to_even: round_half_to_even_compute_vector_fwd(Vmm(idx)); break;
+                case eltwise_round_half_away_from_zero: round_half_away_from_zero_compute_vector_fwd(Vmm(idx)); break;
                 default: assert(!"unsupported eltwise algorithm");
             }
         } else {
@@ -2826,6 +2876,13 @@ void jit_uni_eltwise_injector<isa, Wmm>::register_table_entries() {
                     {0xc2b00f34, true}}, // 63: -88.029693603515625
     };
 
+    // hsigmoid(x) polynomial approximation
+    static const table_t hsigmoid_values {
+            {hsigmoid, {0x40400000, true}}, // 3
+            {hsigmoid, {0x40C00000, true}}, // 6
+            {hsigmoid, {0x3e2aaaaa, true}}, // 1 / 6
+    };
+
     // This object takes care about which constants and polynomials to include.
     struct need_t {
         need_t(alg_kind_t alg) {
@@ -2845,6 +2902,7 @@ void jit_uni_eltwise_injector<isa, Wmm>::register_table_entries() {
                 case eltwise_mish: mish_ = true; break;
                 case eltwise_tanh_use_dst_for_bwd:
                 case eltwise_tanh: tanh_ = true; break;
+                case eltwise_hsigmoid: hsigmoid_ = true; break;
                 default: break;
             }
         }
@@ -2856,6 +2914,7 @@ void jit_uni_eltwise_injector<isa, Wmm>::register_table_entries() {
         bool gelu_tanh_ = false;
         bool gelu_erf_ = false;
         bool log_ = false;
+        bool hsigmoid_ = false;
 
         bool exp() const { return exp_ || soft_relu_ || gelu_erf_ || mish_; }
         bool mish() const { return mish_; }
@@ -2864,6 +2923,7 @@ void jit_uni_eltwise_injector<isa, Wmm>::register_table_entries() {
         bool gelu_tanh() const { return gelu_tanh_; }
         bool gelu_erf() const { return gelu_erf_; }
         bool log() const { return log_; }
+        bool hsigmoid() const { return hsigmoid_; }
     };
 
     need_t need(alg_);
@@ -2903,6 +2963,7 @@ void jit_uni_eltwise_injector<isa, Wmm>::register_table_entries() {
     if (need.log()) push_entries_of(log_consts);
     if (need.log()) push_entries_of(log_polynomial);
     if (need.log()) push_entries_of(log_predefined_values);
+    if (need.hsigmoid()) push_entries_of(hsigmoid_values);
 
     // Now that we registered the entries, we set the offsets.  No
     // entries should be registered after this point.  This allows to

src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp

@@ -174,7 +174,8 @@ struct jit_uni_eltwise_injector {
         _cmp_ge_os = jit_generator::_cmp_nlt_us,
         _cmp_gt_os = jit_generator::_cmp_nle_us,
         _op_floor = jit_generator::_op_floor,
-        _op_mxcsr = jit_generator::_op_mxcsr
+        _op_mxcsr = jit_generator::_op_mxcsr,
+        _op_near = jit_generator::_op_near
     };
 
     const bool is_avx512_ = is_superset(isa, avx512_core);
@@ -245,6 +246,9 @@ struct jit_uni_eltwise_injector {
     void round_compute_vector_fwd(const Vmm &vmm_src);
     void hardswish_compute_vector_fwd(const Vmm &vmm_src);
     void hardsigmoid_compute_vector_fwd(const Vmm &vmm_src);
+    void hsigmoid_compute_vector_fwd(const Vmm &vmm_src);
+    void round_half_to_even_compute_vector_fwd(const Vmm &vmm_src);
+    void round_half_away_from_zero_compute_vector_fwd(const Vmm &vmm_src);
 
     void exp_compute_vector_bwd(const Vmm &vmm_src);
     void relu_compute_vector_bwd(const Vmm &vmm_src);
@@ -324,6 +328,7 @@ struct jit_uni_eltwise_injector {
         log_five_bit_offset, // 5 bits off (31 = 2^5 - 1)
         log_pol, // see correspondent table for float values
         log_predefined_vals, // see correspondent table for float values
+        hsigmoid, // hsigmoid
         undef_key,
     };
 

src/cpu/x64/jit_generator.hpp

@@ -186,6 +186,7 @@ class jit_generator : public Xbyak::MmapAllocator,
         _cmp_nlt_us = 5u,
         _cmp_nle_us = 6u,
 
+        _op_near = 0u,
         _op_floor = 1u,
         _op_mxcsr = 4u,
     };