From f78b243b76454a88dbae88bd1cbb318a25a148a7 Mon Sep 17 00:00:00 2001 From: Dmitrii Zarukin Date: Mon, 17 Mar 2025 10:38:56 -0700 Subject: [PATCH 01/16] cpu: tidy: fix modernize-use-override and modernize-use-equals-default --- src/cpu/cpu_stream.hpp | 4 ++-- src/cpu/ref_concat.hpp | 2 +- src/cpu/ref_deconvolution.hpp | 6 ----- src/cpu/ref_resampling.hpp | 8 ++++--- src/cpu/ref_shuffle.hpp | 4 ++-- src/cpu/rnn/ref_rnn.hpp | 2 +- src/cpu/simple_resampling.hpp | 8 ++++--- src/cpu/x64/brgemm/brgemm_containers.hpp | 6 ++--- src/cpu/x64/brgemm/brgemm_types.hpp | 24 +++++++++---------- .../jit_avx512_core_gemm_bf16bf16f32_kern.hpp | 2 +- .../jit_avx512_core_gemv_bf16bf16f32_kern.hpp | 2 +- src/cpu/x64/gemm/gemm_threading.hpp | 4 ++-- .../injectors/jit_uni_postops_injector.hpp | 4 ++-- src/cpu/x64/ip_convolution.hpp | 6 ----- src/cpu/x64/jit_avx512_common_conv_kernel.hpp | 6 ++--- .../x64/jit_avx512_core_amx_conv_kernel.hpp | 9 +++---- .../jit_avx512_core_bf16_dw_conv_kernel.hpp | 8 ++++--- src/cpu/x64/jit_avx512_core_bf16cvt.hpp | 6 ++--- src/cpu/x64/jit_avx512_core_resampling.hpp | 6 ++--- ...jit_avx512_core_x8s8s32x_deconvolution.hpp | 3 ++- src/cpu/x64/jit_brgemm_1x1_conv.hpp | 2 +- src/cpu/x64/jit_brgemm_conv.hpp | 2 +- src/cpu/x64/jit_brgemm_conv_bwd.hpp | 4 ++-- src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp | 4 ++-- .../x64/jit_brgemm_conv_comp_pad_kernel.hpp | 6 ++--- src/cpu/x64/jit_brgemm_deconv.hpp | 4 +--- src/cpu/x64/jit_brgemm_post_ops.hpp | 2 +- src/cpu/x64/jit_brgemm_transpose_utils.hpp | 14 ++++++----- src/cpu/x64/jit_transpose_utils.hpp | 8 +++---- src/cpu/x64/jit_uni_batch_normalization.hpp | 8 ++++--- .../x64/jit_uni_batch_normalization_s8.hpp | 3 ++- src/cpu/x64/jit_uni_binary.hpp | 5 ++-- src/cpu/x64/jit_uni_eltwise.hpp | 6 +++-- src/cpu/x64/jit_uni_eltwise_int.hpp | 3 ++- src/cpu/x64/jit_uni_i8i8_pooling.hpp | 5 ++-- src/cpu/x64/jit_uni_layer_normalization.hpp | 8 ++++--- src/cpu/x64/jit_uni_ncsp_convolution.hpp | 10 ++++---- src/cpu/x64/jit_uni_pool_kernel.hpp | 3 ++- src/cpu/x64/jit_uni_pooling.hpp | 4 ++-- src/cpu/x64/jit_uni_reduction.hpp | 5 ++-- src/cpu/x64/jit_uni_reduction_kernel.hpp | 6 ++--- src/cpu/x64/jit_uni_reorder.hpp | 6 ++--- src/cpu/x64/jit_uni_resampling.hpp | 5 ++-- src/cpu/x64/jit_uni_resampling_kernel.hpp | 6 ++--- .../x64/jit_uni_tbb_batch_normalization.hpp | 8 ++++--- .../x64/jit_uni_x8s8s32x_deconvolution.hpp | 8 ++++--- src/cpu/x64/jit_uni_xf16_sum.hpp | 6 ++--- src/cpu/x64/lrn/jit_avx512_common_lrn.hpp | 6 +++-- src/cpu/x64/lrn/jit_uni_lrn.hpp | 6 +++-- src/cpu/x64/lrn/jit_uni_lrn_kernel.hpp | 4 ++-- src/cpu/x64/lrn/lrn_avx512_nhwc_executor.hpp | 4 ++-- .../x64/matmul/brgemm_matmul_copy_utils.hpp | 4 ++-- src/cpu/x64/prelu/jit_prelu_backward.hpp | 4 ++-- src/cpu/x64/prelu/jit_prelu_forward.hpp | 4 ++-- .../rnn/jit_uni_gru_cell_postgemm_1_bwd.hpp | 4 ++-- .../rnn/jit_uni_gru_cell_postgemm_2_bwd.hpp | 4 ++-- .../rnn/jit_uni_gru_lbr_cell_postgemm_bwd.hpp | 4 ++-- .../rnn/jit_uni_lstm_cell_postgemm_bwd.hpp | 5 ++-- .../rnn/jit_uni_lstm_cell_postgemm_fwd.hpp | 4 ++-- ..._uni_lstm_cell_projection_postgemm_fwd.hpp | 6 ++--- .../x64/rnn/jit_uni_rnn_cell_postgemm_bwd.hpp | 4 ++-- .../x64/rnn/jit_uni_rnn_common_postgemm.hpp | 4 ++-- src/cpu/x64/shuffle/jit_uni_shuffle.hpp | 4 ++-- 63 files changed, 181 insertions(+), 161 deletions(-) diff --git a/src/cpu/cpu_stream.hpp b/src/cpu/cpu_stream.hpp index 30d5a6e058b..7bf2cac3a44 100644 --- a/src/cpu/cpu_stream.hpp +++ b/src/cpu/cpu_stream.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,7 +34,7 @@ namespace cpu { struct cpu_stream_t : public stream_t { cpu_stream_t(engine_t *engine, impl::stream_impl_t *stream_impl) : stream_t(engine, stream_impl) {} - virtual ~cpu_stream_t() = default; + ~cpu_stream_t() override = default; dnnl::impl::status_t wait() override { // CPU execution is synchronous so return immediately diff --git a/src/cpu/ref_concat.hpp b/src/cpu/ref_concat.hpp index 090fbb6863d..6b87295dfcf 100644 --- a/src/cpu/ref_concat.hpp +++ b/src/cpu/ref_concat.hpp @@ -109,7 +109,7 @@ struct ref_concat_t : public primitive_t { return status::success; } - ~ref_concat_t() = default; + ~ref_concat_t() override = default; status_t execute(const exec_ctx_t &ctx) const override { using namespace memory_tracking::names; diff --git a/src/cpu/ref_deconvolution.hpp b/src/cpu/ref_deconvolution.hpp index a41c871bc6e..63cc0c37fcc 100644 --- a/src/cpu/ref_deconvolution.hpp +++ b/src/cpu/ref_deconvolution.hpp @@ -102,8 +102,6 @@ struct ref_deconvolution_fwd_t : public primitive_t { , dst_tag_(other.dst_tag_) , name_(other.name_) {} - ~pd_t() = default; - DECLARE_COMMON_PD_T(name_.c_str(), ref_deconvolution_fwd_t); status_t init_convolution(engine_t *engine) { @@ -335,8 +333,6 @@ struct ref_deconvolution_bwd_data_t : public primitive_t { , conv_pd_(other.conv_pd_->clone()) , name_(other.name_) {} - ~pd_t() = default; - DECLARE_COMMON_PD_T(name_.c_str(), ref_deconvolution_bwd_data_t); status_t init_convolution(engine_t *engine) { @@ -446,8 +442,6 @@ struct ref_deconvolution_bwd_weights_t : public primitive_t { , dst_tag_(other.dst_tag_) , name_(other.name_) {} - ~pd_t() = default; - DECLARE_COMMON_PD_T(name_.c_str(), ref_deconvolution_bwd_weights_t); status_t init_convolution(engine_t *engine) { diff --git a/src/cpu/ref_resampling.hpp b/src/cpu/ref_resampling.hpp index bb0c4e63465..cc6941ca58e 100644 --- a/src/cpu/ref_resampling.hpp +++ b/src/cpu/ref_resampling.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -67,7 +67,8 @@ struct ref_resampling_fwd_t : public primitive_t { }; ref_resampling_fwd_t(const pd_t *apd); - ~ref_resampling_fwd_t(); + + ~ref_resampling_fwd_t() override; status_t init(engine_t *engine) override { ref_post_ops_ @@ -114,7 +115,8 @@ struct ref_resampling_bwd_t : public primitive_t { }; ref_resampling_bwd_t(const pd_t *apd); - ~ref_resampling_bwd_t(); + + ~ref_resampling_bwd_t() override; status_t execute(const exec_ctx_t &ctx) const override { execute_backward(ctx); diff --git a/src/cpu/ref_shuffle.hpp b/src/cpu/ref_shuffle.hpp index 5d2adf13407..168c7cd6170 100644 --- a/src/cpu/ref_shuffle.hpp +++ b/src/cpu/ref_shuffle.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2018-2024 Intel Corporation +* Copyright 2018-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -92,7 +92,7 @@ struct ref_shuffle_t : public primitive_t { return dnnl_success; } - ~ref_shuffle_t() { free(rev_transposed_); } + ~ref_shuffle_t() override { free(rev_transposed_); } status_t execute(const exec_ctx_t &ctx) const override { const memory_desc_wrapper src_d( diff --git a/src/cpu/rnn/ref_rnn.hpp b/src/cpu/rnn/ref_rnn.hpp index d5315b0fe4b..45abcd83a77 100644 --- a/src/cpu/rnn/ref_rnn.hpp +++ b/src/cpu/rnn/ref_rnn.hpp @@ -172,7 +172,7 @@ struct _ref_rnn_common_t : public primitive_t { : primitive_t(apd), rnn_postgemm_(nullptr) {} status_t init(engine_t *engine) override; - virtual ~_ref_rnn_common_t() { delete rnn_postgemm_; } + ~_ref_rnn_common_t() override { delete rnn_postgemm_; } status_t execute(const exec_ctx_t &ctx) const override; diff --git a/src/cpu/simple_resampling.hpp b/src/cpu/simple_resampling.hpp index f632baa27a4..a9ccef95af2 100644 --- a/src/cpu/simple_resampling.hpp +++ b/src/cpu/simple_resampling.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -103,7 +103,8 @@ struct simple_resampling_fwd_t : public primitive_t { simple_resampling_fwd_t(const pd_t *apd); status_t init(engine_t *engine) override; - ~simple_resampling_fwd_t() = default; + + ~simple_resampling_fwd_t() override = default; status_t execute(const exec_ctx_t &ctx) const override; @@ -149,7 +150,8 @@ struct simple_resampling_bwd_t : public primitive_t { simple_resampling_bwd_t(const pd_t *apd); status_t init(engine_t *engine) override; - ~simple_resampling_bwd_t() = default; + + ~simple_resampling_bwd_t() override = default; status_t execute(const exec_ctx_t &ctx) const override; diff --git a/src/cpu/x64/brgemm/brgemm_containers.hpp b/src/cpu/x64/brgemm/brgemm_containers.hpp index bc4f7ec78c6..5f5a7c67177 100644 --- a/src/cpu/x64/brgemm/brgemm_containers.hpp +++ b/src/cpu/x64/brgemm/brgemm_containers.hpp @@ -34,7 +34,7 @@ namespace brgemm_containers { struct brgemm_desc_container_t { public: - brgemm_desc_container_t() {} + brgemm_desc_container_t() = default; brgemm_desc_container_t(size_t ns) { resize(ns); } void resize(size_t ns) { refs_.resize(ns); } inline const brgemm_desc_t *operator[](int idx) const { return refs_[idx]; } @@ -71,7 +71,7 @@ struct brgemm_desc_container_t { // #define BRGEMM_KERNEL_GLOBAL_STORAGE struct brgemm_kernel_container_t { - brgemm_kernel_container_t() {} + brgemm_kernel_container_t() = default; brgemm_kernel_container_t(size_t ns) { resize(ns); } void resize(size_t ns) { refs_.resize(ns); } inline const brgemm_kernel_t *operator[](int idx) const { @@ -113,7 +113,7 @@ struct brgemm_kernel_container_t { struct brgemm_palette_container_t { using S_t = std::array; - brgemm_palette_container_t() {} + brgemm_palette_container_t() = default; brgemm_palette_container_t(size_t ns) { resize(ns); } void resize(size_t ns) { refs_.resize(ns); } diff --git a/src/cpu/x64/brgemm/brgemm_types.hpp b/src/cpu/x64/brgemm/brgemm_types.hpp index 1b86897858f..809b541e52d 100644 --- a/src/cpu/x64/brgemm/brgemm_types.hpp +++ b/src/cpu/x64/brgemm/brgemm_types.hpp @@ -206,7 +206,7 @@ struct DNNL_API brgemm_attr_t { }; struct brgemm_desc_t { - brgemm_desc_t() {} + brgemm_desc_t() = default; brgemm_desc_t(const brgemm_desc_t &other); DNNL_API ~brgemm_desc_t(); @@ -572,8 +572,8 @@ struct jit_brdgmm_kernel_base_t; class jit_generator; struct brgemm_kernel_t { - brgemm_kernel_t() {}; - virtual ~brgemm_kernel_t() {}; + brgemm_kernel_t() = default; + virtual ~brgemm_kernel_t() = default; virtual status_t create_kernel() = 0; virtual void operator()(brgemm_kernel_params_t *) const = 0; virtual const jit_generator *get_jit_generator() const = 0; @@ -589,12 +589,12 @@ struct jit_base_brgemm_kernel_t : public jit_generator { template struct brgemm_kernel_common_t : public brgemm_kernel_t { brgemm_kernel_common_t(const brgemm_desc_t &abrd); - ~brgemm_kernel_common_t(); + ~brgemm_kernel_common_t() override; status_t create_kernel() override; void operator()(brgemm_kernel_params_t *) const override; - virtual const jit_generator *get_jit_generator() const override; - virtual const brgemm_desc_t &get_brg() const override { + const jit_generator *get_jit_generator() const override; + const brgemm_desc_t &get_brg() const override { return ((jit_base_brgemm_kernel_t *)brgemm_kernel_)->get_brg(); } @@ -606,12 +606,12 @@ struct brgemm_kernel_common_t : public brgemm_kernel_t { struct brgemm_amx_uker_t : public brgemm_kernel_t { brgemm_amx_uker_t(const brgemm_desc_t &abrd); - ~brgemm_amx_uker_t(); + ~brgemm_amx_uker_t() override; status_t create_kernel() override; void operator()(brgemm_kernel_params_t *) const override; - virtual const jit_generator *get_jit_generator() const override; - virtual const brgemm_desc_t &get_brg() const override { + const jit_generator *get_jit_generator() const override; + const brgemm_desc_t &get_brg() const override { return ((jit_base_brgemm_kernel_t *)brgemm_kernel_)->get_brg(); } @@ -624,12 +624,12 @@ struct brgemm_amx_uker_t : public brgemm_kernel_t { template struct brdgmm_kernel_t : public brgemm_kernel_t { brdgmm_kernel_t(const brgemm_desc_t &abrd); - ~brdgmm_kernel_t(); + ~brdgmm_kernel_t() override; status_t create_kernel() override; void operator()(brgemm_kernel_params_t *) const override; - virtual const jit_generator *get_jit_generator() const override; - virtual const brgemm_desc_t &get_brg() const override { + const jit_generator *get_jit_generator() const override; + const brgemm_desc_t &get_brg() const override { return ((jit_base_brgemm_kernel_t *)brgemm_kernel_)->get_brg(); } diff --git a/src/cpu/x64/gemm/bf16/jit_avx512_core_gemm_bf16bf16f32_kern.hpp b/src/cpu/x64/gemm/bf16/jit_avx512_core_gemm_bf16bf16f32_kern.hpp index 076b49c0501..9490a0c6e89 100644 --- a/src/cpu/x64/gemm/bf16/jit_avx512_core_gemm_bf16bf16f32_kern.hpp +++ b/src/cpu/x64/gemm/bf16/jit_avx512_core_gemm_bf16bf16f32_kern.hpp @@ -29,7 +29,7 @@ class jit_avx512_core_gemm_bf16bf16f32_kern : public jit_generator { public: jit_avx512_core_gemm_bf16bf16f32_kern( bool beta_zero, bool alpha_one, bool use_zmm); - ~jit_avx512_core_gemm_bf16bf16f32_kern(); + ~jit_avx512_core_gemm_bf16bf16f32_kern() override; DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_gemm_bf16bf16f32_kern); protected: diff --git a/src/cpu/x64/gemm/bf16/jit_avx512_core_gemv_bf16bf16f32_kern.hpp b/src/cpu/x64/gemm/bf16/jit_avx512_core_gemv_bf16bf16f32_kern.hpp index e370d95f683..a375172924c 100644 --- a/src/cpu/x64/gemm/bf16/jit_avx512_core_gemv_bf16bf16f32_kern.hpp +++ b/src/cpu/x64/gemm/bf16/jit_avx512_core_gemv_bf16bf16f32_kern.hpp @@ -28,7 +28,7 @@ namespace x64 { class jit_avx512_core_gemv_bf16bf16f32_kern : public jit_generator { public: jit_avx512_core_gemv_bf16bf16f32_kern(bool trans); - ~jit_avx512_core_gemv_bf16bf16f32_kern(); + ~jit_avx512_core_gemv_bf16bf16f32_kern() override; DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_gemv_bf16bf16f32_kern); protected: diff --git a/src/cpu/x64/gemm/gemm_threading.hpp b/src/cpu/x64/gemm/gemm_threading.hpp index 3915dd54f12..b0af2760095 100644 --- a/src/cpu/x64/gemm/gemm_threading.hpp +++ b/src/cpu/x64/gemm/gemm_threading.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2022 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,7 +43,7 @@ struct gemm_slice_t { }; struct gemm_threading_t { - gemm_threading_t() {}; + gemm_threading_t() = default; int nthrs_m, nthrs_n, nthrs_k; dim_t block_m, block_n, block_k; // Blocking sizes (-1 = default) diff --git a/src/cpu/x64/injectors/jit_uni_postops_injector.hpp b/src/cpu/x64/injectors/jit_uni_postops_injector.hpp index eee12c7d7b3..53c27436045 100644 --- a/src/cpu/x64/injectors/jit_uni_postops_injector.hpp +++ b/src/cpu/x64/injectors/jit_uni_postops_injector.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -139,7 +139,7 @@ class jit_uni_postops_injector_t : public jit_uni_postops_injector_base_t { const eltwise_injector::static_params_t &eltwise_static_params, const lambda_jit_injectors_t &lambda_jit_injectors); - virtual ~jit_uni_postops_injector_t() = default; + ~jit_uni_postops_injector_t() override = default; // See `jit_uni_postops_injector_base_t::compute_vector_range(...)` void compute_vector_range(const injector_utils::vmm_index_set_t &vmm_idxs, diff --git a/src/cpu/x64/ip_convolution.hpp b/src/cpu/x64/ip_convolution.hpp index 4c4902e37ab..cb57a1bbcba 100644 --- a/src/cpu/x64/ip_convolution.hpp +++ b/src/cpu/x64/ip_convolution.hpp @@ -158,8 +158,6 @@ struct ip_convolution_fwd_t : public primitive_t { , ip_pd_(other.ip_pd_->clone()) , name_(other.name_) {} - ~pd_t() = default; - DECLARE_COMMON_PD_T(name_.c_str(), ip_convolution_fwd_t); status_t init_ip(engine_t *engine) { @@ -273,8 +271,6 @@ struct ip_convolution_bwd_data_t : public primitive_t { , ip_pd_(other.ip_pd_->clone()) , name_(other.name_) {} - ~pd_t() = default; - DECLARE_COMMON_PD_T(name_.c_str(), ip_convolution_bwd_data_t); status_t init_ip(engine_t *engine) { @@ -385,8 +381,6 @@ struct ip_convolution_bwd_weights_t : public primitive_t { , ip_pd_(other.ip_pd_->clone()) , name_(other.name_) {} - ~pd_t() = default; - DECLARE_COMMON_PD_T(name_.c_str(), ip_convolution_bwd_weights_t); status_t init_ip(engine_t *engine) { diff --git a/src/cpu/x64/jit_avx512_common_conv_kernel.hpp b/src/cpu/x64/jit_avx512_common_conv_kernel.hpp index 163ff4ddc78..c741eeb4f8b 100644 --- a/src/cpu/x64/jit_avx512_common_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_common_conv_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2016-2023 Intel Corporation +* Copyright 2016-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -204,7 +204,7 @@ struct jit_avx512_common_conv_fwd_kernel { return status::out_of_memory; } - ~jit_avx512_common_conv_fwd_kernel() {} + ~jit_avx512_common_conv_fwd_kernel() = default; enum { typesize = sizeof(float) }; @@ -374,7 +374,7 @@ struct jit_avx512_common_conv_bwd_data_kernel_f32 { return status::out_of_memory; } - ~jit_avx512_common_conv_bwd_data_kernel_f32() {} + ~jit_avx512_common_conv_bwd_data_kernel_f32() = default; enum { typesize = sizeof(float) }; diff --git a/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp index 4aa52520753..fcd30e2316b 100644 --- a/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -488,7 +488,8 @@ struct jit_avx512_core_amx_bwd_data_kernel_t : public jit_generator { CHECK(bwd_data_copy_kernel_->create_kernel()); return status::success; } - ~jit_avx512_core_amx_bwd_data_kernel_t() = default; + + ~jit_avx512_core_amx_bwd_data_kernel_t() override = default; static bool post_ops_ok(const jit_conv_conf_t &jcp, primitive_attr_t &attr); @@ -614,7 +615,7 @@ struct jit_avx512_core_amx_bwd_weights_kernel_t : public jit_generator { jit_avx512_core_amx_bwd_weights_kernel_t(const jit_conv_conf_t &ajcp) : jit_generator(jit_name(), avx512_core_amx), jcp(ajcp) {} - ~jit_avx512_core_amx_bwd_weights_kernel_t() {} + ~jit_avx512_core_amx_bwd_weights_kernel_t() override = default; DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_amx_bwd_weights_kernel_t) @@ -749,7 +750,7 @@ struct jit_avx512_core_amx_bwd_bias_kernel_t : public jit_generator { jit_avx512_core_amx_bwd_bias_kernel_t(const jit_conv_conf_t &ajcp) : jit_generator(jit_name(), avx512_core_amx), jcp(ajcp) {} - ~jit_avx512_core_amx_bwd_bias_kernel_t() {} + ~jit_avx512_core_amx_bwd_bias_kernel_t() override = default; DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_amx_bwd_bias_kernel_t) diff --git a/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.hpp index 6427ce7e738..885c78a89ff 100644 --- a/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2021 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -138,7 +138,8 @@ struct jit_avx512_dw_conv_bwd_data_kernel_bf16 : public jit_generator { bf16_emu_reserv_1, bf16_emu_reserv_2, bf16_emu_reserv_3, bf16_emu_reserv_4, bf16_emu_reserv_5, bf16_emu_reserv_6); } - ~jit_avx512_dw_conv_bwd_data_kernel_bf16() = default; + + ~jit_avx512_dw_conv_bwd_data_kernel_bf16() override = default; jit_conv_conf_t jcp; @@ -216,7 +217,8 @@ struct jit_avx512_dw_conv_bwd_weights_kernel_bf16 : public jit_generator { bf16_emu_reserv_1, bf16_emu_reserv_2, bf16_emu_reserv_3, bf16_emu_reserv_4, bf16_emu_reserv_5, bf16_emu_reserv_6); } - ~jit_avx512_dw_conv_bwd_weights_kernel_bf16() = default; + + ~jit_avx512_dw_conv_bwd_weights_kernel_bf16() override = default; jit_conv_conf_t jcp; diff --git a/src/cpu/x64/jit_avx512_core_bf16cvt.hpp b/src/cpu/x64/jit_avx512_core_bf16cvt.hpp index bc65d9e1cf2..ecb7529bdbd 100644 --- a/src/cpu/x64/jit_avx512_core_bf16cvt.hpp +++ b/src/cpu/x64/jit_avx512_core_bf16cvt.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2023 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -186,7 +186,7 @@ struct jit_avx512_core_add_cvt_ps_to_bf16_t : public jit_generator { UNUSED_STATUS(create_kernel()); } - ~jit_avx512_core_add_cvt_ps_to_bf16_t() = default; + ~jit_avx512_core_add_cvt_ps_to_bf16_t() override = default; DNNL_DISALLOW_COPY_AND_ASSIGN(jit_avx512_core_add_cvt_ps_to_bf16_t) void generate() override { @@ -297,7 +297,7 @@ struct jit_avx512_core_bf16_reorder_s16c_to_S16c2s_t : public jit_generator { jit_avx512_core_bf16_reorder_s16c_to_S16c2s_t(int in_stride) : jit_generator(jit_name()), simd_w_(16), in_stride_(in_stride) {} - ~jit_avx512_core_bf16_reorder_s16c_to_S16c2s_t() {} + ~jit_avx512_core_bf16_reorder_s16c_to_S16c2s_t() override = default; void generate() override { preamble(); diff --git a/src/cpu/x64/jit_avx512_core_resampling.hpp b/src/cpu/x64/jit_avx512_core_resampling.hpp index 2c1e30fbe1a..35e153c2d39 100644 --- a/src/cpu/x64/jit_avx512_core_resampling.hpp +++ b/src/cpu/x64/jit_avx512_core_resampling.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2022 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ struct jit_resampling_args_t; struct jit_avx512_core_resampling_kernel_base_t : public jit_generator { jit_avx512_core_resampling_kernel_base_t( const resampling_pd_t *pd, const char *name); - virtual ~jit_avx512_core_resampling_kernel_base_t() = default; + ~jit_avx512_core_resampling_kernel_base_t() override = default; protected: const resampling_pd_t *pd_; @@ -55,7 +55,7 @@ struct jit_avx512_core_resampling_bwd_t : public primitive_t { }; jit_avx512_core_resampling_bwd_t(const pd_t *apd) : primitive_t(apd) {} - ~jit_avx512_core_resampling_bwd_t(); + ~jit_avx512_core_resampling_bwd_t() override; status_t init(engine_t *engine) override; diff --git a/src/cpu/x64/jit_avx512_core_x8s8s32x_deconvolution.hpp b/src/cpu/x64/jit_avx512_core_x8s8s32x_deconvolution.hpp index 4a665b55503..2cc1c458168 100644 --- a/src/cpu/x64/jit_avx512_core_x8s8s32x_deconvolution.hpp +++ b/src/cpu/x64/jit_avx512_core_x8s8s32x_deconvolution.hpp @@ -77,7 +77,8 @@ struct jit_avx512_core_x8s8s32x_deconv_fwd_kernel : public jit_generator { jit_avx512_core_x8s8s32x_deconv_fwd_kernel(const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md); - ~jit_avx512_core_x8s8s32x_deconv_fwd_kernel(); + + ~jit_avx512_core_x8s8s32x_deconv_fwd_kernel() override; const jit_conv_conf_t &jcp; const primitive_attr_t &attr_; diff --git a/src/cpu/x64/jit_brgemm_1x1_conv.hpp b/src/cpu/x64/jit_brgemm_1x1_conv.hpp index 744b7bec479..868d95b05ad 100644 --- a/src/cpu/x64/jit_brgemm_1x1_conv.hpp +++ b/src/cpu/x64/jit_brgemm_1x1_conv.hpp @@ -106,7 +106,7 @@ struct brgemm_1x1_convolution_fwd_t : public primitive_t { brgemm_1x1_convolution_fwd_t(const pd_t *apd) : primitive_t(apd), bias_d(pd()->weights_md(1)) {} - ~brgemm_1x1_convolution_fwd_t() {} + ~brgemm_1x1_convolution_fwd_t() override = default; status_t execute(const exec_ctx_t &ctx) const override { execute_forward_all(ctx); diff --git a/src/cpu/x64/jit_brgemm_conv.hpp b/src/cpu/x64/jit_brgemm_conv.hpp index e432efb2772..bb63e9a5979 100644 --- a/src/cpu/x64/jit_brgemm_conv.hpp +++ b/src/cpu/x64/jit_brgemm_conv.hpp @@ -163,7 +163,7 @@ struct brgemm_convolution_fwd_t : public primitive_t { brgemm_convolution_fwd_t(const pd_t *apd); - ~brgemm_convolution_fwd_t() = default; + ~brgemm_convolution_fwd_t() override = default; status_t execute(const exec_ctx_t &ctx) const override; diff --git a/src/cpu/x64/jit_brgemm_conv_bwd.hpp b/src/cpu/x64/jit_brgemm_conv_bwd.hpp index 6fe1cb6139f..3b123f64401 100644 --- a/src/cpu/x64/jit_brgemm_conv_bwd.hpp +++ b/src/cpu/x64/jit_brgemm_conv_bwd.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2022-2024 Intel Corporation +* Copyright 2022-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -57,7 +57,7 @@ struct brgemm_convolution_bwd_t : public primitive_t { brgemm_convolution_bwd_t(const pd_t *apd) : primitive_t(apd) {}; - ~brgemm_convolution_bwd_t() = default; + ~brgemm_convolution_bwd_t() override = default; status_t init(engine_t *engine) override; diff --git a/src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp b/src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp index 2f499f56684..ec8d52a5a76 100644 --- a/src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp +++ b/src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2022-2024 Intel Corporation +* Copyright 2022-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -106,7 +106,7 @@ struct brgemm_convolution_bwd_strided_t : public primitive_t { brgemm_convolution_bwd_strided_t(const pd_t *apd) : primitive_t(apd), bias_d(pd()->weights_md(1)) {} - ~brgemm_convolution_bwd_strided_t() = default; + ~brgemm_convolution_bwd_strided_t() override = default; status_t execute(const exec_ctx_t &ctx) const override; diff --git a/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.hpp b/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.hpp index 070c009d24f..6e0ecafd071 100644 --- a/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.hpp +++ b/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2022-2024 Intel Corporation +* Copyright 2022-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,7 +50,7 @@ struct jit_uni_brgemm_conv_comp_pad_kernel_t : public jit_generator { jit_uni_brgemm_conv_comp_pad_kernel_t(const jit_brgemm_conv_conf_t &ajcp); - ~jit_uni_brgemm_conv_comp_pad_kernel_t() = default; + ~jit_uni_brgemm_conv_comp_pad_kernel_t() override = default; protected: static constexpr bool is_ymm_ = std::is_same::value; @@ -143,7 +143,7 @@ struct jit_uni_brgemm_conv_relo_comp_pad_kernel_t : public jit_generator { jit_uni_brgemm_conv_relo_comp_pad_kernel_t( const jit_brgemm_conv_conf_t &ajcp); - ~jit_uni_brgemm_conv_relo_comp_pad_kernel_t() = default; + ~jit_uni_brgemm_conv_relo_comp_pad_kernel_t() override = default; protected: jit_brgemm_conv_conf_t jcp_ = utils::zero(); diff --git a/src/cpu/x64/jit_brgemm_deconv.hpp b/src/cpu/x64/jit_brgemm_deconv.hpp index 119f02c0e09..80e51023a83 100644 --- a/src/cpu/x64/jit_brgemm_deconv.hpp +++ b/src/cpu/x64/jit_brgemm_deconv.hpp @@ -46,8 +46,6 @@ struct brgemm_deconvolution_fwd_t : public primitive_t { , has_strides_(other.has_strides_) , name_(other.name_) {} - ~pd_t() = default; - DECLARE_COMMON_PD_T(name_.c_str(), brgemm_deconvolution_fwd_t); status_t init(engine_t *engine); @@ -99,7 +97,7 @@ struct brgemm_deconvolution_fwd_t : public primitive_t { brgemm_deconvolution_fwd_t(const pd_t *apd) : primitive_t(apd) {}; - ~brgemm_deconvolution_fwd_t() = default; + ~brgemm_deconvolution_fwd_t() override = default; status_t init(engine_t *engine) override; diff --git a/src/cpu/x64/jit_brgemm_post_ops.hpp b/src/cpu/x64/jit_brgemm_post_ops.hpp index 0724f1a7fb1..67ccc6896f5 100644 --- a/src/cpu/x64/jit_brgemm_post_ops.hpp +++ b/src/cpu/x64/jit_brgemm_post_ops.hpp @@ -177,7 +177,7 @@ struct jit_brgemm_kernel_post_ops_t : public jit_brgemm_kernel_post_ops_base_t, return jit_generator::operator()(args); } - ~jit_brgemm_kernel_post_ops_t() = default; + ~jit_brgemm_kernel_post_ops_t() override = default; DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_brgemm_kernel_post_ops_t) diff --git a/src/cpu/x64/jit_brgemm_transpose_utils.hpp b/src/cpu/x64/jit_brgemm_transpose_utils.hpp index 051257ef966..affbcfb8316 100644 --- a/src/cpu/x64/jit_brgemm_transpose_utils.hpp +++ b/src/cpu/x64/jit_brgemm_transpose_utils.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,7 +39,7 @@ struct jit_brgemm_trans_src_t { jit_brgemm_trans_src_t(const jit_brgemm_primitive_conf_t *conf) : conf_(conf) {} - virtual ~jit_brgemm_trans_src_t() {} + virtual ~jit_brgemm_trans_src_t() = default; const jit_brgemm_primitive_conf_t *conf_; }; @@ -80,7 +80,8 @@ struct jit_brgemm_copy_to_coarse_t : public jit_generator { MAYBE_UNUSED(row_granularity_); } - ~jit_brgemm_copy_to_coarse_t() {} + + ~jit_brgemm_copy_to_coarse_t() override = default; private: enum { @@ -153,7 +154,8 @@ struct jit_brgemm_trans_to_vnni_t { jit_brgemm_trans_to_vnni_t(const jit_brgemm_primitive_conf_t *conf, matrix_to_transform_t matrix_to_transform) : conf_(conf), matrix_to_transform_(matrix_to_transform) {} - virtual ~jit_brgemm_trans_to_vnni_t() {} + + virtual ~jit_brgemm_trans_to_vnni_t() = default; const jit_brgemm_primitive_conf_t *conf_; matrix_to_transform_t matrix_to_transform_; @@ -173,7 +175,7 @@ struct jit_brgemm_trans_wei_t { jit_brgemm_trans_wei_t(const jit_brgemm_primitive_conf_t *conf) : conf_(conf) {} - virtual ~jit_brgemm_trans_wei_t() {} + virtual ~jit_brgemm_trans_wei_t() = default; const jit_brgemm_primitive_conf_t *conf_; }; @@ -239,7 +241,7 @@ struct jit_amx_ip_trans_diff_wei { , ext_ic_block_(ext_ic_block) , ext_oc_block_(ext_oc_block) {} - virtual ~jit_amx_ip_trans_diff_wei() {} + virtual ~jit_amx_ip_trans_diff_wei() = default; const jit_brgemm_primitive_conf_t *jbgp_; diff --git a/src/cpu/x64/jit_transpose_utils.hpp b/src/cpu/x64/jit_transpose_utils.hpp index d3c99a79609..720493a95c9 100644 --- a/src/cpu/x64/jit_transpose_utils.hpp +++ b/src/cpu/x64/jit_transpose_utils.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2017-2024 Intel Corporation +* Copyright 2017-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,7 +38,7 @@ struct jit_trans_src_t { virtual status_t create_kernel() = 0; jit_trans_src_t(const jit_conv_conf_t *conf) : conf_(conf) {} - virtual ~jit_trans_src_t() {} + virtual ~jit_trans_src_t() = default; const jit_conv_conf_t *conf_; }; @@ -61,7 +61,7 @@ struct jit_trans_dst_t { }; jit_trans_dst_t(const jit_conv_conf_t *conf) : conf_(conf) {} - virtual ~jit_trans_dst_t() {} + virtual ~jit_trans_dst_t() = default; virtual void operator()(ctx_t *ctx) = 0; virtual status_t create_kernel() = 0; @@ -132,7 +132,7 @@ struct jit_diff_wei_trans_to_vnni_t : public jit_generator { , oc_block_(oc_block) , nb_ic_(nb_ic) {} - ~jit_diff_wei_trans_to_vnni_t() {} + ~jit_diff_wei_trans_to_vnni_t() override = default; status_t create_kernel() override { return jit_generator::create_kernel(); } diff --git a/src/cpu/x64/jit_uni_batch_normalization.hpp b/src/cpu/x64/jit_uni_batch_normalization.hpp index 66582a058d6..9f5f8b9f78b 100644 --- a/src/cpu/x64/jit_uni_batch_normalization.hpp +++ b/src/cpu/x64/jit_uni_batch_normalization.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2017-2024 Intel Corporation +* Copyright 2017-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -63,7 +63,8 @@ struct jit_uni_batch_normalization_fwd_t : public primitive_t { }; jit_uni_batch_normalization_fwd_t(const pd_t *apd); - ~jit_uni_batch_normalization_fwd_t(); + + ~jit_uni_batch_normalization_fwd_t() override; status_t init(engine_t *engine) override; @@ -98,7 +99,8 @@ struct jit_uni_batch_normalization_bwd_t : public primitive_t { }; jit_uni_batch_normalization_bwd_t(const pd_t *apd); - ~jit_uni_batch_normalization_bwd_t(); + + ~jit_uni_batch_normalization_bwd_t() override; status_t init(engine_t *engine) override; diff --git a/src/cpu/x64/jit_uni_batch_normalization_s8.hpp b/src/cpu/x64/jit_uni_batch_normalization_s8.hpp index b26b4f5933f..41e5eb1aff1 100644 --- a/src/cpu/x64/jit_uni_batch_normalization_s8.hpp +++ b/src/cpu/x64/jit_uni_batch_normalization_s8.hpp @@ -52,7 +52,8 @@ struct jit_uni_batch_normalization_s8_fwd_t : public primitive_t { using data_t = int8_t; jit_uni_batch_normalization_s8_fwd_t(const pd_t *apd); - ~jit_uni_batch_normalization_s8_fwd_t(); + + ~jit_uni_batch_normalization_s8_fwd_t() override; status_t init(engine_t *engine) override; diff --git a/src/cpu/x64/jit_uni_binary.hpp b/src/cpu/x64/jit_uni_binary.hpp index 5b20445ae95..6e387dfa8aa 100644 --- a/src/cpu/x64/jit_uni_binary.hpp +++ b/src/cpu/x64/jit_uni_binary.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -67,7 +67,8 @@ struct jit_uni_binary_t : public primitive_t { }; jit_uni_binary_t(const pd_t *apd); - ~jit_uni_binary_t() = default; + + ~jit_uni_binary_t() override = default; status_t init(engine_t *engine) override; diff --git a/src/cpu/x64/jit_uni_eltwise.hpp b/src/cpu/x64/jit_uni_eltwise.hpp index 84f51b7b0d0..109c98d3cee 100644 --- a/src/cpu/x64/jit_uni_eltwise.hpp +++ b/src/cpu/x64/jit_uni_eltwise.hpp @@ -52,7 +52,8 @@ struct jit_uni_eltwise_fwd_t : public primitive_t { }; jit_uni_eltwise_fwd_t(const pd_t *apd); - virtual ~jit_uni_eltwise_fwd_t(); + + ~jit_uni_eltwise_fwd_t() override; using data_t = typename prec_traits_t::type; @@ -82,7 +83,8 @@ struct jit_uni_eltwise_bwd_t : public primitive_t { }; jit_uni_eltwise_bwd_t(const pd_t *apd); - virtual ~jit_uni_eltwise_bwd_t(); + + ~jit_uni_eltwise_bwd_t() override; using data_t = typename prec_traits_t::type; diff --git a/src/cpu/x64/jit_uni_eltwise_int.hpp b/src/cpu/x64/jit_uni_eltwise_int.hpp index 418db6a9ed9..41ed838d4ad 100644 --- a/src/cpu/x64/jit_uni_eltwise_int.hpp +++ b/src/cpu/x64/jit_uni_eltwise_int.hpp @@ -47,7 +47,8 @@ struct jit_uni_eltwise_int_fwd_t : public primitive_t { }; jit_uni_eltwise_int_fwd_t(const pd_t *apd); - ~jit_uni_eltwise_int_fwd_t(); + + ~jit_uni_eltwise_int_fwd_t() override; using data_t = typename prec_traits_t::type; diff --git a/src/cpu/x64/jit_uni_i8i8_pooling.hpp b/src/cpu/x64/jit_uni_i8i8_pooling.hpp index 78708c32d18..81418190173 100644 --- a/src/cpu/x64/jit_uni_i8i8_pooling.hpp +++ b/src/cpu/x64/jit_uni_i8i8_pooling.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2017-2024 Intel Corporation +* Copyright 2017-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -92,7 +92,8 @@ struct jit_uni_i8i8_pooling_fwd_t : public primitive_t { }; jit_uni_i8i8_pooling_fwd_t(const pd_t *apd); - ~jit_uni_i8i8_pooling_fwd_t(); + + ~jit_uni_i8i8_pooling_fwd_t() override; status_t init(engine_t *engine) override; diff --git a/src/cpu/x64/jit_uni_layer_normalization.hpp b/src/cpu/x64/jit_uni_layer_normalization.hpp index f4c335f752b..736d87f43c4 100644 --- a/src/cpu/x64/jit_uni_layer_normalization.hpp +++ b/src/cpu/x64/jit_uni_layer_normalization.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -128,7 +128,8 @@ struct jit_uni_layer_normalization_fwd_t : public primitive_t { } jit_uni_layer_normalization_fwd_t(const pd_t *apd) : primitive_t(apd) {} - virtual ~jit_uni_layer_normalization_fwd_t() = default; + + ~jit_uni_layer_normalization_fwd_t() override = default; void reorder_stat(const exec_ctx_t &ctx, engine_t *engine, const memory_arg_t &in, const memory_arg_t &out) const { @@ -292,7 +293,8 @@ struct jit_uni_layer_normalization_bwd_t : public primitive_t { } jit_uni_layer_normalization_bwd_t(const pd_t *apd) : primitive_t(apd) {} - virtual ~jit_uni_layer_normalization_bwd_t() = default; + + ~jit_uni_layer_normalization_bwd_t() override = default; void reorder_stat(const exec_ctx_t &ctx, engine_t *engine, const memory_arg_t &in, const memory_arg_t &out) const { diff --git a/src/cpu/x64/jit_uni_ncsp_convolution.hpp b/src/cpu/x64/jit_uni_ncsp_convolution.hpp index be43419c8da..c08fdb5de9f 100644 --- a/src/cpu/x64/jit_uni_ncsp_convolution.hpp +++ b/src/cpu/x64/jit_uni_ncsp_convolution.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2024 Intel Corporation +* Copyright 2024-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -88,7 +88,7 @@ struct jit_uni_ncsp_convolution_fwd_t : public primitive_t { jit_uni_ncsp_convolution_fwd_t(const pd_t *apd) : primitive_t(apd) {}; - ~jit_uni_ncsp_convolution_fwd_t() = default; + ~jit_uni_ncsp_convolution_fwd_t() override = default; status_t init(engine_t *engine) override; status_t execute(const exec_ctx_t &ctx) const override; @@ -136,7 +136,8 @@ struct jit_uni_ncsp_convolution_bwd_weights_t : public primitive_t { }; jit_uni_ncsp_convolution_bwd_weights_t(const pd_t *cpd) : primitive_t(cpd) {}; - ~jit_uni_ncsp_convolution_bwd_weights_t() = default; + + ~jit_uni_ncsp_convolution_bwd_weights_t() override = default; status_t init(engine_t *engine) override; status_t execute(const exec_ctx_t &ctx) const override; @@ -187,7 +188,8 @@ struct jit_uni_ncsp_convolution_bwd_data_t : public primitive_t { } }; jit_uni_ncsp_convolution_bwd_data_t(const pd_t *cpd) : primitive_t(cpd) {}; - ~jit_uni_ncsp_convolution_bwd_data_t() = default; + + ~jit_uni_ncsp_convolution_bwd_data_t() override = default; status_t init(engine_t *engine) override; status_t execute(const exec_ctx_t &ctx) const override; diff --git a/src/cpu/x64/jit_uni_pool_kernel.hpp b/src/cpu/x64/jit_uni_pool_kernel.hpp index 96538d1caf9..bf861af4c6c 100644 --- a/src/cpu/x64/jit_uni_pool_kernel.hpp +++ b/src/cpu/x64/jit_uni_pool_kernel.hpp @@ -40,7 +40,8 @@ struct jit_uni_pool_kernel : public jit_generator { jit_uni_pool_kernel( const jit_pool_conf_t &ajpp, const memory_desc_t *dst_md); jit_pool_conf_t jpp; - ~jit_uni_pool_kernel(); + + ~jit_uni_pool_kernel() override; DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_pool_kernel) diff --git a/src/cpu/x64/jit_uni_pooling.hpp b/src/cpu/x64/jit_uni_pooling.hpp index 7665b694a3b..5dd6241e79b 100644 --- a/src/cpu/x64/jit_uni_pooling.hpp +++ b/src/cpu/x64/jit_uni_pooling.hpp @@ -84,7 +84,7 @@ struct jit_uni_pooling_fwd_t : public primitive_t { explicit jit_uni_pooling_fwd_t(const pd_t *apd); jit_uni_pooling_fwd_t(jit_uni_pooling_fwd_t &&) = default; jit_uni_pooling_fwd_t &operator=(jit_uni_pooling_fwd_t &&) = default; - ~jit_uni_pooling_fwd_t(); + ~jit_uni_pooling_fwd_t() override; using data_t = typename prec_traits_t::type; @@ -160,7 +160,7 @@ struct jit_uni_pooling_bwd_t : public primitive_t { explicit jit_uni_pooling_bwd_t(const pd_t *apd); jit_uni_pooling_bwd_t(jit_uni_pooling_bwd_t &&) = default; jit_uni_pooling_bwd_t &operator=(jit_uni_pooling_bwd_t &&) = default; - ~jit_uni_pooling_bwd_t(); + ~jit_uni_pooling_bwd_t() override; using data_t = typename prec_traits_t::type; diff --git a/src/cpu/x64/jit_uni_reduction.hpp b/src/cpu/x64/jit_uni_reduction.hpp index 844fa730413..7b78568b1bf 100644 --- a/src/cpu/x64/jit_uni_reduction.hpp +++ b/src/cpu/x64/jit_uni_reduction.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2021-2022 Intel Corporation +* Copyright 2021-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,7 +49,8 @@ struct jit_uni_reduction_t : public primitive_t { }; jit_uni_reduction_t(const pd_t *apd) : primitive_t(apd) {} - virtual ~jit_uni_reduction_t() = default; + + ~jit_uni_reduction_t() override = default; status_t init(engine_t *engine) override; status_t execute(const exec_ctx_t &ctx) const override; diff --git a/src/cpu/x64/jit_uni_reduction_kernel.hpp b/src/cpu/x64/jit_uni_reduction_kernel.hpp index 57d2393ce2a..c844e5d7935 100644 --- a/src/cpu/x64/jit_uni_reduction_kernel.hpp +++ b/src/cpu/x64/jit_uni_reduction_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2021-2024 Intel Corporation +* Copyright 2021-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -40,7 +40,7 @@ struct jit_uni_reduction_kernel_base_t : public jit_generator { : jit_generator(jit_name(), conf.isa) , conf_(conf) , sum_scales_(conf_.sum_scales) {} - virtual ~jit_uni_reduction_kernel_base_t() = default; + ~jit_uni_reduction_kernel_base_t() override = default; virtual std::size_t get_simd_w() = 0; @@ -54,7 +54,7 @@ struct jit_uni_reduction_kernel_t : public jit_uni_reduction_kernel_base_t { jit_uni_reduction_kernel_t( const jit_reduction_conf_t &conf, const memory_desc_t *dst_md); - virtual ~jit_uni_reduction_kernel_t() = default; + ~jit_uni_reduction_kernel_t() override = default; std::size_t get_simd_w() override { return simd_w_; } diff --git a/src/cpu/x64/jit_uni_reorder.hpp b/src/cpu/x64/jit_uni_reorder.hpp index 806155825ec..e8f4ae27383 100644 --- a/src/cpu/x64/jit_uni_reorder.hpp +++ b/src/cpu/x64/jit_uni_reorder.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2018-2024 Intel Corporation +* Copyright 2018-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -185,7 +185,7 @@ struct kernel_t { virtual void operator()(const call_param_t *c) const = 0; virtual void operator()(const tail_call_param_t *c) const = 0; virtual status_t create_kernel() = 0; - virtual ~kernel_t() {} + virtual ~kernel_t() = default; /** inits kernel descriptor: * desc -- kernel descriptor (output) @@ -297,7 +297,7 @@ struct jit_blk_reorder_t : public primitive_t { status_t execute(const exec_ctx_t &ctx) const override; jit_blk_reorder_t(const pd_t *apd); - ~jit_blk_reorder_t(); + ~jit_blk_reorder_t() override; private: const pd_t *pd() const { return (const pd_t *)primitive_t::pd().get(); } diff --git a/src/cpu/x64/jit_uni_resampling.hpp b/src/cpu/x64/jit_uni_resampling.hpp index 0676475e221..5e764cb82d5 100644 --- a/src/cpu/x64/jit_uni_resampling.hpp +++ b/src/cpu/x64/jit_uni_resampling.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2021 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,7 +50,8 @@ struct jit_uni_resampling_fwd_t : public primitive_t { }; jit_uni_resampling_fwd_t(const pd_t *apd) : primitive_t(apd) {} - virtual ~jit_uni_resampling_fwd_t() = default; + + ~jit_uni_resampling_fwd_t() override = default; status_t init(engine_t *engine) override; status_t execute(const exec_ctx_t &ctx) const override; diff --git a/src/cpu/x64/jit_uni_resampling_kernel.hpp b/src/cpu/x64/jit_uni_resampling_kernel.hpp index e9dc0d546bb..d1b807e1665 100644 --- a/src/cpu/x64/jit_uni_resampling_kernel.hpp +++ b/src/cpu/x64/jit_uni_resampling_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,7 +42,7 @@ struct jit_uni_resampling_kernel_base_t : public jit_generator { , conf_(conf) , sum_scales_(conf_.sum_scales) {} - virtual ~jit_uni_resampling_kernel_base_t() = default; + ~jit_uni_resampling_kernel_base_t() override = default; virtual std::size_t get_simd_w() = 0; @@ -57,7 +57,7 @@ struct jit_uni_resampling_kernel_t : public jit_uni_resampling_kernel_base_t { jit_uni_resampling_kernel_t( const jit_resampling_conf_t &conf, const memory_desc_t *dst_md); - virtual ~jit_uni_resampling_kernel_t() = default; + ~jit_uni_resampling_kernel_t() override = default; std::size_t get_simd_w() override { return simd_w_; } diff --git a/src/cpu/x64/jit_uni_tbb_batch_normalization.hpp b/src/cpu/x64/jit_uni_tbb_batch_normalization.hpp index f205338c574..777937c0174 100644 --- a/src/cpu/x64/jit_uni_tbb_batch_normalization.hpp +++ b/src/cpu/x64/jit_uni_tbb_batch_normalization.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -62,7 +62,8 @@ struct jit_uni_tbb_batch_normalization_fwd_t : public primitive_t { }; jit_uni_tbb_batch_normalization_fwd_t(const pd_t *apd); - ~jit_uni_tbb_batch_normalization_fwd_t(); + + ~jit_uni_tbb_batch_normalization_fwd_t() override; status_t init(engine_t *engine) override; @@ -98,7 +99,8 @@ struct jit_uni_tbb_batch_normalization_bwd_t : public primitive_t { }; jit_uni_tbb_batch_normalization_bwd_t(const pd_t *apd); - ~jit_uni_tbb_batch_normalization_bwd_t(); + + ~jit_uni_tbb_batch_normalization_bwd_t() override; status_t init(engine_t *engine) override; diff --git a/src/cpu/x64/jit_uni_x8s8s32x_deconvolution.hpp b/src/cpu/x64/jit_uni_x8s8s32x_deconvolution.hpp index b72708a9725..6e1eb663f3c 100644 --- a/src/cpu/x64/jit_uni_x8s8s32x_deconvolution.hpp +++ b/src/cpu/x64/jit_uni_x8s8s32x_deconvolution.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -50,7 +50,8 @@ struct _jit_uni_x8s8s32x_deconv_fwd_kernel : public jit_generator { _jit_uni_x8s8s32x_deconv_fwd_kernel(const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_wrapper &dst_d); - ~_jit_uni_x8s8s32x_deconv_fwd_kernel(); + + ~_jit_uni_x8s8s32x_deconv_fwd_kernel() override; const jit_conv_conf_t jcp_ = utils::zero(); @@ -197,7 +198,8 @@ struct jit_uni_x8s8s32x_deconvolution_fwd_t : public primitive_t { }; jit_uni_x8s8s32x_deconvolution_fwd_t(const pd_t *apd); - ~jit_uni_x8s8s32x_deconvolution_fwd_t(); + + ~jit_uni_x8s8s32x_deconvolution_fwd_t() override; status_t init(engine_t *engine) override; status_t execute(const exec_ctx_t &ctx) const override; diff --git a/src/cpu/x64/jit_uni_xf16_sum.hpp b/src/cpu/x64/jit_uni_xf16_sum.hpp index d199169e55b..a45c4c1ea18 100644 --- a/src/cpu/x64/jit_uni_xf16_sum.hpp +++ b/src/cpu/x64/jit_uni_xf16_sum.hpp @@ -59,7 +59,7 @@ struct jit_uni_xf16_sum_kernel_t : public jit_generator { , reg_src {r8, r9, r10, r11, r12, r13, r14, r15} , num_acc_iters(num_acc_iters) {} - ~jit_uni_xf16_sum_kernel_t() {} + ~jit_uni_xf16_sum_kernel_t() override = default; DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_xf16_sum_kernel_t) @@ -113,7 +113,7 @@ struct jit_avx512_core_bf16_sum_kernel_t bf16_emu_reserved_5); } - ~jit_avx512_core_bf16_sum_kernel_t() = default; + ~jit_avx512_core_bf16_sum_kernel_t() override = default; DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_bf16_sum_kernel_t) @@ -204,7 +204,7 @@ struct jit_avx2_vnni_2_xf16_sum_kernel_t jit_avx2_vnni_2_xf16_sum_kernel_t(jit_sum_conf_t ajsp) : jit_uni_xf16_sum_kernel_t(ajsp, ajsp.num_srcs) {} - ~jit_avx2_vnni_2_xf16_sum_kernel_t() {} + ~jit_avx2_vnni_2_xf16_sum_kernel_t() override = default; DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_vnni_2_xf16_sum_kernel_t) diff --git a/src/cpu/x64/lrn/jit_avx512_common_lrn.hpp b/src/cpu/x64/lrn/jit_avx512_common_lrn.hpp index e491f521b13..f2c4f704c1c 100644 --- a/src/cpu/x64/lrn/jit_avx512_common_lrn.hpp +++ b/src/cpu/x64/lrn/jit_avx512_common_lrn.hpp @@ -52,7 +52,8 @@ struct jit_avx512_common_lrn_fwd_t : public primitive_t { }; jit_avx512_common_lrn_fwd_t(const pd_t *apd); - ~jit_avx512_common_lrn_fwd_t(); + + ~jit_avx512_common_lrn_fwd_t() override; using data_t = typename prec_traits_t::type; @@ -91,7 +92,8 @@ struct jit_avx512_common_lrn_bwd_t : public primitive_t { }; jit_avx512_common_lrn_bwd_t(const pd_t *apd); - ~jit_avx512_common_lrn_bwd_t(); + + ~jit_avx512_common_lrn_bwd_t() override; using data_t = typename prec_traits_t::type; diff --git a/src/cpu/x64/lrn/jit_uni_lrn.hpp b/src/cpu/x64/lrn/jit_uni_lrn.hpp index 7d7e2e417b0..7d35c5bb366 100644 --- a/src/cpu/x64/lrn/jit_uni_lrn.hpp +++ b/src/cpu/x64/lrn/jit_uni_lrn.hpp @@ -46,7 +46,8 @@ struct jit_uni_lrn_fwd_t : public primitive_t { }; jit_uni_lrn_fwd_t(const pd_t *apd); - ~jit_uni_lrn_fwd_t(); + + ~jit_uni_lrn_fwd_t() override; using data_t = typename prec_traits_t::type; @@ -80,7 +81,8 @@ struct jit_uni_lrn_bwd_t : public primitive_t { }; jit_uni_lrn_bwd_t(const pd_t *apd); - ~jit_uni_lrn_bwd_t(); + + ~jit_uni_lrn_bwd_t() override; using data_t = typename prec_traits_t::type; diff --git a/src/cpu/x64/lrn/jit_uni_lrn_kernel.hpp b/src/cpu/x64/lrn/jit_uni_lrn_kernel.hpp index 72191ee281a..e445518f636 100644 --- a/src/cpu/x64/lrn/jit_uni_lrn_kernel.hpp +++ b/src/cpu/x64/lrn/jit_uni_lrn_kernel.hpp @@ -90,7 +90,7 @@ class jit_uni_lrn_kernel_t> : public jit_generator { jit_uni_lrn_kernel_t( const within_config_t &J, const char *name = jit_name()); - ~jit_uni_lrn_kernel_t(); + ~jit_uni_lrn_kernel_t() override; DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_lrn_kernel_t); // TODO: why use double simd for sse41? @@ -140,7 +140,7 @@ class jit_uni_lrn_fwd_kernel_t const nhwc_across_t &J, float A, float K, prop_kind_t pk); jit_uni_lrn_fwd_kernel_t( const nchw_across_t &J, float A, float K, prop_kind_t pk); - ~jit_uni_lrn_fwd_kernel_t(); + ~jit_uni_lrn_fwd_kernel_t() override; private: using Base = jit_uni_lrn_kernel_t>; diff --git a/src/cpu/x64/lrn/lrn_avx512_nhwc_executor.hpp b/src/cpu/x64/lrn/lrn_avx512_nhwc_executor.hpp index 2db6e27ce5c..d7b8eabd882 100644 --- a/src/cpu/x64/lrn/lrn_avx512_nhwc_executor.hpp +++ b/src/cpu/x64/lrn/lrn_avx512_nhwc_executor.hpp @@ -73,7 +73,7 @@ class lrn_avx512_nhwc_executor_fwd_t : public i_lrn_executor_t { return status::success; } - virtual ~lrn_avx512_nhwc_executor_fwd_t() = default; + ~lrn_avx512_nhwc_executor_fwd_t() override = default; private: std::unique_ptr> ker_; @@ -126,7 +126,7 @@ class lrn_avx512_nhwc_executor_bwd_t : public i_lrn_executor_t { return status::success; } - virtual ~lrn_avx512_nhwc_executor_bwd_t() = default; + ~lrn_avx512_nhwc_executor_bwd_t() override = default; private: std::unique_ptr> ker_; diff --git a/src/cpu/x64/matmul/brgemm_matmul_copy_utils.hpp b/src/cpu/x64/matmul/brgemm_matmul_copy_utils.hpp index 36efe8d7859..eacf8f52da2 100644 --- a/src/cpu/x64/matmul/brgemm_matmul_copy_utils.hpp +++ b/src/cpu/x64/matmul/brgemm_matmul_copy_utils.hpp @@ -47,7 +47,7 @@ struct jit_brgemm_matmul_copy_b_t { jit_brgemm_matmul_copy_b_t(const brgemm_matmul_conf_t *conf) : conf_(conf) {} - virtual ~jit_brgemm_matmul_copy_b_t() {} + virtual ~jit_brgemm_matmul_copy_b_t() = default; const brgemm_matmul_conf_t *conf_; }; @@ -72,7 +72,7 @@ struct jit_brgemm_matmul_copy_a_t { jit_brgemm_matmul_copy_a_t(const brgemm_matmul_conf_t *conf) : conf_(conf) {} - virtual ~jit_brgemm_matmul_copy_a_t() {} + virtual ~jit_brgemm_matmul_copy_a_t() = default; const brgemm_matmul_conf_t *conf_; }; diff --git a/src/cpu/x64/prelu/jit_prelu_backward.hpp b/src/cpu/x64/prelu/jit_prelu_backward.hpp index f51988c4867..7a79faef6cc 100644 --- a/src/cpu/x64/prelu/jit_prelu_backward.hpp +++ b/src/cpu/x64/prelu/jit_prelu_backward.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2022 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,7 +51,7 @@ class jit_prelu_bwd_t : public primitive_t { }; jit_prelu_bwd_t(const pd_t *apd); - ~jit_prelu_bwd_t(); + ~jit_prelu_bwd_t() override; status_t init(engine_t *engine) override; status_t execute(const exec_ctx_t &ctx) const override; diff --git a/src/cpu/x64/prelu/jit_prelu_forward.hpp b/src/cpu/x64/prelu/jit_prelu_forward.hpp index 30c0b5f4c34..45768f86e05 100644 --- a/src/cpu/x64/prelu/jit_prelu_forward.hpp +++ b/src/cpu/x64/prelu/jit_prelu_forward.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2021 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,7 +45,7 @@ class jit_prelu_fwd_t : public primitive_t { }; jit_prelu_fwd_t(const pd_t *apd); - ~jit_prelu_fwd_t(); + ~jit_prelu_fwd_t() override; status_t init(engine_t *engine) override; status_t execute(const exec_ctx_t &ctx) const override; diff --git a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_bwd.hpp b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_bwd.hpp index 5abc8e17102..bc783e89c5e 100644 --- a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_bwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_bwd.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2023 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,7 @@ struct jit_uni_gru_cell_postgemm_part1_bwd : public jit_uni_rnn_postgemm { const rnn_utils::rnn_conf_t &rnn, const rnn_pd_t *pd) : jit_uni_rnn_postgemm(rnn, pd, jit_name()) {} - ~jit_uni_gru_cell_postgemm_part1_bwd() {} + ~jit_uni_gru_cell_postgemm_part1_bwd() override = default; status_t init(data_type_t sdt) override { CHECK(jit_uni_rnn_postgemm::init(src_data_t)); diff --git a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_bwd.hpp b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_bwd.hpp index 6922969268a..dd89b5deab0 100644 --- a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_bwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_bwd.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2023 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,7 @@ struct jit_uni_gru_cell_postgemm_part2_bwd : public jit_uni_rnn_postgemm { const rnn_utils::rnn_conf_t &rnn, const rnn_pd_t *pd) : jit_uni_rnn_postgemm(rnn, pd, jit_name()) {} - ~jit_uni_gru_cell_postgemm_part2_bwd() {} + ~jit_uni_gru_cell_postgemm_part2_bwd() override = default; status_t init(data_type_t sdt) override { CHECK(jit_uni_rnn_postgemm::init(src_data_t)); diff --git a/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_bwd.hpp b/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_bwd.hpp index 4e8a5a04904..9af12243f26 100644 --- a/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_bwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_bwd.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2023 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,7 @@ struct jit_uni_gru_lbr_cell_postgemm_bwd : public jit_uni_rnn_postgemm { const rnn_utils::rnn_conf_t &rnn, const rnn_pd_t *pd) : jit_uni_rnn_postgemm(rnn, pd, jit_name()) {} - ~jit_uni_gru_lbr_cell_postgemm_bwd() {} + ~jit_uni_gru_lbr_cell_postgemm_bwd() override = default; status_t init(data_type_t sdt) override { CHECK(jit_uni_rnn_postgemm::init(src_data_t)); diff --git a/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_bwd.hpp b/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_bwd.hpp index 44dcc92c2db..6d7590a0c0a 100644 --- a/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_bwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_bwd.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,7 +45,8 @@ struct jit_uni_lstm_cell_postgemm_bwd // stage src_data_t == data_type::bf16 && !mayiuse(avx512_core_bf16)) { } - ~jit_uni_lstm_cell_postgemm_bwd() = default; + + ~jit_uni_lstm_cell_postgemm_bwd() override = default; status_t init(data_type_t sdt) override { CHECK(jit_uni_rnn_postgemm::init(src_data_t)); diff --git a/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_fwd.hpp b/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_fwd.hpp index 010c0c6e324..b61dc40bc4b 100644 --- a/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_fwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_fwd.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,7 +46,7 @@ struct jit_uni_lstm_cell_postgemm_fwd src_data_t == data_type::bf16 && !mayiuse(avx512_core_bf16)) { } - ~jit_uni_lstm_cell_postgemm_fwd() = default; + ~jit_uni_lstm_cell_postgemm_fwd() override = default; status_t init(data_type_t sdt) override { CHECK(jit_uni_rnn_postgemm::init(src_data_t)); diff --git a/src/cpu/x64/rnn/jit_uni_lstm_cell_projection_postgemm_fwd.hpp b/src/cpu/x64/rnn/jit_uni_lstm_cell_projection_postgemm_fwd.hpp index d82ebd9f069..fa03c716d34 100644 --- a/src/cpu/x64/rnn/jit_uni_lstm_cell_projection_postgemm_fwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_lstm_cell_projection_postgemm_fwd.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,7 @@ struct jit_uni_lstm_cell_projection_postgemm_fwd : public jit_uni_rnn_postgemm { const rnn_utils::rnn_conf_t &rnn, const rnn_pd_t *pd) : jit_uni_rnn_postgemm(rnn, pd, jit_name()) {} - ~jit_uni_lstm_cell_projection_postgemm_fwd() {} + ~jit_uni_lstm_cell_projection_postgemm_fwd() override = default; status_t init(data_type_t sdt) override { jit_uni_rnn_postgemm::init(src_data_t); @@ -51,7 +51,7 @@ struct jit_uni_lstm_cell_projection_postgemm_fwd : public jit_uni_rnn_postgemm { const size_t hstate_dt_size = types::data_type_size(src_data_t); const size_t scratch_dt_size = types::data_type_size(scratch_data_t); - void generate() { + void generate() override { using namespace Xbyak; // Labels declaration diff --git a/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_bwd.hpp b/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_bwd.hpp index 514983e7cb2..73838aa8874 100644 --- a/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_bwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_bwd.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2023 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,7 @@ struct jit_uni_rnn_cell_postgemm_bwd : public jit_uni_rnn_postgemm { const rnn_utils::rnn_conf_t &rnn, const rnn_pd_t *pd) : jit_uni_rnn_postgemm(rnn, pd, jit_name()) {} - ~jit_uni_rnn_cell_postgemm_bwd() {} + ~jit_uni_rnn_cell_postgemm_bwd() override = default; status_t init(data_type_t sdt) override { CHECK(jit_uni_rnn_postgemm::init(src_data_t)); diff --git a/src/cpu/x64/rnn/jit_uni_rnn_common_postgemm.hpp b/src/cpu/x64/rnn/jit_uni_rnn_common_postgemm.hpp index 3246142aa62..0df5757cd70 100644 --- a/src/cpu/x64/rnn/jit_uni_rnn_common_postgemm.hpp +++ b/src/cpu/x64/rnn/jit_uni_rnn_common_postgemm.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -69,7 +69,7 @@ struct jit_uni_rnn_postgemm : public jit_generator { , zmm_tail_k_mask(k3) , xf16_dq_reg_idx(tmp_vector_register_idx) {} - ~jit_uni_rnn_postgemm() { + ~jit_uni_rnn_postgemm() override { if (bf16_emu_) delete bf16_emu_; } diff --git a/src/cpu/x64/shuffle/jit_uni_shuffle.hpp b/src/cpu/x64/shuffle/jit_uni_shuffle.hpp index bbc6707d387..42c3612ee7a 100644 --- a/src/cpu/x64/shuffle/jit_uni_shuffle.hpp +++ b/src/cpu/x64/shuffle/jit_uni_shuffle.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2021 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -51,7 +51,7 @@ struct jit_uni_shuffle_t : public primitive_t { jit_uni_shuffle_t(const pd_t *apd); - ~jit_uni_shuffle_t(); + ~jit_uni_shuffle_t() override; status_t init(engine_t *engine) override; From e4472bd44b768ef5f860a990cb7447e0a7bbcda9 Mon Sep 17 00:00:00 2001 From: Dmitrii Zarukin Date: Mon, 17 Mar 2025 12:15:28 -0700 Subject: [PATCH 02/16] cpu: tidy: readability-static-definition-in-anonymous-namespace --- src/cpu/x64/cpu_isa_traits.hpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/cpu/x64/cpu_isa_traits.hpp b/src/cpu/x64/cpu_isa_traits.hpp index 189e030ac31..31a6ac47732 100644 --- a/src/cpu/x64/cpu_isa_traits.hpp +++ b/src/cpu/x64/cpu_isa_traits.hpp @@ -354,7 +354,7 @@ bool DNNL_API is_available(); namespace { -static inline bool mayiuse(const cpu_isa_t cpu_isa, bool soft = false) { +inline bool mayiuse(const cpu_isa_t cpu_isa, bool soft = false) { using namespace Xbyak::util; unsigned cpu_isa_mask = x64::get_max_cpu_isa_mask(soft); @@ -418,23 +418,23 @@ static inline bool mayiuse(const cpu_isa_t cpu_isa, bool soft = false) { return false; } -static inline bool isa_has_int8_vnni(cpu_isa_t isa) { +inline bool isa_has_int8_vnni(cpu_isa_t isa) { return is_superset(isa, avx512_core_vnni) || is_superset(isa, avx2_vnni); } -static inline bool isa_has_s8s8(cpu_isa_t isa) { +inline bool isa_has_s8s8(cpu_isa_t isa) { return is_superset(isa, amx_int8) || is_superset(isa, avx2_vnni_2); } -static inline bool isa_has_bf16(cpu_isa_t isa) { +inline bool isa_has_bf16(cpu_isa_t isa) { return is_superset(isa, avx512_core_bf16); } -static inline bool isa_has_masks(cpu_isa_t isa) { +inline bool isa_has_masks(cpu_isa_t isa) { return is_superset(isa, avx512_core); } -static inline int isa_max_vlen(cpu_isa_t isa) { +inline int isa_max_vlen(cpu_isa_t isa) { const bool is_avx512 = is_superset(isa, avx512_core); const bool is_avx = is_superset(isa, avx); const bool is_sse41 = is_superset(isa, sse41); @@ -450,7 +450,7 @@ static inline int isa_max_vlen(cpu_isa_t isa) { return cpu_isa_traits::vlen; } -static inline int isa_num_vregs(cpu_isa_t isa) { +inline int isa_num_vregs(cpu_isa_t isa) { const bool is_avx512 = is_superset(isa, avx512_core); const bool is_avx = is_superset(isa, avx); const bool is_sse41 = is_superset(isa, sse41); From 1b146d6a5cbbc9584f92a509e07c91859e776aa3 Mon Sep 17 00:00:00 2001 From: Dmitrii Zarukin Date: Mon, 17 Mar 2025 12:37:24 -0700 Subject: [PATCH 03/16] cpu: tidy: readability-make-member-function-const --- src/cpu/x64/jit_avx2_conv_kernel_f32.hpp | 34 +++++++++---------- src/cpu/x64/jit_avx512_common_conv_kernel.hpp | 6 ++-- .../x64/jit_avx512_core_amx_conv_kernel.hpp | 28 ++++++++------- .../x64/jit_avx512_core_amx_conv_utils.hpp | 12 +++---- .../jit_avx512_core_bf16_1x1_conv_kernel.hpp | 8 ++--- .../jit_avx512_core_bf16_dw_conv_kernel.hpp | 20 +++++------ src/cpu/x64/jit_brgemm_transpose_utils.hpp | 2 +- src/cpu/x64/jit_generator.hpp | 4 ++- src/cpu/x64/jit_sse41_conv_kernel_f32.hpp | 12 +++---- 9 files changed, 66 insertions(+), 60 deletions(-) diff --git a/src/cpu/x64/jit_avx2_conv_kernel_f32.hpp b/src/cpu/x64/jit_avx2_conv_kernel_f32.hpp index 9050c113a2f..4fc17c34e03 100644 --- a/src/cpu/x64/jit_avx2_conv_kernel_f32.hpp +++ b/src/cpu/x64/jit_avx2_conv_kernel_f32.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2016-2023 Intel Corporation +* Copyright 2016-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -87,18 +87,18 @@ struct jit_avx2_conv_fwd_kernel_f32 : public jit_generator { inline void width_blk_step(int ur_w, int pad_l, int pad_r, int oc_blocks); inline void solve_common(int oc_blocks); - inline dim_t filter_w_to_input(int ki, int oi = 0, int pad_l = 0) { + inline dim_t filter_w_to_input(int ki, int oi = 0, int pad_l = 0) const { return static_cast(ki) * (jcp.dilate_w + 1) + oi * jcp.stride_w - pad_l; }; - inline dim_t filter_h_to_input(int ki) { + inline dim_t filter_h_to_input(int ki) const { return static_cast(ki) * (jcp.dilate_h + 1) * jcp.iw; }; - inline dim_t filter_d_to_input(int ki) { + inline dim_t filter_d_to_input(int ki) const { return static_cast(ki) * (jcp.dilate_d + 1) * jcp.iw * jcp.ih; }; - inline dim_t get_input_offset(int i_ic, int i_iw) { + inline dim_t get_input_offset(int i_ic, int i_iw) const { dim_t offset; if (utils::one_of(jcp.src_tag, format_tag::ncw, format_tag::nchw, format_tag::ncdhw)) { @@ -112,7 +112,7 @@ struct jit_avx2_conv_fwd_kernel_f32 : public jit_generator { return sizeof(float) * offset; } - inline dim_t get_output_offset(int i_oc_block, int i_ow) { + inline dim_t get_output_offset(int i_oc_block, int i_ow) const { dim_t offset; if (utils::one_of(jcp.dst_tag, format_tag::nwc, format_tag::nhwc, format_tag::ndhwc)) { @@ -126,7 +126,7 @@ struct jit_avx2_conv_fwd_kernel_f32 : public jit_generator { return sizeof(float) * offset; } - inline dim_t get_kernel_offset(int i_oc_block, int ki, int i_ic) { + inline dim_t get_kernel_offset(int i_oc_block, int ki, int i_ic) const { dim_t block_step_size = jcp.ic_block * jcp.oc_block; dim_t ic_block_step_size = static_cast(jcp.kd) * jcp.kh * jcp.kw * block_step_size; @@ -137,7 +137,7 @@ struct jit_avx2_conv_fwd_kernel_f32 : public jit_generator { return sizeof(float) * offset; } - inline bool is_src_layout_nxc() { + inline bool is_src_layout_nxc() const { return utils::one_of(jcp.src_tag, format_tag::ndhwc, format_tag::nhwc, format_tag::nwc); } @@ -189,7 +189,7 @@ struct jit_avx2_conv_bwd_data_kernel_f32 : public jit_generator { void generate() override; - inline int get_iw_start(int ki, int l_overflow) { + inline int get_iw_start(int ki, int l_overflow) const { int res = (jcp.iw - 1 + jcp.r_pad) % jcp.stride_w + l_overflow * jcp.stride_w - (jcp.kw - 1 - ki) * (jcp.dilate_w + 1); @@ -199,7 +199,7 @@ struct jit_avx2_conv_bwd_data_kernel_f32 : public jit_generator { return res; } - inline int get_iw_end(int ur_w, int ki, int r_overflow) { + inline int get_iw_end(int ur_w, int ki, int r_overflow) const { if (utils::one_of(ur_w, jcp.iw, jcp.ur_w_tail)) ur_w += nstl::min(0, jcp.r_pad); // remove negative padding int res = (ur_w - 1 + jcp.l_pad) % jcp.stride_w @@ -210,11 +210,11 @@ struct jit_avx2_conv_bwd_data_kernel_f32 : public jit_generator { return ur_w - res; } - inline dim_t filter_w_to_ddst(int ki, int oi = 0, int pad_l = 0) { + inline dim_t filter_w_to_ddst(int ki, int oi = 0, int pad_l = 0) const { return (oi + pad_l - ki * (jcp.dilate_w + 1)) / jcp.stride_w; } - inline dim_t get_ddst_offset(int i_oc_block, int i_ow, int i_oc) { + inline dim_t get_ddst_offset(int i_oc_block, int i_ow, int i_oc) const { dim_t offset; if (utils::one_of(jcp.dst_tag, format_tag::nwc, format_tag::nhwc, format_tag::ndhwc)) { @@ -228,7 +228,7 @@ struct jit_avx2_conv_bwd_data_kernel_f32 : public jit_generator { return sizeof(float) * offset; } - inline dim_t get_dsrc_offset(int i_ic_block, int i_iw) { + inline dim_t get_dsrc_offset(int i_ic_block, int i_iw) const { dim_t offset; if (utils::one_of(jcp.src_tag, format_tag::nwc, format_tag::nhwc, format_tag::ndhwc)) { @@ -243,7 +243,7 @@ struct jit_avx2_conv_bwd_data_kernel_f32 : public jit_generator { } inline dim_t get_kernel_offset( - int i_oc_block, int i_ic_block, int ki, int i_oc) { + int i_oc_block, int i_ic_block, int ki, int i_oc) const { dim_t block_step_size = jcp.ic_block * jcp.oc_block; dim_t ic_block_step_size = static_cast(jcp.kd) * jcp.kh * jcp.kw * block_step_size; @@ -300,7 +300,7 @@ struct jit_avx2_conv_bwd_weights_kernel_f32 : public jit_generator { inline void compute_oh_step_common(int ic_block_step, int max_ur_w); inline void compute_oh_loop_common(); - inline dim_t get_input_offset(int i_ic, int i_iw) { + inline dim_t get_input_offset(int i_ic, int i_iw) const { dim_t offset; if (utils::one_of(jcp.src_tag, format_tag::ncw, format_tag::nchw, format_tag::ncdhw)) { @@ -314,7 +314,7 @@ struct jit_avx2_conv_bwd_weights_kernel_f32 : public jit_generator { return sizeof(float) * offset; } - inline dim_t get_output_offset(int i_oc_block, int i_ow) { + inline dim_t get_output_offset(int i_oc_block, int i_ow) const { dim_t offset; if (utils::one_of(jcp.dst_tag, format_tag::nwc, format_tag::nhwc, format_tag::ndhwc)) { @@ -328,7 +328,7 @@ struct jit_avx2_conv_bwd_weights_kernel_f32 : public jit_generator { return sizeof(float) * offset; } - inline dim_t get_kernel_offset(int ki, int i_ic) { + inline dim_t get_kernel_offset(int ki, int i_ic) const { dim_t block_step_size = jcp.ic_block * jcp.oc_block; dim_t offset = static_cast(ki) * block_step_size + i_ic * jcp.oc_block; diff --git a/src/cpu/x64/jit_avx512_common_conv_kernel.hpp b/src/cpu/x64/jit_avx512_common_conv_kernel.hpp index c741eeb4f8b..f4bc66a0b52 100644 --- a/src/cpu/x64/jit_avx512_common_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_common_conv_kernel.hpp @@ -471,11 +471,11 @@ struct jit_avx512_common_conv_bwd_weights_kernel_f32 : public jit_generator { inline void compute_od_loop_partial(); inline void compute_loop(); - inline bool is_src_layout_nxc() { + inline bool is_src_layout_nxc() const { return utils::one_of(jcp.src_tag, format_tag::ndhwc, format_tag::nhwc, format_tag::nwc); } - inline bool is_ddst_layout_nxc() { + inline bool is_ddst_layout_nxc() const { return utils::one_of(jcp.dst_tag, format_tag::ndhwc, format_tag::nhwc, format_tag::nwc); } @@ -494,7 +494,7 @@ struct jit_avx512_common_conv_bwd_weights_kernel_f32 : public jit_generator { return input_offset + typesize * local_input_offset; }; - inline int get_iw_idx(int ow, int kw, int l_pad) { + inline int get_iw_idx(int ow, int kw, int l_pad) const { return ow * jcp.stride_w + kw * (jcp.dilate_w + 1) - l_pad; } diff --git a/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp index fcd30e2316b..b7c37723ae9 100644 --- a/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp @@ -89,16 +89,16 @@ struct jit_avx512_core_amx_compute_zp_pbuff_t : public jit_generator { void generate() override; - Xbyak::Zmm zmm_out(int i_ur, int i_oc) { + Xbyak::Zmm zmm_out(int i_ur, int i_oc) const { int idx = i_ur * jcp.nb_oc_blocking + i_oc; assert(idx < max_regs_ur); return Xbyak::Zmm(idx); } - int get_ow_start(int ki, int pad_l) { + int get_ow_start(int ki, int pad_l) const { return nstl::max(0, utils::div_up(pad_l - ki * (jcp.dilate_w + 1), jcp.stride_w)); } - int get_ow_end(int ur_w, int ki, int pad_r) { + int get_ow_end(int ur_w, int ki, int pad_r) const { int filter_overlap = pad_r - (jcp.kw - 1 - ki) * (jcp.dilate_w + 1); return ur_w - nstl::max(0, utils::div_up(filter_overlap, jcp.stride_w)); } @@ -373,7 +373,7 @@ struct jit_avx512_core_amx_fwd_kernel_t : public jit_generator { const int s_pad_output, const int e_pad_output); void cvt2ps(data_type_t type_in, const Xbyak::Zmm &ymm_in, const Xbyak::Operand &op, bool mask_flag = false); - Xbyak::Zmm zmm_out(const int idx) { + Xbyak::Zmm zmm_out(const int idx) const { const int upper_limit = jcp.src_dt == data_type::bf16 ? zmm_idx_limit_bf16 : zmm_idx_limit_int8; @@ -578,7 +578,7 @@ struct jit_avx512_core_amx_bwd_data_kernel_t : public jit_generator { int get_inp_tensor(int h) const; int get_wei_tensor(int i) const; - inline bool gaps_in_store() { + inline bool gaps_in_store() const { const int gen_kd = (jcp.kd - 1) * (jcp.dilate_d + 1) + 1; return gen_kd < jcp.stride_d || jcp.dilate_d > 0; } @@ -697,32 +697,36 @@ struct jit_avx512_core_amx_bwd_weights_kernel_t : public jit_generator { static void balance(const jit_conv_conf_t &j, int &nthr, int &nthr_mb, int &nthr_g, int &nthr_oc_b, int &nthr_ic_b); - inline dim_t filter_w_to_src(int kw, int ow = 0, int pad_l = 0) { + inline dim_t filter_w_to_src(int kw, int ow = 0, int pad_l = 0) const { return static_cast(kw) * (jcp.dilate_w + 1) + ow - pad_l; } - inline dim_t filter_h_to_src(int kh) { return kh * (jcp.dilate_h + 1); } - inline dim_t filter_d_to_src(int kd) { + inline dim_t filter_h_to_src(int kh) const { + return kh * (jcp.dilate_h + 1); + } + inline dim_t filter_d_to_src(int kd) const { return static_cast(kd) * (jcp.dilate_d + 1) * jcp.ih; } - inline dim_t get_src_offset(dim_t ic_idx, dim_t w_idx, dim_t hd_idx = 0) { + inline dim_t get_src_offset( + dim_t ic_idx, dim_t w_idx, dim_t hd_idx = 0) const { return static_cast(jcp.typesize_in) * (hd_idx * jcp.tr_iw * jcp.ic_block + jcp.tr_iw * ic_idx + w_idx); } - inline dim_t get_ddst_offset(dim_t w_idx, dim_t hd_idx = 0) { + inline dim_t get_ddst_offset(dim_t w_idx, dim_t hd_idx = 0) const { int ow_per_oc = 2; dim_t w_off = w_idx / ow_per_oc * ow_per_oc * jcp.oc_block + w_idx % ow_per_oc; return jcp.typesize_in * (w_off + jcp.tr_ow * jcp.oc_block * hd_idx); } - inline dim_t get_kernel_offset(int ic_idx, dim_t ksp_idx) { + inline dim_t get_kernel_offset(int ic_idx, dim_t ksp_idx) const { return jcp.typesize_out * jcp.oc_block * (ksp_idx * jcp.ic_block + ic_idx); } - inline dim_t get_full_kernel_offset(int ocb, int icb, int kh, int kw) { + inline dim_t get_full_kernel_offset( + int ocb, int icb, int kh, int kw) const { return jcp.typesize_out * (static_cast(ocb) * jcp.nb_ic * jcp.kd * jcp.kh * jcp.kw * jcp.ic_block * jcp.oc_block diff --git a/src/cpu/x64/jit_avx512_core_amx_conv_utils.hpp b/src/cpu/x64/jit_avx512_core_amx_conv_utils.hpp index 9578db8ef7b..22f4536e7b0 100644 --- a/src/cpu/x64/jit_avx512_core_amx_conv_utils.hpp +++ b/src/cpu/x64/jit_avx512_core_amx_conv_utils.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2021-2022 Intel Corporation +* Copyright 2021-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,7 +48,7 @@ struct spatial_features_3d { , init_overflow_(0) , end_overflow_(0) {} - inline int get_init_overflow(const int in) { + inline int get_init_overflow(const int in) const { if (is_fast_path_) return nstl::max(0, filter_size_ - 1 - in - init_pad_); if (dilate_ != 1) @@ -58,7 +58,7 @@ struct spatial_features_3d { return nstl::max(0, (filter_size_ - 1 - in - init_pad_) / stride_); } - inline int get_end_overflow(const int in) { + inline int get_end_overflow(const int in) const { if (is_fast_path_) return nstl::max(0, filter_size_ - input_size_ + in - end_pad_); if (dilate_ != 1) @@ -97,13 +97,13 @@ struct spatial_features_3d { : in + init_pad_ - end_overflow_ * dilate_; } - inline int get_filter_padding() { + inline int get_filter_padding() const { return filter_ - init_overflow_ - end_overflow_; } - inline int get_lower_offset() { return lower_offset_; } + inline int get_lower_offset() const { return lower_offset_; } - inline int get_output_offset() { return output_offset_; } + inline int get_output_offset() const { return output_offset_; } private: const int input_size_; diff --git a/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.hpp index b7b457f8cc9..45ca07b4bb7 100644 --- a/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -126,7 +126,7 @@ struct jit_avx512_core_bf16_1x1_conv_kernel : public jit_generator { void apply_postops(const int load_loop_blk, const int ur); void generate() override; static void balance(jit_1x1_conv_conf_t &jcp, int nthreads); - inline bool is_bcast_layout_nxc() { + inline bool is_bcast_layout_nxc() const { switch (jcp.prop_kind) { case prop_kind::forward_training: case prop_kind::forward_inference: @@ -142,13 +142,13 @@ struct jit_avx512_core_bf16_1x1_conv_kernel : public jit_generator { default: assert(!"invalid prop_kind"); return false; } } - inline bool is_load_layout_nxc() { + inline bool is_load_layout_nxc() const { return jcp.prop_kind == prop_kind::backward_weights && jcp.uses_permw_transposition && utils::one_of(jcp.dst_tag, format_tag::ndhwc, format_tag::nhwc, format_tag::nwc); } - inline bool is_out_layout_nxc() { + inline bool is_out_layout_nxc() const { switch (jcp.prop_kind) { case prop_kind::forward_training: case prop_kind::forward_inference: diff --git a/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.hpp index 885c78a89ff..d0361bcd0de 100644 --- a/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.hpp @@ -87,12 +87,12 @@ struct jit_avx512_dw_conv_fwd_kernel_bf16 : public jit_generator { Xbyak::Zmm get_acc_reg(int idx); - int get_ow_start(int ki, int pad_l) { + int get_ow_start(int ki, int pad_l) const { return nstl::max(0, utils::div_up(pad_l - ki * (jcp.dilate_w + 1), jcp.stride_w)); } - int get_ow_end(int ur_w, int ki, int pad_r) { + int get_ow_end(int ur_w, int ki, int pad_r) const { return ur_w - nstl::max(0, utils::div_up( @@ -100,12 +100,12 @@ struct jit_avx512_dw_conv_fwd_kernel_bf16 : public jit_generator { jcp.stride_w)); } - inline bool is_src_layout_nxc() { + inline bool is_src_layout_nxc() const { return utils::one_of(jcp.src_tag, format_tag::ndhwc, format_tag::nhwc, format_tag::nwc); } - inline bool is_dst_layout_nxc() { + inline bool is_dst_layout_nxc() const { return utils::one_of(jcp.dst_tag, format_tag::ndhwc, format_tag::nhwc, format_tag::nwc); } @@ -147,7 +147,7 @@ struct jit_avx512_dw_conv_bwd_data_kernel_bf16 : public jit_generator { using reg64_t = const Xbyak::Reg64; const int acc_idx_start = 2; - inline int get_max_regs() { return isa_has_bf16(jcp.isa) ? 30 : 25; }; + inline int get_max_regs() const { return isa_has_bf16(jcp.isa) ? 30 : 25; }; Xbyak::Zmm zmm_ker_reg = Xbyak::Zmm(0); Xbyak::Zmm zmm_dst_reg = Xbyak::Zmm(1); @@ -193,11 +193,11 @@ struct jit_avx512_dw_conv_bwd_data_kernel_bf16 : public jit_generator { inline void store_dsrc(int ur_ch_blocks, int ur_str_w, bool is_last_ch); void generate() override; - inline bool is_dsrc_layout_nxc() { + inline bool is_dsrc_layout_nxc() const { return utils::one_of(jcp.src_tag, format_tag::ndhwc, format_tag::nhwc, format_tag::nwc); } - inline bool is_ddst_layout_nxc() { + inline bool is_ddst_layout_nxc() const { return utils::one_of(jcp.dst_tag, format_tag::ndhwc, format_tag::nhwc, format_tag::nwc); } @@ -317,15 +317,15 @@ struct jit_avx512_dw_conv_bwd_weights_kernel_bf16 : public jit_generator { void generate() override; - inline bool is_layout_nxc() { + inline bool is_layout_nxc() const { return utils::everyone_is( true, is_src_layout_nxc(), is_ddst_layout_nxc()); } - inline bool is_src_layout_nxc() { + inline bool is_src_layout_nxc() const { return utils::one_of(jcp.src_tag, format_tag::ndhwc, format_tag::nhwc, format_tag::nwc); } - inline bool is_ddst_layout_nxc() { + inline bool is_ddst_layout_nxc() const { return utils::one_of(jcp.dst_tag, format_tag::ndhwc, format_tag::nhwc, format_tag::nwc); } diff --git a/src/cpu/x64/jit_brgemm_transpose_utils.hpp b/src/cpu/x64/jit_brgemm_transpose_utils.hpp index affbcfb8316..64d7e63ece8 100644 --- a/src/cpu/x64/jit_brgemm_transpose_utils.hpp +++ b/src/cpu/x64/jit_brgemm_transpose_utils.hpp @@ -97,7 +97,7 @@ struct jit_brgemm_copy_to_coarse_t : public jit_generator { row_step_; const dim_t data_stride_, tr_data_stride_; - inline size_t addr_offset(int row_idx) { + inline size_t addr_offset(int row_idx) const { return row_idx * row_step_ * typesize_; } diff --git a/src/cpu/x64/jit_generator.hpp b/src/cpu/x64/jit_generator.hpp index 4e8cf1b1de8..4c7bbf14fed 100644 --- a/src/cpu/x64/jit_generator.hpp +++ b/src/cpu/x64/jit_generator.hpp @@ -187,7 +187,9 @@ class jit_generator : public Xbyak::MmapAllocator, const int EVEX_max_8b_offt = 0x200; const Xbyak::Reg64 reg_EVEX_max_8b_offt = rbp; - inline size_t get_size_of_abi_save_regs() { return size_of_abi_save_regs; } + inline size_t get_size_of_abi_save_regs() const { + return size_of_abi_save_regs; + } void preamble() { if (xmm_to_preserve) { diff --git a/src/cpu/x64/jit_sse41_conv_kernel_f32.hpp b/src/cpu/x64/jit_sse41_conv_kernel_f32.hpp index 330be81bc83..50f03d9e5aa 100644 --- a/src/cpu/x64/jit_sse41_conv_kernel_f32.hpp +++ b/src/cpu/x64/jit_sse41_conv_kernel_f32.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2017-2023 Intel Corporation +* Copyright 2017-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -72,15 +72,15 @@ struct jit_sse41_conv_fwd_kernel_f32 : public jit_generator { inline void width_blk_step(int ur_w, int pad_l, int pad_r, int oc_blocks); inline void solve_common(int oc_blocks); - inline dim_t filter_w_to_input(int ki, int oi = 0, int pad_l = 0) { + inline dim_t filter_w_to_input(int ki, int oi = 0, int pad_l = 0) const { return ki * (jcp.dilate_w + 1) + oi * jcp.stride_w - pad_l; } - inline dim_t filter_h_to_input(int ki) { + inline dim_t filter_h_to_input(int ki) const { return static_cast(ki) * (jcp.dilate_h + 1) * jcp.iw; } - inline dim_t get_input_offset(int i_ic, int i_iw) { + inline dim_t get_input_offset(int i_ic, int i_iw) const { dim_t offset; if (utils::one_of(jcp.src_tag, format_tag::ncw, format_tag::nchw, format_tag::ncdhw)) { @@ -94,7 +94,7 @@ struct jit_sse41_conv_fwd_kernel_f32 : public jit_generator { return sizeof(float) * offset; } - inline dim_t get_output_offset(int i_oc_block, int i_ow) { + inline dim_t get_output_offset(int i_oc_block, int i_ow) const { dim_t offset; if (utils::one_of(jcp.dst_tag, format_tag::nwc, format_tag::nhwc, format_tag::ndhwc)) { @@ -107,7 +107,7 @@ struct jit_sse41_conv_fwd_kernel_f32 : public jit_generator { return sizeof(float) * offset; } - inline dim_t get_kernel_offset(int i_oc_block, int ki, int i_ic) { + inline dim_t get_kernel_offset(int i_oc_block, int ki, int i_ic) const { dim_t block_step_size = jcp.ic_block * jcp.oc_block; dim_t ic_block_step_size = jcp.kh * jcp.kw * block_step_size; dim_t oc_block_step_size = jcp.nb_ic * ic_block_step_size; From 1e53027631e29338217a985fc3cd42932f21c188 Mon Sep 17 00:00:00 2001 From: Dmitrii Zarukin Date: Mon, 17 Mar 2025 12:43:17 -0700 Subject: [PATCH 04/16] cpu: tidy: readability-redundant-member-init --- src/cpu/x64/jit_primitive_conf.hpp | 4 ++-- src/cpu/x64/utils/jit_io_helper.hpp | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/cpu/x64/jit_primitive_conf.hpp b/src/cpu/x64/jit_primitive_conf.hpp index 8d5d4a2a20d..e515cb96e72 100644 --- a/src/cpu/x64/jit_primitive_conf.hpp +++ b/src/cpu/x64/jit_primitive_conf.hpp @@ -609,7 +609,7 @@ struct jit_resampling_conf_t { cpu_isa_t isa = isa_undef; - post_ops_t post_ops = post_ops_t(); + post_ops_t post_ops; bool with_postops = false; bool with_eltwise = false; bool with_binary = false; @@ -944,7 +944,7 @@ struct jit_reduction_conf_t { bool is_saturation_needed = false; - post_ops_t post_ops = post_ops_t(); + post_ops_t post_ops; bool with_postops = false; bool with_eltwise = false; bool with_binary = false; diff --git a/src/cpu/x64/utils/jit_io_helper.hpp b/src/cpu/x64/utils/jit_io_helper.hpp index 7b03598bd12..68d2ebacdd1 100644 --- a/src/cpu/x64/utils/jit_io_helper.hpp +++ b/src/cpu/x64/utils/jit_io_helper.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2021-2024 Intel Corporation +* Copyright 2021-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -61,9 +61,9 @@ class io_tail_conf_t { std::size_t simd_w_ = 0; std::size_t tail_size_ = 0; - Xbyak::Opmask tail_opmask_ = Xbyak::Opmask(); + Xbyak::Opmask tail_opmask_; int tail_vmm_mask_idx_ = 0; - Xbyak::Reg64 reg_tmp_ = Xbyak::Reg64(); + Xbyak::Reg64 reg_tmp_; }; class io_emu_bf16_conf_t { @@ -124,7 +124,7 @@ class io_saturation_conf_t { int vreg_zero_saturation_idx_ = 0; int vreg_saturation_ubound_idx_ = 0; - Xbyak::Reg64 reg_tmp_ = Xbyak::Reg64(); + Xbyak::Reg64 reg_tmp_; }; class io_gather_conf_t { @@ -138,10 +138,10 @@ class io_gather_conf_t { io_gather_conf_t &operator=(const io_gather_conf_t &other) = default; std::size_t simd_w_ = 0; - Xbyak::Opmask full_opmask_ = Xbyak::Opmask(); + Xbyak::Opmask full_opmask_; int full_vmm_mask_idx_ = 0; - Xbyak::Reg64 reg_tmp_ = Xbyak::Reg64(); - Xbyak::Reg64 reg_tmp1_ = Xbyak::Reg64(); + Xbyak::Reg64 reg_tmp_; + Xbyak::Reg64 reg_tmp1_; // It is needed, when io_helper use emulation for gather // and it is not needed for sse. utils::optional_t vmm_tmp_idx_ = utils::nullopt; From 89d08100d8e646d7eda9e58d4e857671d4e84208 Mon Sep 17 00:00:00 2001 From: Dmitrii Zarukin Date: Mon, 17 Mar 2025 13:31:23 -0700 Subject: [PATCH 05/16] cpu: tidy: bugprone-macro-parentheses --- src/cpu/cpu_primitive.hpp | 22 ++++++++++++++-------- src/cpu/rnn/postgemm_dispatcher.hpp | 15 ++++++++++----- src/cpu/x64/gemm/s8x8s32/common_u8.hpp | 4 ++-- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/cpu/cpu_primitive.hpp b/src/cpu/cpu_primitive.hpp index d82b9ae0d92..baea88d774a 100644 --- a/src/cpu/cpu_primitive.hpp +++ b/src/cpu/cpu_primitive.hpp @@ -29,6 +29,9 @@ #include "cpu/ref_io_helper.hpp" +//NOLINTBEGIN(bugprone-macro-parentheses) +// These macros are actual pieces of code, can't put certain pieces into `()`. +// TODO: consider making them functions. #define DEFINE_ARG_SCALES_BUFFER_ATTR(attr, scales, arg) \ alignas(16) float CONCAT2(scales, _buf16)[16] = {0}; \ const float *scales {nullptr}; \ @@ -37,10 +40,11 @@ utils::array_set(CONCAT2(scales, _buf16), 1.0f, 16); \ scales = CONCAT2(scales, _buf16); \ } else { \ - scales = CTX_IN_MEM(const float *, DNNL_ARG_ATTR_SCALES | arg); \ + scales = CTX_IN_MEM(const float *, DNNL_ARG_ATTR_SCALES | (arg)); \ VCHECK_ATTR(scales != nullptr, \ - "Scales buffer for arg %d is missing", arg); \ - const auto scales_d = ctx.memory_mdw(DNNL_ARG_ATTR_SCALES | arg); \ + "Scales buffer for arg %d is missing", (arg)); \ + const auto scales_d \ + = ctx.memory_mdw(DNNL_ARG_ATTR_SCALES | (arg)); \ VCHECK_ATTR( \ utils::one_of(scales_d.data_type(), data_type::f32, \ data_type::f16, data_type::bf16, data_type::e8m0), \ @@ -48,7 +52,7 @@ if (scales_d.nelems() == 1) { \ const float s = cpu::io::load_float_value( \ scales_d.data_type(), scales, 0); \ - if (utils::one_of(arg, DNNL_ARG_DST, \ + if (utils::one_of((arg), DNNL_ARG_DST, \ DNNL_ARG_ATTR_POST_OP_DW | DNNL_ARG_DST)) { \ utils::array_set(CONCAT2(scales, _buf16), 1.f / s, 16); \ } else { \ @@ -61,7 +65,7 @@ MAYBE_UNUSED(scales); #define DEFINE_ARG_SCALES_BUFFER(scales, arg) \ - DEFINE_ARG_SCALES_BUFFER_ATTR(pd()->attr(), scales, arg) + DEFINE_ARG_SCALES_BUFFER_ATTR(pd()->attr(), scales, (arg)) #define DEFINE_ZERO_POINTS_BUFFER_ATTR(attr, zero_points_ptr, arg) \ int32_t CONCAT2(default_zero_point_, arg) = 0; \ @@ -74,11 +78,11 @@ * Accessing `zero_points_ptr` by index will lead to a crash for * datatypes different from s32. */ \ zero_points_ptr = CTX_IN_MEM( \ - const int32_t *, DNNL_ARG_ATTR_ZERO_POINTS | arg); \ + const int32_t *, DNNL_ARG_ATTR_ZERO_POINTS | (arg)); \ VCHECK_ATTR(zero_points_ptr != nullptr, \ - "Zero points buffer for arg %d is missing", arg); \ + "Zero points buffer for arg %d is missing", (arg)); \ const auto zero_points_d \ - = ctx.memory_mdw(DNNL_ARG_ATTR_ZERO_POINTS | arg); \ + = ctx.memory_mdw(DNNL_ARG_ATTR_ZERO_POINTS | (arg)); \ VCHECK_ATTR(utils::one_of(zero_points_d.data_type(), \ data_type::s32, data_type::s8, data_type::u8, \ data_type::s4, data_type::u4), \ @@ -132,4 +136,6 @@ #define DEFINE_ZERO_POINT_VALUE(zero_point, mem_arg) \ DEFINE_ZERO_POINT_VALUE_ATTR(pd()->attr(), zero_point, mem_arg) +//NOLINTEND(bugprone-macro-parentheses) + #endif // CPU_CPU_PRIMITIVE_HPP diff --git a/src/cpu/rnn/postgemm_dispatcher.hpp b/src/cpu/rnn/postgemm_dispatcher.hpp index 0d2637c1f86..be43824f7aa 100644 --- a/src/cpu/rnn/postgemm_dispatcher.hpp +++ b/src/cpu/rnn/postgemm_dispatcher.hpp @@ -253,20 +253,25 @@ struct rnn_postgemm_dispatcher { && !mayiuse(avx512_core)) return status::success; +//NOLINTBEGIN(bugprone-macro-parentheses) +// Can't put types into `()`: +// error: expected type-specifier before ‘)’ token #define CREATE_WITH_DIR(k, ker_t) \ do { \ if (mayiuse(avx512_core)) \ - k.reset(new ker_t(rnn, pd_)); \ + (k).reset( \ + new ker_t(rnn, pd_)); \ else if (mayiuse(avx2)) \ - k.reset(new ker_t(rnn, pd_)); \ + (k).reset(new ker_t(rnn, pd_)); \ else \ - k.reset(new ker_t(rnn, pd_)); \ + (k).reset(new ker_t(rnn, pd_)); \ } while (0) #define CREATE(k, ker_t) \ do { \ - if (jit_fwd) CREATE_WITH_DIR(k, CONCAT2(ker_t, _fwd)); \ - if (jit_bwd) CREATE_WITH_DIR(k, CONCAT2(ker_t, _bwd)); \ + if (jit_fwd) CREATE_WITH_DIR((k), CONCAT2(ker_t, _fwd)); \ + if (jit_bwd) CREATE_WITH_DIR((k), CONCAT2(ker_t, _bwd)); \ } while (0) + //NOLINTEND(bugprone-macro-parentheses) if (pd_->cell_kind() == alg_kind::vanilla_lstm) { CREATE(rnn_postgemm_, jit_uni_lstm_cell_postgemm); diff --git a/src/cpu/x64/gemm/s8x8s32/common_u8.hpp b/src/cpu/x64/gemm/s8x8s32/common_u8.hpp index c083378627e..6d5f8edbc95 100644 --- a/src/cpu/x64/gemm/s8x8s32/common_u8.hpp +++ b/src/cpu/x64/gemm/s8x8s32/common_u8.hpp @@ -22,8 +22,8 @@ #include "cpu/x64/jit_generator.hpp" #define PADD_BYTESIZE_ONPAGE(x, size) \ - (((x) * (size) + PAGE_4K - 1) / PAGE_4K) * PAGE_4K -#define NEXT_THR_STRIDE(x, size) (PADD_BYTESIZE_ONPAGE(x, size)) / size + ((((x) * (size) + PAGE_4K - 1) / PAGE_4K) * PAGE_4K) +#define NEXT_THR_STRIDE(x, size) (PADD_BYTESIZE_ONPAGE(x, (size)) / (size)) namespace dnnl { namespace impl { From d311dd08fdecd89e6ca877ef23c7d0d5c520008b Mon Sep 17 00:00:00 2001 From: Dmitrii Zarukin Date: Mon, 17 Mar 2025 13:37:20 -0700 Subject: [PATCH 06/16] cpu: x64: vreg_traits -> vreg_traits_t --- src/cpu/x64/brgemm/jit_brdgmm_kernel.cpp | 6 +++--- src/cpu/x64/brgemm/jit_brdgmm_kernel.hpp | 2 +- src/cpu/x64/brgemm/jit_brgemm_kernel.cpp | 8 ++++---- src/cpu/x64/cpu_isa_traits.hpp | 16 ++++++++-------- .../x64/injectors/jit_uni_binary_injector.cpp | 10 +++++----- .../x64/injectors/jit_uni_eltwise_injector.hpp | 4 ++-- .../x64/jit_avx512_core_amx_conv_kernel.cpp | 8 ++++---- .../x64/jit_brgemm_conv_bwd_copy_kernel.cpp | 4 ++-- .../x64/jit_brgemm_conv_bwd_trans_kernel.cpp | 4 ++-- .../x64/jit_brgemm_conv_comp_pad_kernel.hpp | 2 +- src/cpu/x64/jit_brgemm_post_ops.cpp | 4 ++-- src/cpu/x64/jit_brgemm_post_ops.hpp | 6 +++--- src/cpu/x64/jit_brgemm_transpose_utils.cpp | 4 ++-- src/cpu/x64/jit_generator.cpp | 4 ++-- src/cpu/x64/jit_generator.hpp | 4 ++-- src/cpu/x64/jit_uni_batch_normalization.cpp | 2 +- src/cpu/x64/jit_uni_binary_kernel.cpp | 3 ++- src/cpu/x64/jit_uni_convert_xf16.hpp | 6 +++--- src/cpu/x64/jit_uni_reorder_direct_copy.cpp | 2 +- src/cpu/x64/jit_uni_xf16_sum.cpp | 14 +++++++------- .../x64/matmul/brgemm_matmul_copy_utils.cpp | 18 +++++++++--------- .../x64/prelu/jit_prelu_reduction_kernel.cpp | 4 ++-- src/cpu/x64/prelu/jit_prelu_utils.cpp | 4 ++-- .../prelu/jit_uni_prelu_backward_kernel.cpp | 4 ++-- .../x64/prelu/jit_uni_prelu_forward_kernel.cpp | 4 ++-- src/cpu/x64/utils/jit_io_helper.cpp | 10 +++++----- 26 files changed, 79 insertions(+), 78 deletions(-) diff --git a/src/cpu/x64/brgemm/jit_brdgmm_kernel.cpp b/src/cpu/x64/brgemm/jit_brdgmm_kernel.cpp index 461e9142bd1..f94533ce98f 100644 --- a/src/cpu/x64/brgemm/jit_brdgmm_kernel.cpp +++ b/src/cpu/x64/brgemm/jit_brdgmm_kernel.cpp @@ -40,7 +40,7 @@ jit_brdgmm_kernel_base_t::jit_brdgmm_kernel_base_t( const brgemm_desc_t &abrd) : jit_base_brgemm_kernel_t(jit_name(), abrd.isa_impl) , brg(abrd) - , simd_w_(vreg_traits::vlen / brg.typesize_C) + , simd_w_(vreg_traits_t::vlen / brg.typesize_C) , max_vmms_(isa_num_vregs(brg.isa_impl)) , compute_dst_zp_(brg.zp_type_c != brgemm_broadcast_t::none) , compute_src_zp_(brg.zp_type_a != brgemm_broadcast_t::none) @@ -238,8 +238,8 @@ void jit_brdgmm_kernel_base_t::cvt2ps(data_type_t type_in, bool store) { const int tail_size = tail_length(); const bool is_load_tail = op.isMEM() && mask_flag && tail_size > 0 - && (tail_size - < static_cast(vreg_traits::vlen / sizeof(float))); + && (tail_size < static_cast( + vreg_traits_t::vlen / sizeof(float))); if (IMPLICATION(is_load_tail, isa_has_masks(brg.isa_impl))) { const Vmm vmm = maybe_mask(vmm_in, is_load_tail, store); switch (type_in) { diff --git a/src/cpu/x64/brgemm/jit_brdgmm_kernel.hpp b/src/cpu/x64/brgemm/jit_brdgmm_kernel.hpp index 66932f9ef09..236d027de56 100644 --- a/src/cpu/x64/brgemm/jit_brdgmm_kernel.hpp +++ b/src/cpu/x64/brgemm/jit_brdgmm_kernel.hpp @@ -168,7 +168,7 @@ struct jit_brdgmm_kernel_base_t : public jit_base_brgemm_kernel_t { using Vmm = typename utils::conditional::value, Xbyak::Zmm, Wmm>::type; - using Vmm_low_t = typename vreg_traits::Vmm_lower_t; + using Vmm_low_t = typename vreg_traits_t::Vmm_lower_t; using po_injector_t = injector::jit_uni_postops_injector_base_t; std::unique_ptr postops_injector_; std::unique_ptr bf16_emu_; diff --git a/src/cpu/x64/brgemm/jit_brgemm_kernel.cpp b/src/cpu/x64/brgemm/jit_brgemm_kernel.cpp index ef87f1c4c84..ee4c2ab5502 100644 --- a/src/cpu/x64/brgemm/jit_brgemm_kernel.cpp +++ b/src/cpu/x64/brgemm/jit_brgemm_kernel.cpp @@ -140,7 +140,7 @@ struct jit_brgemm_kernel_t : public jit_base_brgemm_kernel_t { using Vmm = typename utils::conditional::value, Xbyak::Zmm, Wmm>::type; - using Vmm_lower_t = typename vreg_traits::Vmm_lower_t; + using Vmm_lower_t = typename vreg_traits_t::Vmm_lower_t; using po_injector_t = injector::jit_uni_postops_injector_base_t; std::unique_ptr postops_injector_; std::unique_ptr bf16_emu_; @@ -669,8 +669,8 @@ void jit_brgemm_kernel_t::cvt2ps(data_type_t type_in, const Vmm vmm_in, const Xbyak::Operand &op, bool mask_flag, bool store, Xbyak::Opmask ktail_mask, dim_t tail_size) { Vmm vmm = vmm_in; - const bool has_tail - = op.isMEM() && tail_size != vreg_traits::vlen / sizeof(float); + const bool has_tail = op.isMEM() + && tail_size != vreg_traits_t::vlen / sizeof(float); if (IMPLICATION(has_tail, is_superset(brg.isa_impl, avx512_core))) { vmm = vmm_mask(vmm_in, mask_flag, store, ktail_mask); } else { @@ -2862,7 +2862,7 @@ void jit_brgemm_kernel_t::generate() { postamble(); align(32); - const dim_t simd = vreg_traits::vlen / sizeof(float); + const dim_t simd = vreg_traits_t::vlen / sizeof(float); if (!isa_has_masks(brg.isa_impl) && brg.ldb_tail > 0) { L(avx_tail_mask_); for (dim_t i = 0; i < brg.ldb_tail; ++i) diff --git a/src/cpu/x64/cpu_isa_traits.hpp b/src/cpu/x64/cpu_isa_traits.hpp index 31a6ac47732..b4e2d9ee36a 100644 --- a/src/cpu/x64/cpu_isa_traits.hpp +++ b/src/cpu/x64/cpu_isa_traits.hpp @@ -209,22 +209,22 @@ static inline bool is_superset(cpu_isa_t isa_1, cpu_isa_t isa_2) { } template -struct vreg_traits {}; +struct vreg_traits_t {}; template <> -struct vreg_traits { +struct vreg_traits_t { using Vmm_lower_t = Xbyak::Ymm; static constexpr size_t vlen = 64; }; template <> -struct vreg_traits { +struct vreg_traits_t { using Vmm_lower_t = Xbyak::Xmm; static constexpr size_t vlen = 32; }; template <> -struct vreg_traits { +struct vreg_traits_t { using Vmm_lower_t = Xbyak::Xmm; static constexpr size_t vlen = 16; }; @@ -253,7 +253,7 @@ template <> struct cpu_isa_traits { using Vmm = Xbyak::Xmm; static constexpr int vlen_shift = 4; - static constexpr int vlen = vreg_traits::vlen; + static constexpr int vlen = vreg_traits_t::vlen; static constexpr int n_vregs = 16; static constexpr dnnl_cpu_isa_t user_option_val = dnnl_cpu_isa_sse41; static constexpr const char *user_option_env = "sse41"; @@ -263,7 +263,7 @@ template <> struct cpu_isa_traits { using Vmm = Xbyak::Ymm; static constexpr int vlen_shift = 5; - static constexpr int vlen = vreg_traits::vlen; + static constexpr int vlen = vreg_traits_t::vlen; static constexpr int n_vregs = 16; static constexpr dnnl_cpu_isa_t user_option_val = dnnl_cpu_isa_avx; static constexpr const char *user_option_env = "avx"; @@ -291,7 +291,7 @@ template <> struct cpu_isa_traits { using Vmm = Xbyak::Zmm; static constexpr int vlen_shift = 6; - static constexpr int vlen = vreg_traits::vlen; + static constexpr int vlen = vreg_traits_t::vlen; static constexpr int n_vregs = 32; static constexpr dnnl_cpu_isa_t user_option_val = dnnl_cpu_isa_avx512_core; static constexpr const char *user_option_env = "avx512_core"; @@ -314,7 +314,7 @@ struct cpu_isa_traits : public cpu_isa_traits { template <> struct cpu_isa_traits { using Vmm = Xbyak::Zmm; - static constexpr int vlen = vreg_traits::vlen; + static constexpr int vlen = vreg_traits_t::vlen; static constexpr dnnl_cpu_isa_t user_option_val = dnnl_cpu_isa_avx10_1_512_amx; static constexpr const char *user_option_env = "avx10_1_512_amx"; diff --git a/src/cpu/x64/injectors/jit_uni_binary_injector.cpp b/src/cpu/x64/injectors/jit_uni_binary_injector.cpp index 8f1a8e217cd..7a45796c4df 100644 --- a/src/cpu/x64/injectors/jit_uni_binary_injector.cpp +++ b/src/cpu/x64/injectors/jit_uni_binary_injector.cpp @@ -327,14 +327,14 @@ int jit_uni_binary_injector_t::adjust_temp_vmm_hint( template static void push_vmm(jit_generator *host, const Vmm &vmm) { - host->sub(host->rsp, vreg_traits::vlen); + host->sub(host->rsp, vreg_traits_t::vlen); host->uni_vmovups(host->ptr[host->rsp], vmm); } template static void pop_vmm(jit_generator *host, const Vmm &vmm) { host->uni_vmovups(vmm, host->ptr[host->rsp]); - host->add(host->rsp, vreg_traits::vlen); + host->add(host->rsp, vreg_traits_t::vlen); } static void push_opmask(jit_generator *host, const Xbyak::Opmask &k) { @@ -357,7 +357,7 @@ static void pop_opmask(jit_generator *host, const Xbyak::Opmask &k) { template static void restore_stack(jit_generator *host, const Vmm &vmm) { - host->add(host->rsp, vreg_traits::vlen); + host->add(host->rsp, vreg_traits_t::vlen); } template @@ -2874,7 +2874,7 @@ struct helper_bcast_tail_t { if (utils::one_of(data_type, data_type::bf16, data_type::f16, data_type::f8_e5m2, data_type::f8_e4m3)) { const auto tmp_lower_vmm = - typename vreg_traits::Vmm_lower_t(tmp_vmm.getIdx()); + typename vreg_traits_t::Vmm_lower_t(tmp_vmm.getIdx()); host->load_bytes(tmp_lower_vmm, rhs_addr, tail_size * types::data_type_size(data_type)); if (data_type == data_type::bf16) { @@ -3244,7 +3244,7 @@ struct helper_load_tail_t { const Xbyak::Address &rhs_addr) { if (utils::one_of(data_type, data_type::bf16, data_type::f16)) { const auto tmp_lower_vmm = - typename vreg_traits::Vmm_lower_t(tmp_vmm.getIdx()); + typename vreg_traits_t::Vmm_lower_t(tmp_vmm.getIdx()); host->load_bytes(tmp_lower_vmm, rhs_addr_reg, 0, tail_size * sizeof(bfloat16_t)); if (data_type == data_type::bf16) { diff --git a/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp b/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp index 2348672e41a..9c2d93fa730 100644 --- a/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp +++ b/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -179,7 +179,7 @@ struct jit_uni_eltwise_injector { const bool is_avx512_ = is_superset(isa, avx512_core); - static constexpr size_t vlen_ = vreg_traits::vlen; + static constexpr size_t vlen_ = vreg_traits_t::vlen; static constexpr size_t preserved_vecs_max_ = 6; static constexpr size_t preserved_gprs_max_ = 5; static constexpr size_t n_vregs_ = cpu_isa_traits::n_vregs; diff --git a/src/cpu/x64/jit_avx512_core_amx_conv_kernel.cpp b/src/cpu/x64/jit_avx512_core_amx_conv_kernel.cpp index 2be42d91371..a7741a0c563 100644 --- a/src/cpu/x64/jit_avx512_core_amx_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_core_amx_conv_kernel.cpp @@ -5785,7 +5785,7 @@ void jit_avx512_core_amx_bwd_bias_kernel_t::compute_diff_bias( // and [P, O, N, M, L, K, J, I] -> [P, O, L, K, N, M, J, I] vpermq(yreg_bias_acc0, ptr[reg_bias + offset], 0xd8); vpermq(yreg_bias_acc1, - ptr[reg_bias + offset + vreg_traits::vlen], 0xd8); + ptr[reg_bias + offset + vreg_traits_t::vlen], 0xd8); } else if (one_of(jcp.ddst_dt, f8_e5m2, f8_e4m3)) { // the data is in plain format, transform while loading to // pseudo-vnni layout to 2 ymm registers conforming to calculations @@ -5796,7 +5796,7 @@ void jit_avx512_core_amx_bwd_bias_kernel_t::compute_diff_bias( vpermd(yreg_bias_acc0, yreg_permute_to_vnni, ptr[reg_bias + offset]); vpermd(yreg_bias_acc1, yreg_permute_to_vnni, - ptr[reg_bias + offset + vreg_traits::vlen]); + ptr[reg_bias + offset + vreg_traits_t::vlen]); } else { assert(!"non-supported type"); } @@ -5820,7 +5820,7 @@ void jit_avx512_core_amx_bwd_bias_kernel_t::compute_diff_bias( vpermq(yreg_bias_acc0, yreg_bias_acc0, 0xd8); vpermq(yreg_bias_acc1, yreg_bias_acc1, 0xd8); vmovups(ptr[reg_bias + offset], yreg_bias_acc0); - vmovups(ptr[reg_bias + offset + vreg_traits::vlen], + vmovups(ptr[reg_bias + offset + vreg_traits_t::vlen], yreg_bias_acc1); } else if (one_of(jcp.ddst_dt, f8_e5m2, f8_e4m3)) { // transform to plain before storing. @@ -5829,7 +5829,7 @@ void jit_avx512_core_amx_bwd_bias_kernel_t::compute_diff_bias( vpermd(yreg_bias_acc0, yreg_permute_to_plain, yreg_bias_acc0); vpermd(yreg_bias_acc1, yreg_permute_to_plain, yreg_bias_acc1); vmovups(ptr[reg_bias + offset], yreg_bias_acc0); - vmovups(ptr[reg_bias + offset + vreg_traits::vlen], + vmovups(ptr[reg_bias + offset + vreg_traits_t::vlen], yreg_bias_acc1); } else { assert(!"non-supported type"); diff --git a/src/cpu/x64/jit_brgemm_conv_bwd_copy_kernel.cpp b/src/cpu/x64/jit_brgemm_conv_bwd_copy_kernel.cpp index 25be385360e..f26fb2c0961 100644 --- a/src/cpu/x64/jit_brgemm_conv_bwd_copy_kernel.cpp +++ b/src/cpu/x64/jit_brgemm_conv_bwd_copy_kernel.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2023-2024 Intel Corporation +* Copyright 2023-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -87,7 +87,7 @@ template void jit_avx512_core_brgemm_conv_bwd_copy_kernel_t::generate() { preamble(); - const auto VL = vreg_traits::vlen; + const auto VL = vreg_traits_t::vlen; const auto simd_w = VL / jcp.dst_dsz; const auto n_vec = jcp.ic_block / simd_w; const auto n_tail_vec = (jcp.ic % jcp.ic_block) / simd_w; diff --git a/src/cpu/x64/jit_brgemm_conv_bwd_trans_kernel.cpp b/src/cpu/x64/jit_brgemm_conv_bwd_trans_kernel.cpp index 95ceaba44eb..c222cecf6f7 100644 --- a/src/cpu/x64/jit_brgemm_conv_bwd_trans_kernel.cpp +++ b/src/cpu/x64/jit_brgemm_conv_bwd_trans_kernel.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2022-2024 Intel Corporation +* Copyright 2022-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,7 +43,7 @@ jit_avx512_core_brgemm_conv_bwd_trans_kernel_t:: , dst_stride(jcp.owp) , dst_w_offset(oc_block_sz) , dst_h_offset(dst_stride * dst_w_offset) - , VL(vreg_traits::vlen) + , VL(vreg_traits_t::vlen) , n_vec(jcp.oc_block / jcp.simd_w) , n_tail_vec((jcp.oc_without_padding % jcp.oc_block) / jcp.simd_w) {} diff --git a/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.hpp b/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.hpp index 6e0ecafd071..8f6378717d5 100644 --- a/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.hpp +++ b/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.hpp @@ -96,7 +96,7 @@ struct jit_uni_brgemm_conv_comp_pad_kernel_t : public jit_generator { Xbyak::Zmm zmm_int8_temp = Xbyak::Zmm(26); const int last_ic_block_ = 4; - const int m_block2_ = vreg_traits::vlen / sizeof(int32_t); + const int m_block2_ = vreg_traits_t::vlen / sizeof(int32_t); static constexpr int max_oc_block_ = 64; const int n_max_regs_ = max_oc_block_ / m_block2_; diff --git a/src/cpu/x64/jit_brgemm_post_ops.cpp b/src/cpu/x64/jit_brgemm_post_ops.cpp index f810905c4f4..d45a23a7603 100644 --- a/src/cpu/x64/jit_brgemm_post_ops.cpp +++ b/src/cpu/x64/jit_brgemm_post_ops.cpp @@ -627,7 +627,7 @@ void dnnl::impl::cpu::x64::jit_brgemm_kernel_post_ops_t::cvt2ps( int tail_size, bool store, Xbyak::Opmask ktail_mask, bool skip_cvt2ps /*= false*/) { const bool is_tail = op.isMEM() - && tail_size != vreg_traits::vlen / sizeof(float) + && tail_size != vreg_traits_t::vlen / sizeof(float) // The current kernel is written such that tail_size = 0 implies // no tail and full vmm must be processed. && tail_size > 0; @@ -983,7 +983,7 @@ void dnnl::impl::cpu::x64::jit_brgemm_kernel_post_ops_t::apply_post_ops( default: assert(!"unknown dst_dt"); } } else { - const int simd_w = vreg_traits::vlen / sizeof(float); + const int simd_w = vreg_traits_t::vlen / sizeof(float); const int nelems = tail > 0 ? tail : simd_w; store_data(out_dt_, vmm, aux_reg_out, offset, nelems); } diff --git a/src/cpu/x64/jit_brgemm_post_ops.hpp b/src/cpu/x64/jit_brgemm_post_ops.hpp index 67ccc6896f5..78c2c08f1cf 100644 --- a/src/cpu/x64/jit_brgemm_post_ops.hpp +++ b/src/cpu/x64/jit_brgemm_post_ops.hpp @@ -71,7 +71,7 @@ struct jit_brgemm_kernel_diff_bias_t : public jit_generator { int acc_typesize_; int mult_; - using Vmm_lower_t = typename vreg_traits::Vmm_lower_t; + using Vmm_lower_t = typename vreg_traits_t::Vmm_lower_t; using reg64_t = const Xbyak::Reg64; // Register decomposition const reg64_t param1 = abi_param1; @@ -197,8 +197,8 @@ struct jit_brgemm_kernel_post_ops_t : public jit_brgemm_kernel_post_ops_base_t, data_type_t out_dt_; data_type_t bia_dt_; - using Vmm_lower_t = typename vreg_traits::Vmm_lower_t; - using Vmm_lower2_t = typename vreg_traits::Vmm_lower_t; + using Vmm_lower_t = typename vreg_traits_t::Vmm_lower_t; + using Vmm_lower2_t = typename vreg_traits_t::Vmm_lower_t; using po_injector_t = injector::jit_uni_postops_injector_base_t; std::unique_ptr postops_injector_; std::unique_ptr bf16_emu_; diff --git a/src/cpu/x64/jit_brgemm_transpose_utils.cpp b/src/cpu/x64/jit_brgemm_transpose_utils.cpp index 37d771dea9a..a5606a6bb0c 100644 --- a/src/cpu/x64/jit_brgemm_transpose_utils.cpp +++ b/src/cpu/x64/jit_brgemm_transpose_utils.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -365,7 +365,7 @@ void jit_brgemm_trans_m_k_f32_t::init_masks(int tail_length) { } else if (tail_length) { lea(reg_tmp, ptr[rip + mask_label_]); vmovups(ymm_tail_mask, ptr[reg_tmp]); - vmovups(xmm_upper_tail_mask, ptr[reg_tmp + vreg_traits::vlen]); + vmovups(xmm_upper_tail_mask, ptr[reg_tmp + vreg_traits_t::vlen]); } } diff --git a/src/cpu/x64/jit_generator.cpp b/src/cpu/x64/jit_generator.cpp index 17f88c68389..40fc96ea966 100644 --- a/src/cpu/x64/jit_generator.cpp +++ b/src/cpu/x64/jit_generator.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2024 Intel Corporation +* Copyright 2024-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -35,7 +35,7 @@ void jit_generator::transpose(const Xbyak::Reg64 ®_src, // only avx2 version is supported for now. TODO for others const int transpose_size - = vreg_traits::vlen / types::data_type_size(dt); + = vreg_traits_t::vlen / types::data_type_size(dt); assert(is_valid_isa(avx2)); assert(nrows <= transpose_size && ncolumns <= transpose_size); diff --git a/src/cpu/x64/jit_generator.hpp b/src/cpu/x64/jit_generator.hpp index 4c7bbf14fed..2440512c5f7 100644 --- a/src/cpu/x64/jit_generator.hpp +++ b/src/cpu/x64/jit_generator.hpp @@ -2639,7 +2639,7 @@ class jit_generator : public Xbyak::MmapAllocator, load_bytes( vmm, src_addr, sizeof(float16_t) * load_size, zero_vmm); vcvtph2ps(vmm, - typename vreg_traits::Vmm_lower_t(vmm.getIdx())); + typename vreg_traits_t::Vmm_lower_t(vmm.getIdx())); break; default: assert(!"unsupported source data type"); } @@ -2658,7 +2658,7 @@ class jit_generator : public Xbyak::MmapAllocator, const std::function &tail_process, const data_type_t data_type = data_type::f32) { const auto simd_w - = vreg_traits::vlen / types::data_type_size(data_type); + = vreg_traits_t::vlen / types::data_type_size(data_type); Xbyak::Label label_tbl, label_tbl_end; std::vector l_case(simd_w); diff --git a/src/cpu/x64/jit_uni_batch_normalization.cpp b/src/cpu/x64/jit_uni_batch_normalization.cpp index f705592a05c..bd022ebeffd 100644 --- a/src/cpu/x64/jit_uni_batch_normalization.cpp +++ b/src/cpu/x64/jit_uni_batch_normalization.cpp @@ -636,7 +636,7 @@ struct jit_bnorm_t : public jit_generator { } else if (is_f16_) { auto src_reg = Vmm(src.getIdx()); auto dst_reg = - typename vreg_traits::Vmm_lower_t(src.getIdx()); + typename vreg_traits_t::Vmm_lower_t(src.getIdx()); if (is_nt_store) { if (mayiuse(avx512_core_fp16)) vcvtps2phx(dst_reg, src_reg); diff --git a/src/cpu/x64/jit_uni_binary_kernel.cpp b/src/cpu/x64/jit_uni_binary_kernel.cpp index 949e9247139..16fbdd97ca8 100644 --- a/src/cpu/x64/jit_uni_binary_kernel.cpp +++ b/src/cpu/x64/jit_uni_binary_kernel.cpp @@ -90,7 +90,8 @@ size_t binary_kernel_t::get_tail_size() const { template jit_uni_binary_kernel_t::jit_uni_binary_kernel_t( const binary_pd_t *pd, const jit_binary_conf_t conf, bool tail_kernel) - : binary_kernel_t(vreg_traits::vlen, pd, conf, jit_name(), tail_kernel) + : binary_kernel_t( + vreg_traits_t::vlen, pd, conf, jit_name(), tail_kernel) , offt_src0_(vlen_ / ((conf_.is_bf16 || conf_.is_f16) ? 2 : 1)) , offt_src1_(conf_.use_stride_src1 ? offt_src0_ : 0) , offt_src2_(offt_src0_) { diff --git a/src/cpu/x64/jit_uni_convert_xf16.hpp b/src/cpu/x64/jit_uni_convert_xf16.hpp index 775beaace64..94fc8287758 100644 --- a/src/cpu/x64/jit_uni_convert_xf16.hpp +++ b/src/cpu/x64/jit_uni_convert_xf16.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2022 Intel Corporation +* Copyright 2022-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -72,7 +72,7 @@ struct jit_uni_cvt_ps_to_xf16_t : public jit_generator { constexpr static int simd_w_ = cpu_isa_traits::vlen / sizeof(float); using Vmm = typename cpu_isa_traits::Vmm; - using Vmm_down_t = typename vreg_traits::Vmm_lower_t; + using Vmm_down_t = typename vreg_traits_t::Vmm_lower_t; const Vmm vmm_input = Vmm(0); const Vmm_down_t vmm_output = Vmm_down_t(1); @@ -176,7 +176,7 @@ struct jit_uni_cvt_xf16_to_ps_t : public jit_generator { constexpr static int elem_granularity = isa == avx2_vnni_2 ? 2 : 1; constexpr static int simd_w_ = cpu_isa_traits::vlen / sizeof(float); using Vmm = typename cpu_isa_traits::Vmm; - using Vmm_down_t = typename vreg_traits::Vmm_lower_t; + using Vmm_down_t = typename vreg_traits_t::Vmm_lower_t; const impl::data_type_t input_dt_; const bool with_add_; diff --git a/src/cpu/x64/jit_uni_reorder_direct_copy.cpp b/src/cpu/x64/jit_uni_reorder_direct_copy.cpp index 165c18900be..f280e6d1965 100644 --- a/src/cpu/x64/jit_uni_reorder_direct_copy.cpp +++ b/src/cpu/x64/jit_uni_reorder_direct_copy.cpp @@ -62,7 +62,7 @@ struct direct_copy_kernel_t {{dst_dt_, io_saturation_conf}}, utils::nullopt, io_fp8_conf); } - static constexpr int vlen_ = vreg_traits::vlen; + static constexpr int vlen_ = vreg_traits_t::vlen; static constexpr int simd_w_ = vlen_ / sizeof(float); static constexpr int unroll_12_ = 12; static constexpr int unroll_4_ = 4; diff --git a/src/cpu/x64/jit_uni_xf16_sum.cpp b/src/cpu/x64/jit_uni_xf16_sum.cpp index 8b9e787a0a8..c7e6c02e9ad 100644 --- a/src/cpu/x64/jit_uni_xf16_sum.cpp +++ b/src/cpu/x64/jit_uni_xf16_sum.cpp @@ -107,7 +107,7 @@ void jit_avx512_core_bf16_sum_kernel_t::tail_iteration() { cmp(reg_sz, 0); jle(exit_label, T_NEAR); - const int bf16_half_reg = vreg_traits::vlen / 4; + const int bf16_half_reg = vreg_traits_t::vlen / 4; mov(reg32_mask, 0xffff); cmp(reg_sz, bf16_half_reg); jge(mask_label, T_NEAR); @@ -165,7 +165,7 @@ void jit_avx512_core_bf16_sum_kernel_t::tail_iteration() { for (int s = 0; s < jsp.num_srcs; s++) add(reg_src[s], bf16_half_reg * jsp.typesize_in); - add(reg_dst, (vreg_traits::vlen / 4) * jsp.typesize_out); + add(reg_dst, (vreg_traits_t::vlen / 4) * jsp.typesize_out); jmp(tail_label, T_NEAR); } @@ -196,7 +196,7 @@ status_t jit_avx512_core_bf16_sum_kernel_t::init_conf( break; } if (jsp.loop_unroll == 0) return status::unimplemented; - jsp.size_blocking = (vreg_traits::vlen / 2) * jsp.loop_unroll; + jsp.size_blocking = (vreg_traits_t::vlen / 2) * jsp.loop_unroll; const memory_desc_wrapper o_d(&dst_d); jsp.is_bf16_dst = data_type::bf16 == o_d.data_type(); @@ -329,7 +329,7 @@ status_t jit_avx2_vnni_2_xf16_sum_kernel_t::init_conf(jit_sum_conf_t &jsp, jsp.isa = avx2_vnni_2; jsp.loop_unroll = 6; jsp.unroll_reg_count = 2 * num_srcs + 4; - jsp.size_blocking = (vreg_traits::vlen / 2) * jsp.loop_unroll; + jsp.size_blocking = (vreg_traits_t::vlen / 2) * jsp.loop_unroll; const memory_desc_wrapper i_d(&(src_d.front())); const memory_desc_wrapper o_d(&dst_d); @@ -348,9 +348,9 @@ template void jit_uni_xf16_sum_kernel_t::loop_iteration(int current_unroll) { Label loop_label, loop_exit_label; const int num_compute_elements - = (vreg_traits::vlen / 2) * current_unroll; - dim_t src_shift = (vreg_traits::vlen / 2) * jsp.typesize_in; - dim_t dst_shift = (vreg_traits::vlen / 4) * jsp.typesize_out; + = (vreg_traits_t::vlen / 2) * current_unroll; + dim_t src_shift = (vreg_traits_t::vlen / 2) * jsp.typesize_in; + dim_t dst_shift = (vreg_traits_t::vlen / 4) * jsp.typesize_out; L(loop_label); cmp(reg_sz, num_compute_elements); jl(loop_exit_label, T_NEAR); diff --git a/src/cpu/x64/matmul/brgemm_matmul_copy_utils.cpp b/src/cpu/x64/matmul/brgemm_matmul_copy_utils.cpp index d1588696732..43ed3b8d858 100644 --- a/src/cpu/x64/matmul/brgemm_matmul_copy_utils.cpp +++ b/src/cpu/x64/matmul/brgemm_matmul_copy_utils.cpp @@ -72,7 +72,7 @@ struct jit_brgemm_matmul_copy_a_impl_t : public jit_brgemm_matmul_copy_a_t, using reg32_t = const Xbyak::Reg32; using opmask_t = const Xbyak::Opmask; - static constexpr int vlen_ = vreg_traits::vlen; + static constexpr int vlen_ = vreg_traits_t::vlen; static constexpr bool is_ymm_ = std::is_same::value; static constexpr int num_comp_acc_ = is_ymm_ ? 7 : 8; @@ -2089,7 +2089,7 @@ struct jit_brgemm_matmul_copy_b_int8_t : public jit_brgemm_matmul_copy_b_t, static constexpr int k_blk_step_ = 4; static constexpr int n_blk_step_ = 64; static constexpr int blk_sz_ = 6; - static constexpr int simd_w_ = vreg_traits::vlen; + static constexpr int simd_w_ = vreg_traits_t::vlen; const dim_t src_stride_; const dim_t tr_src_stride_; @@ -2888,7 +2888,7 @@ struct jit_brgemm_matmul_copy_b_bf16_t : public jit_brgemm_matmul_copy_b_t, using opmask_t = const Xbyak::Opmask; using zmm = const Xbyak::Zmm; using ymm = const Xbyak::Ymm; - using Vmm_lower_t = typename vreg_traits::Vmm_lower_t; + using Vmm_lower_t = typename vreg_traits_t::Vmm_lower_t; enum { k_blk_step = 2, n_blk_step = 16 }; const int typesize, tr_typesize, scales_typesize; @@ -3092,7 +3092,7 @@ void jit_brgemm_matmul_copy_b_bf16_t::copy_2x32( const auto store_addr = maybe_EVEX_compress_addr(reg_tr_src, tr_src_off); const auto store_addr_ymm1 - = ptr[reg_tr_src + tr_src_off + vreg_traits::vlen]; + = ptr[reg_tr_src + tr_src_off + vreg_traits_t::vlen]; const int blk_idx = iter % max_unroll; const auto src_vmm0 = get_vmm(blk_idx, 0); const auto src_zmm0 = zmm(src_vmm0.getIdx()); @@ -3350,7 +3350,7 @@ struct jit_brgemm_matmul_copy_b_f32_t : public jit_brgemm_matmul_copy_b_t, : jit_brgemm_matmul_copy_b_t(conf) , jit_generator(jit_name()) , dt_in_(conf->orig_wei_dt) - , simd_w_(vreg_traits::vlen / sizeof(float)) + , simd_w_(vreg_traits_t::vlen / sizeof(float)) , is_src_int4_(one_of(conf->orig_wei_dt, data_type::s4, data_type::u4)) , req_zp_b_shift_( conf->has_zero_point_b && conf->with_wei_decompression) @@ -3369,7 +3369,7 @@ struct jit_brgemm_matmul_copy_b_f32_t : public jit_brgemm_matmul_copy_b_t, using reg64_t = const Xbyak::Reg64; using reg32_t = const Xbyak::Reg32; using opmask_t = const Xbyak::Opmask; - using Vmm_lower_t = typename vreg_traits::Vmm_lower_t; + using Vmm_lower_t = typename vreg_traits_t::Vmm_lower_t; const data_type_t dt_in_; const int simd_w_; @@ -3677,12 +3677,12 @@ struct jit_brgemm_matmul_copy_b_transposed_t using reg64_t = const Xbyak::Reg64; using reg32_t = const Xbyak::Reg32; using opmask_t = const Xbyak::Opmask; - using Vmm_lower_t = typename vreg_traits::Vmm_lower_t; + using Vmm_lower_t = typename vreg_traits_t::Vmm_lower_t; static constexpr bool is_ymm_ = std::is_same::value; static constexpr cpu_isa_t isa_ = is_ymm_ ? avx2 : avx512_core; static constexpr int max_vmm_regs_ = cpu_isa_traits::n_vregs; - static constexpr int vlen_ = vreg_traits::vlen; + static constexpr int vlen_ = vreg_traits_t::vlen; static constexpr int n_blk_step_ = is_ymm_ ? 8 : 16; static constexpr int req_cvt_bf16_k_blk_step_ = 16; static constexpr size_t comp_shift_ = vlen_; @@ -4576,7 +4576,7 @@ struct jit_brgemm_matmul_copy_b_cvt_bf16_t : public jit_brgemm_matmul_copy_b_t, using reg64_t = const Xbyak::Reg64; using reg32_t = const Xbyak::Reg32; using opmask_t = const Xbyak::Opmask; - using Vmm_lower_t = typename vreg_traits::Vmm_lower_t; + using Vmm_lower_t = typename vreg_traits_t::Vmm_lower_t; using zmm = const Xbyak::Zmm; using ymm = const Xbyak::Ymm; diff --git a/src/cpu/x64/prelu/jit_prelu_reduction_kernel.cpp b/src/cpu/x64/prelu/jit_prelu_reduction_kernel.cpp index ae67a8b9ed4..9a5ac599742 100644 --- a/src/cpu/x64/prelu/jit_prelu_reduction_kernel.cpp +++ b/src/cpu/x64/prelu/jit_prelu_reduction_kernel.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2022 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -128,7 +128,7 @@ Xbyak::Address jit_prelu_reduction_kernel_t::diff_scratch_ptr( template jit_uni_prelu_reduction_kernel_t::jit_uni_prelu_reduction_kernel_t( const cpu_prelu_bwd_pd_t *pd, const cpu_isa_t &isa) - : jit_prelu_reduction_kernel_t(pd, vreg_traits::vlen / sizeof(float)) + : jit_prelu_reduction_kernel_t(pd, vreg_traits_t::vlen / sizeof(float)) , isa_(isa) , saturation_needed_(utils::one_of( data_type_, data_type::s8, data_type::u8, data_type::s32)) diff --git a/src/cpu/x64/prelu/jit_prelu_utils.cpp b/src/cpu/x64/prelu/jit_prelu_utils.cpp index 459757c91dd..d98a3b69274 100644 --- a/src/cpu/x64/prelu/jit_prelu_utils.cpp +++ b/src/cpu/x64/prelu/jit_prelu_utils.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2022 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -88,7 +88,7 @@ int get_simd_w(const std::set &tensor_data_types) noexcept { const auto &isa = prelu::get_supported_isa(); return (isa == avx && is_s8u8(tensor_data_types)) - ? vreg_traits::vlen / sizeof(float) + ? vreg_traits_t::vlen / sizeof(float) : prelu::get_vlen(isa) / sizeof(float); } diff --git a/src/cpu/x64/prelu/jit_uni_prelu_backward_kernel.cpp b/src/cpu/x64/prelu/jit_uni_prelu_backward_kernel.cpp index 680645b9e74..da4179a9609 100644 --- a/src/cpu/x64/prelu/jit_uni_prelu_backward_kernel.cpp +++ b/src/cpu/x64/prelu/jit_uni_prelu_backward_kernel.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2022 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -83,7 +83,7 @@ bool jit_prelu_backward_kernel_t::any_tensor_bf16() const { template jit_uni_prelu_backward_kernel_t::jit_uni_prelu_backward_kernel_t( const cpu_prelu_bwd_pd_t *pd, const cpu_isa_t &isa) - : jit_prelu_backward_kernel_t(pd, isa, vreg_traits::vlen, + : jit_prelu_backward_kernel_t(pd, isa, vreg_traits_t::vlen, std::is_same::value ? 4u : 6u) , saturation_needed_diff_src_(utils::one_of( diff_src_dt_, data_type::u8, data_type::s8, data_type::s32)) diff --git a/src/cpu/x64/prelu/jit_uni_prelu_forward_kernel.cpp b/src/cpu/x64/prelu/jit_uni_prelu_forward_kernel.cpp index 28b2b89574e..d707594db28 100644 --- a/src/cpu/x64/prelu/jit_uni_prelu_forward_kernel.cpp +++ b/src/cpu/x64/prelu/jit_uni_prelu_forward_kernel.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2022 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -71,7 +71,7 @@ bool jit_prelu_forward_kernel_t::any_tensor_bf16() const { template jit_uni_prelu_forward_kernel_t::jit_uni_prelu_forward_kernel_t( const cpu_prelu_fwd_pd_t *pd, const cpu_isa_t &isa) - : jit_prelu_forward_kernel_t(pd, isa, vreg_traits::vlen, + : jit_prelu_forward_kernel_t(pd, isa, vreg_traits_t::vlen, (utils::one_of(isa, sse41, avx) || pd->src_md(0)->data_type != data_type::f32) ? 4u diff --git a/src/cpu/x64/utils/jit_io_helper.cpp b/src/cpu/x64/utils/jit_io_helper.cpp index 63c1a4d86cd..ef3bcf65a48 100644 --- a/src/cpu/x64/utils/jit_io_helper.cpp +++ b/src/cpu/x64/utils/jit_io_helper.cpp @@ -278,7 +278,7 @@ void jit_io_helper_t::prepare_i8_data_to_store(const Vmm &i8_vmm) { template void jit_io_helper_t::prepare_xf16_data_to_store(const Vmm &vmm) { const auto &cvt_lower_vmm = - typename vreg_traits::Vmm_lower_t(vmm.getIdx()); + typename vreg_traits_t::Vmm_lower_t(vmm.getIdx()); if (data_type_ == data_type::bf16) host_->vcvtneps2bf16(cvt_lower_vmm, vmm, host_->get_encoding()); @@ -778,7 +778,7 @@ void jit_io_helper_t::store(const Vmm &src_raw_vmm, // TODO: Consider adding opmask to store xf16 data from Xmm. // This could allow to use store_bf16/store_f16 functions for isa >= avx512_core. const size_t xmm_length - = vreg_traits::vlen / sizeof(int32_t); + = vreg_traits_t::vlen / sizeof(int32_t); const size_t store_size = (tail ? tail_conf_->tail_size_ : xmm_length) * types::data_type_size(data_type_); store_byte_by_byte(src_vmm, dst_addr, store_size); @@ -817,7 +817,7 @@ void jit_io_helper_t::store_byte_by_byte(const Vmm &src_vmm, const bool is_xf16 = utils::one_of(data_type_, data_type::bf16, data_type::f16); const auto &cvt_lower_vmm = - typename vreg_traits::Vmm_lower_t(src_vmm.getIdx()); + typename vreg_traits_t::Vmm_lower_t(src_vmm.getIdx()); if (is_i8) prepare_i8_data_to_store(src_vmm); if (is_xf16) prepare_xf16_data_to_store(src_vmm); @@ -845,7 +845,7 @@ void jit_io_helper_t::store_bf16( && "Store operation for bf16 is not supported for Xmms."); const auto &cvt_lower_vmm = - typename vreg_traits::Vmm_lower_t(src_vmm.getIdx()); + typename vreg_traits_t::Vmm_lower_t(src_vmm.getIdx()); if (bf16_emu_) bf16_emu_->vcvtneps2bf16(cvt_lower_vmm, src_vmm); @@ -866,7 +866,7 @@ void jit_io_helper_t::store_f16( && "Store operation for f16 is not supported for Xmms."); const auto &cvt_lower_vmm = - typename vreg_traits::Vmm_lower_t(src_vmm.getIdx()); + typename vreg_traits_t::Vmm_lower_t(src_vmm.getIdx()); host_->uni_vcvtps2phx(cvt_lower_vmm, src_vmm); From e3589bd5e726f8ea3e531a4bb73049618a761076 Mon Sep 17 00:00:00 2001 From: Dmitrii Zarukin Date: Mon, 17 Mar 2025 13:44:08 -0700 Subject: [PATCH 07/16] cpu: cpu_isa_traits -> cpu_isa_traits_t --- src/cpu/platform.cpp | 8 ++-- src/cpu/x64/brgemm/jit_brgemm_amx_uker.cpp | 2 +- src/cpu/x64/brgemm/jit_brgemm_kernel.cpp | 4 +- src/cpu/x64/cpu_isa_traits.cpp | 8 ++-- src/cpu/x64/cpu_isa_traits.hpp | 40 ++++++++++--------- src/cpu/x64/cpu_reducer.cpp | 4 +- src/cpu/x64/gemm/gemm_driver.cpp | 6 +-- src/cpu/x64/gemm_bf16_convolution.cpp | 4 +- .../x64/injectors/jit_uni_binary_injector.cpp | 8 ++-- .../x64/injectors/jit_uni_binary_injector.hpp | 4 +- .../injectors/jit_uni_eltwise_injector.hpp | 4 +- .../injectors/jit_uni_postops_injector.hpp | 2 +- src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.hpp | 4 +- src/cpu/x64/jit_avx2_conv_kernel_f32.hpp | 2 +- .../x64/jit_avx512_common_1x1_conv_kernel.cpp | 4 +- .../x64/jit_avx512_common_1x1_conv_kernel.hpp | 4 +- src/cpu/x64/jit_avx512_common_conv_kernel.cpp | 8 ++-- src/cpu/x64/jit_avx512_common_conv_kernel.hpp | 2 +- .../jit_avx512_core_amx_1x1_conv_kernel.hpp | 4 +- .../x64/jit_avx512_core_amx_conv_kernel.hpp | 2 +- .../jit_avx512_core_bf16_1x1_conv_kernel.cpp | 2 +- .../jit_avx512_core_bf16_1x1_conv_kernel.hpp | 2 +- .../x64/jit_avx512_core_bf16_conv_kernel.cpp | 8 ++-- .../x64/jit_avx512_core_bf16_conv_kernel.hpp | 2 +- .../jit_avx512_core_bf16_dw_conv_kernel.cpp | 4 +- src/cpu/x64/jit_avx512_core_fp8cvt.cpp | 6 +-- src/cpu/x64/jit_avx512_core_resampling.cpp | 4 +- .../x64/jit_avx512_core_scale_precompute.hpp | 4 +- ...t_avx512_core_x8s8s32x_1x1_conv_kernel.hpp | 4 +- .../jit_avx512_core_x8s8s32x_conv_kernel.hpp | 4 +- src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp | 2 +- src/cpu/x64/jit_brgemm_conv_trans_kernel.cpp | 4 +- src/cpu/x64/jit_brgemm_inner_product.hpp | 4 +- src/cpu/x64/jit_gemm_inner_product_utils.cpp | 2 +- .../jit_gemm_x8s8s32x_convolution_utils.cpp | 6 +-- src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.hpp | 5 ++- src/cpu/x64/jit_sse41_conv_kernel_f32.hpp | 3 +- src/cpu/x64/jit_uni_batch_normalization.cpp | 6 +-- .../x64/jit_uni_batch_normalization_s8.cpp | 8 ++-- src/cpu/x64/jit_uni_binary.cpp | 4 +- src/cpu/x64/jit_uni_convert_xf16.hpp | 8 ++-- .../x64/jit_uni_deconv_zp_pad_str_kernel.cpp | 2 +- src/cpu/x64/jit_uni_dw_conv_kernel_f32.cpp | 12 +++--- src/cpu/x64/jit_uni_dw_conv_kernel_f32.hpp | 8 ++-- src/cpu/x64/jit_uni_eltwise.cpp | 10 ++--- src/cpu/x64/jit_uni_eltwise_int.cpp | 6 +-- src/cpu/x64/jit_uni_group_normalization.cpp | 8 ++-- src/cpu/x64/jit_uni_i8i8_pooling.cpp | 28 ++++++------- .../x64/jit_uni_instance_normalization.cpp | 8 ++-- src/cpu/x64/jit_uni_layer_normalization.cpp | 12 +++--- src/cpu/x64/jit_uni_pool_kernel.cpp | 6 +-- src/cpu/x64/jit_uni_pool_kernel.hpp | 2 +- src/cpu/x64/jit_uni_reduction_kernel.hpp | 2 +- src/cpu/x64/jit_uni_resampling_kernel.hpp | 2 +- src/cpu/x64/jit_uni_softmax.cpp | 10 ++--- .../x64/jit_uni_tbb_batch_normalization.cpp | 18 ++++----- src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.hpp | 4 +- src/cpu/x64/jit_uni_xf16_sum.cpp | 2 +- src/cpu/x64/jit_uni_xf16_sum.hpp | 2 +- src/cpu/x64/lrn/jit_uni_lrn_kernel.cpp | 2 +- src/cpu/x64/lrn/jit_uni_lrn_kernel.hpp | 7 ++-- src/cpu/x64/matmul/brgemm_matmul.hpp | 2 +- .../x64/matmul/brgemm_matmul_copy_utils.cpp | 2 +- src/cpu/x64/matmul/jit_uni_sparse_matmul.cpp | 6 +-- src/cpu/x64/prelu/jit_prelu_utils.cpp | 28 ++++++------- .../rnn/jit_uni_gru_cell_postgemm_1_bwd.hpp | 8 ++-- .../rnn/jit_uni_gru_cell_postgemm_1_fwd.hpp | 4 +- .../rnn/jit_uni_gru_cell_postgemm_2_bwd.hpp | 4 +- .../rnn/jit_uni_gru_cell_postgemm_2_fwd.hpp | 4 +- .../rnn/jit_uni_gru_lbr_cell_postgemm_bwd.hpp | 8 ++-- .../rnn/jit_uni_gru_lbr_cell_postgemm_fwd.hpp | 4 +- .../x64/rnn/jit_uni_lstm_cell_postgemm.hpp | 8 ++-- .../rnn/jit_uni_lstm_cell_postgemm_bwd.hpp | 2 +- .../rnn/jit_uni_lstm_cell_postgemm_fwd.hpp | 2 +- ..._uni_lstm_cell_projection_postgemm_fwd.hpp | 2 +- .../x64/rnn/jit_uni_rnn_cell_postgemm_bwd.hpp | 4 +- .../x64/rnn/jit_uni_rnn_cell_postgemm_fwd.hpp | 4 +- .../x64/rnn/jit_uni_rnn_common_postgemm.hpp | 2 +- src/cpu/x64/shuffle/jit_uni_shuffle.cpp | 4 +- .../x64/shuffle/jit_uni_shuffle_kernel.hpp | 6 +-- 80 files changed, 239 insertions(+), 232 deletions(-) diff --git a/src/cpu/platform.cpp b/src/cpu/platform.cpp index 8dbbeffa56b..51abf9b4e90 100644 --- a/src/cpu/platform.cpp +++ b/src/cpu/platform.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * Copyright 2020-2024 FUJITSU LIMITED * Copyright 2022-2024 Arm Ltd. and affiliates * @@ -258,9 +258,9 @@ unsigned get_max_threads_to_use() { int get_vector_register_size() { #if DNNL_X64 using namespace x64; - if (mayiuse(avx512_core)) return cpu_isa_traits::vlen; - if (mayiuse(avx)) return cpu_isa_traits::vlen; - if (mayiuse(sse41)) return cpu_isa_traits::vlen; + if (mayiuse(avx512_core)) return cpu_isa_traits_t::vlen; + if (mayiuse(avx)) return cpu_isa_traits_t::vlen; + if (mayiuse(sse41)) return cpu_isa_traits_t::vlen; #elif DNNL_AARCH64 using namespace aarch64; if (mayiuse(asimd)) return cpu_isa_traits::vlen; diff --git a/src/cpu/x64/brgemm/jit_brgemm_amx_uker.cpp b/src/cpu/x64/brgemm/jit_brgemm_amx_uker.cpp index c5165b8a3a9..f270c82d347 100644 --- a/src/cpu/x64/brgemm/jit_brgemm_amx_uker.cpp +++ b/src/cpu/x64/brgemm/jit_brgemm_amx_uker.cpp @@ -146,7 +146,7 @@ struct jit_brgemm_amx_uker_base_t : public jit_base_brgemm_kernel_t { using reg64_t = const Xbyak::Reg64; enum { simd_w = 16, - zmm_width_in_bytes = cpu_isa_traits::vlen, + zmm_width_in_bytes = cpu_isa_traits_t::vlen, }; // Register decomposition diff --git a/src/cpu/x64/brgemm/jit_brgemm_kernel.cpp b/src/cpu/x64/brgemm/jit_brgemm_kernel.cpp index ee4c2ab5502..21ea0f25533 100644 --- a/src/cpu/x64/brgemm/jit_brgemm_kernel.cpp +++ b/src/cpu/x64/brgemm/jit_brgemm_kernel.cpp @@ -136,7 +136,7 @@ struct jit_brgemm_kernel_t : public jit_base_brgemm_kernel_t { enum matrix_kind_t { matrix_A, matrix_B }; static constexpr int zmm_width_in_bytes_ - = cpu_isa_traits::vlen; + = cpu_isa_traits_t::vlen; using Vmm = typename utils::conditional::value, Xbyak::Zmm, Wmm>::type; @@ -1993,7 +1993,7 @@ bool jit_brgemm_kernel_t::maybe_pre_process_k_tail(bool last_bdb, && brg.rdb_tail != 0 && last_bdb && is_rd_tail; if (!need_k_tail_processing) return false; - const auto zmm_width_in_bytes = cpu_isa_traits::vlen; + const auto zmm_width_in_bytes = cpu_isa_traits_t::vlen; auto transform_offset = brg.get_num_C_tiles() * brgemm_desc_t::tilesize + brg.get_convert_wsp_buffer_size(); diff --git a/src/cpu/x64/cpu_isa_traits.cpp b/src/cpu/x64/cpu_isa_traits.cpp index 931f13a8c2b..c9d718e1132 100644 --- a/src/cpu/x64/cpu_isa_traits.cpp +++ b/src/cpu/x64/cpu_isa_traits.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,7 +46,7 @@ cpu_isa_t init_max_cpu_isa() { if (!isa_val.empty()) { #define IF_HANDLE_CASE(cpu_isa) \ - if (isa_val.compare(cpu_isa_traits::user_option_env) == 0) \ + if (isa_val.compare(cpu_isa_traits_t::user_option_env) == 0) \ max_cpu_isa_val = cpu_isa #define ELSEIF_HANDLE_CASE(cpu_isa) else IF_HANDLE_CASE(cpu_isa) @@ -206,7 +206,9 @@ status_t set_max_cpu_isa(dnnl_cpu_isa_t isa) { cpu_isa_t isa_to_set = isa_undef; #define HANDLE_CASE(cpu_isa) \ - case cpu_isa_traits::user_option_val: isa_to_set = cpu_isa; break; + case cpu_isa_traits_t::user_option_val: \ + isa_to_set = cpu_isa; \ + break; switch (isa) { HANDLE_CASE(isa_all); HANDLE_CASE(sse41); diff --git a/src/cpu/x64/cpu_isa_traits.hpp b/src/cpu/x64/cpu_isa_traits.hpp index b4e2d9ee36a..7fc472335c8 100644 --- a/src/cpu/x64/cpu_isa_traits.hpp +++ b/src/cpu/x64/cpu_isa_traits.hpp @@ -230,7 +230,7 @@ struct vreg_traits_t { }; template -struct cpu_isa_traits {}; /* ::vlen -> 32 (for avx2) */ +struct cpu_isa_traits_t {}; /* ::vlen -> 32 (for avx2) */ // pack struct so it can fit into a single 64-byte cache line #pragma pack(push, 1) @@ -244,13 +244,13 @@ struct palette_config_t { #pragma pack(pop) template <> -struct cpu_isa_traits { +struct cpu_isa_traits_t { static constexpr dnnl_cpu_isa_t user_option_val = dnnl_cpu_isa_default; static constexpr const char *user_option_env = "default"; }; template <> -struct cpu_isa_traits { +struct cpu_isa_traits_t { using Vmm = Xbyak::Xmm; static constexpr int vlen_shift = 4; static constexpr int vlen = vreg_traits_t::vlen; @@ -260,7 +260,7 @@ struct cpu_isa_traits { }; template <> -struct cpu_isa_traits { +struct cpu_isa_traits_t { using Vmm = Xbyak::Ymm; static constexpr int vlen_shift = 5; static constexpr int vlen = vreg_traits_t::vlen; @@ -270,25 +270,25 @@ struct cpu_isa_traits { }; template <> -struct cpu_isa_traits : public cpu_isa_traits { +struct cpu_isa_traits_t : public cpu_isa_traits_t { static constexpr dnnl_cpu_isa_t user_option_val = dnnl_cpu_isa_avx2; static constexpr const char *user_option_env = "avx2"; }; template <> -struct cpu_isa_traits : public cpu_isa_traits { +struct cpu_isa_traits_t : public cpu_isa_traits_t { static constexpr dnnl_cpu_isa_t user_option_val = dnnl_cpu_isa_avx2_vnni; static constexpr const char *user_option_env = "avx2_vnni"; }; template <> -struct cpu_isa_traits : public cpu_isa_traits { +struct cpu_isa_traits_t : public cpu_isa_traits_t { static constexpr dnnl_cpu_isa_t user_option_val = dnnl_cpu_isa_avx2_vnni_2; static constexpr const char *user_option_env = "avx2_vnni_2"; }; template <> -struct cpu_isa_traits { +struct cpu_isa_traits_t { using Vmm = Xbyak::Zmm; static constexpr int vlen_shift = 6; static constexpr int vlen = vreg_traits_t::vlen; @@ -298,21 +298,23 @@ struct cpu_isa_traits { }; template <> -struct cpu_isa_traits : public cpu_isa_traits { +struct cpu_isa_traits_t + : public cpu_isa_traits_t { static constexpr dnnl_cpu_isa_t user_option_val = dnnl_cpu_isa_avx512_core_vnni; static constexpr const char *user_option_env = "avx512_core_vnni"; }; template <> -struct cpu_isa_traits : public cpu_isa_traits { +struct cpu_isa_traits_t + : public cpu_isa_traits_t { static constexpr dnnl_cpu_isa_t user_option_val = dnnl_cpu_isa_avx512_core_bf16; static constexpr const char *user_option_env = "avx512_core_bf16"; }; template <> -struct cpu_isa_traits { +struct cpu_isa_traits_t { using Vmm = Xbyak::Zmm; static constexpr int vlen = vreg_traits_t::vlen; static constexpr dnnl_cpu_isa_t user_option_val @@ -321,13 +323,13 @@ struct cpu_isa_traits { }; template <> -struct cpu_isa_traits : public cpu_isa_traits { +struct cpu_isa_traits_t : public cpu_isa_traits_t { static constexpr dnnl_cpu_isa_t user_option_val = dnnl_cpu_isa_avx10_1_512; static constexpr const char *user_option_env = "avx10_1_512"; }; template <> -struct cpu_isa_traits { +struct cpu_isa_traits_t { using Vmm = Xbyak::Zmm; static constexpr dnnl_cpu_isa_t user_option_val = dnnl_cpu_isa_avx10_1_512_amx_fp16; @@ -443,11 +445,11 @@ inline int isa_max_vlen(cpu_isa_t isa) { MAYBE_UNUSED(is_sse41); if (is_avx512) - return cpu_isa_traits::vlen; + return cpu_isa_traits_t::vlen; else if (is_avx) - return cpu_isa_traits::vlen; + return cpu_isa_traits_t::vlen; else - return cpu_isa_traits::vlen; + return cpu_isa_traits_t::vlen; } inline int isa_num_vregs(cpu_isa_t isa) { @@ -459,11 +461,11 @@ inline int isa_num_vregs(cpu_isa_t isa) { MAYBE_UNUSED(is_sse41); if (is_avx512) - return cpu_isa_traits::n_vregs; + return cpu_isa_traits_t::n_vregs; else if (is_avx) - return cpu_isa_traits::n_vregs; + return cpu_isa_traits_t::n_vregs; else - return cpu_isa_traits::n_vregs; + return cpu_isa_traits_t::n_vregs; } } // namespace diff --git a/src/cpu/x64/cpu_reducer.cpp b/src/cpu/x64/cpu_reducer.cpp index b75cc838613..28beab84a9e 100644 --- a/src/cpu/x64/cpu_reducer.cpp +++ b/src/cpu/x64/cpu_reducer.cpp @@ -145,7 +145,7 @@ struct reducer_2d_driver_f_s_32_t : public reducer_2d_driver_t { this->paddd(x1, op); } - const int vlen = cpu_isa_traits::vlen; + const int vlen = cpu_isa_traits_t::vlen; const int typesize = sizeof(typename dnnl::impl::prec_traits_t::type); Xbyak::Reg64 reg_dst = abi_param1; @@ -205,7 +205,7 @@ struct reducer_2d_driver_f_s_32_t : public reducer_2d_driver_t { } void loop_x() { - const int nloads[] = {cpu_isa_traits::n_vregs, 1, 1}; + const int nloads[] = {cpu_isa_traits_t::n_vregs, 1, 1}; const int nbranches = sizeof(nloads) / sizeof(nloads[0]); const int load_len[nbranches] = {vlen, vlen, typesize}; diff --git a/src/cpu/x64/gemm/gemm_driver.cpp b/src/cpu/x64/gemm/gemm_driver.cpp index f07533587b5..aaacd0931e2 100644 --- a/src/cpu/x64/gemm/gemm_driver.cpp +++ b/src/cpu/x64/gemm/gemm_driver.cpp @@ -80,15 +80,15 @@ int get_vector_length() { //dummy if #if __BUILD_GEMM_AVX512 } else if (mayiuse(avx512_core)) { - v_bytes = cpu_isa_traits::vlen; + v_bytes = cpu_isa_traits_t::vlen; #endif #if __BUILD_GEMM_AVX2 } else if (mayiuse(avx)) { - v_bytes = cpu_isa_traits::vlen; + v_bytes = cpu_isa_traits_t::vlen; #endif #if __BUILD_GEMM_SSE41 } else if (mayiuse(sse41)) { - v_bytes = cpu_isa_traits::vlen; + v_bytes = cpu_isa_traits_t::vlen; #endif } else { assert(!"not supposed to be reached."); diff --git a/src/cpu/x64/gemm_bf16_convolution.cpp b/src/cpu/x64/gemm_bf16_convolution.cpp index 518be7b26c2..4fd77fddbfd 100644 --- a/src/cpu/x64/gemm_bf16_convolution.cpp +++ b/src/cpu/x64/gemm_bf16_convolution.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -117,7 +117,7 @@ gemm_bf16_convolution_fwd_t::pp_ker_t::pp_ker_t(const pd_t *pd) if (jcp_.with_bias) vreg_bias = Zmm(data_reg_base_idx_++); - vlen_ = cpu_isa_traits::vlen / sizeof(float); + vlen_ = cpu_isa_traits_t::vlen / sizeof(float); isa_ = mayiuse(avx512_core_bf16) ? avx512_core_bf16 : bf16_emulation_t::get_isa(); diff --git a/src/cpu/x64/injectors/jit_uni_binary_injector.cpp b/src/cpu/x64/injectors/jit_uni_binary_injector.cpp index 7a45796c4df..5fa3cb8d475 100644 --- a/src/cpu/x64/injectors/jit_uni_binary_injector.cpp +++ b/src/cpu/x64/injectors/jit_uni_binary_injector.cpp @@ -395,7 +395,7 @@ void jit_uni_binary_injector_t::compute_vector_range( const auto end_idx = *(vmm_idxs.rbegin()); // Phase 1 Validate temporary vmm user hint - static constexpr int max_vmm_idx = cpu_isa_traits::n_vregs - 1; + static constexpr int max_vmm_idx = cpu_isa_traits_t::n_vregs - 1; auto &vmm_hint = rhs_arg_static_params_.rhs_dt_helper_vmm_idx; vmm_hint = adjust_temp_vmm_hint(vmm_hint, start_idx, end_idx, max_vmm_idx); @@ -422,7 +422,7 @@ void jit_uni_binary_injector_t::compute_vector_range( || rhs_arg_data_type != data_type::f32 || bcast_f32_non_avx512 || should_preserve_vmm_tail || post_op.is_prelu(); const auto tail_load_mode = rhs_arg_params.tail_load_mode; - const int simd_w = cpu_isa_traits::vlen + const int simd_w = cpu_isa_traits_t::vlen / types::data_type_size(dst_d.data_type()); const int blk_size = dst_d.blocking_desc().inner_blks[0]; const bool use_offset_conversions @@ -847,7 +847,7 @@ void jit_uni_binary_injector_t::calculate_oc_blocked_base( // c = ((offset % strides[0]) / strides[1]) * strides[ndims - 1] + offset % blk_size // output = rax const auto dst_d = rhs_arg_static_params_.dst_d; - const int simd_w = cpu_isa_traits::vlen + const int simd_w = cpu_isa_traits_t::vlen / types::data_type_size(dst_d.data_type()); const int blk_size = dst_d.blocking_desc().inner_blks[0]; const auto rax = host_->rax; @@ -1234,7 +1234,7 @@ void jit_uni_binary_injector_t::calculate_mb_sp_blocked_base( // mb_sp_off = offset - (c * stride_c) - (n * (C - 1)DHW) - c % blk_size // output = rax const auto dst_d = rhs_arg_static_params_.dst_d; - const int simd_w = cpu_isa_traits::vlen + const int simd_w = cpu_isa_traits_t::vlen / types::data_type_size(dst_d.data_type()); const int blk_size = dst_d.blocking_desc().inner_blks[0]; diff --git a/src/cpu/x64/injectors/jit_uni_binary_injector.hpp b/src/cpu/x64/injectors/jit_uni_binary_injector.hpp index a38d34c6d19..1896e5fefaa 100644 --- a/src/cpu/x64/injectors/jit_uni_binary_injector.hpp +++ b/src/cpu/x64/injectors/jit_uni_binary_injector.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -255,7 +255,7 @@ bool is_supported(cpu_isa_t isa, const dnnl::impl::memory_desc_t &src1_desc, * isa: sse41, avx, avx2, avx512 with core, bf16 extensions as well as data * types: f32, bf16, s32, u8, s8. */ -template ::Vmm> +template ::Vmm> class jit_uni_binary_injector_t { public: jit_uni_binary_injector_t( diff --git a/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp b/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp index 9c2d93fa730..924611c74c0 100644 --- a/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp +++ b/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp @@ -75,7 +75,7 @@ bool is_supported(cpu_isa_t isa, alg_kind_t alg, data_type_t dt); } // namespace eltwise_injector -template ::Vmm> +template ::Vmm> struct jit_uni_eltwise_injector { using Vmm = Wmm; @@ -182,7 +182,7 @@ struct jit_uni_eltwise_injector { static constexpr size_t vlen_ = vreg_traits_t::vlen; static constexpr size_t preserved_vecs_max_ = 6; static constexpr size_t preserved_gprs_max_ = 5; - static constexpr size_t n_vregs_ = cpu_isa_traits::n_vregs; + static constexpr size_t n_vregs_ = cpu_isa_traits_t::n_vregs; static constexpr int n_mantissa_bits_ = 23; const size_t n_vregs_to_preserve_; diff --git a/src/cpu/x64/injectors/jit_uni_postops_injector.hpp b/src/cpu/x64/injectors/jit_uni_postops_injector.hpp index 53c27436045..6fedc8a01c6 100644 --- a/src/cpu/x64/injectors/jit_uni_postops_injector.hpp +++ b/src/cpu/x64/injectors/jit_uni_postops_injector.hpp @@ -112,7 +112,7 @@ class jit_uni_postops_injector_base_t { // A parent isa-specific post-ops injector class. A specific instance is // assigned based on `cpu_isa_t isa` argument in the base class. -template ::Vmm> +template ::Vmm> class jit_uni_postops_injector_t : public jit_uni_postops_injector_base_t { public: /* diff --git a/src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.hpp b/src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.hpp index b7d429e00e4..a2e7e3c48ff 100644 --- a/src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.hpp +++ b/src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2016-2023 Intel Corporation +* Copyright 2016-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,7 +52,7 @@ struct jit_avx2_1x1_conv_kernel_f32 : public jit_generator { postops_injector_; constexpr static int isa_simd_width_ - = cpu_isa_traits::vlen / sizeof(float); + = cpu_isa_traits_t::vlen / sizeof(float); using reg64_t = const Xbyak::Reg64; using ymm_t = const Xbyak::Ymm; diff --git a/src/cpu/x64/jit_avx2_conv_kernel_f32.hpp b/src/cpu/x64/jit_avx2_conv_kernel_f32.hpp index 4fc17c34e03..72fab8e4758 100644 --- a/src/cpu/x64/jit_avx2_conv_kernel_f32.hpp +++ b/src/cpu/x64/jit_avx2_conv_kernel_f32.hpp @@ -51,7 +51,7 @@ struct jit_avx2_conv_fwd_kernel_f32 : public jit_generator { postops_injector_; constexpr static int isa_simd_width_ - = cpu_isa_traits::vlen / sizeof(float); + = cpu_isa_traits_t::vlen / sizeof(float); using reg64_t = const Xbyak::Reg64; reg64_t reg_input = rax; reg64_t aux_reg_input = r8; diff --git a/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.cpp b/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.cpp index e2476a14263..ef3dbfddea4 100644 --- a/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.cpp @@ -339,7 +339,7 @@ void jit_avx512_common_1x1_conv_kernel::reduce_loop( }; Label unaligned_store, end_store; - test(aux_reg_output_data, cpu_isa_traits::vlen - 1); + test(aux_reg_output_data, cpu_isa_traits_t::vlen - 1); jnz(unaligned_store, T_NEAR); store_output(true); jmp(end_store, T_NEAR); @@ -574,7 +574,7 @@ status_t jit_avx512_common_1x1_conv_kernel::init_conf(jit_1x1_conv_conf_t &jcp, jcp.nthr = nthreads; const bool with_groups = weights_d.ndims() == src_d.ndims() + 1; - const int simd_w = cpu_isa_traits::vlen / sizeof(float); + const int simd_w = cpu_isa_traits_t::vlen / sizeof(float); const int ndims = src_d.ndims(); jcp.prop_kind = cd.prop_kind; diff --git a/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.hpp b/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.hpp index 9a926841fc0..64dd8792ca5 100644 --- a/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2017-2023 Intel Corporation +* Copyright 2017-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,7 +52,7 @@ struct jit_avx512_common_1x1_conv_kernel : public jit_generator { postops_injector_; constexpr static int isa_simd_width_ - = cpu_isa_traits::vlen / sizeof(float); + = cpu_isa_traits_t::vlen / sizeof(float); using reg64_t = const Xbyak::Reg64; using zmm_t = const Xbyak::Zmm; diff --git a/src/cpu/x64/jit_avx512_common_conv_kernel.cpp b/src/cpu/x64/jit_avx512_common_conv_kernel.cpp index 174410d77eb..71004d8f505 100644 --- a/src/cpu/x64/jit_avx512_common_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_common_conv_kernel.cpp @@ -865,7 +865,7 @@ status_t jit_avx512_common_conv_fwd_kernel::init_conf(jit_conv_conf_t &jcp, bool ok_to_pad_channels = true && !is_data_layout_nxc && jcp.ngroups == 1 && src_d.data_type() == data_type::f32; - const int full_simd_w = cpu_isa_traits::vlen / typesize; + const int full_simd_w = cpu_isa_traits_t::vlen / typesize; jcp.simd_w = full_simd_w; bool ok_to_try_lower_zmm = true && IMPLICATION(is_data_layout_nxc, @@ -1938,7 +1938,7 @@ status_t jit_avx512_common_conv_bwd_data_kernel_f32::init_conf( bool ok_to_pad_channels = true && !is_data_layout_nxc && jcp.ngroups == 1 && diff_src_d.data_type() == data_type::f32; - const int full_simd_w = cpu_isa_traits::vlen / typesize; + const int full_simd_w = cpu_isa_traits_t::vlen / typesize; jcp.simd_w = full_simd_w; bool ok_to_try_lower_zmm = true && IMPLICATION(is_data_layout_nxc, @@ -3046,7 +3046,7 @@ void jit_avx512_common_conv_bwd_weights_kernel_f32::maybe_zero_kernel() { L(zeroing_loop); { assert(jcp.oc_block * jcp.typesize_out - == cpu_isa_traits::vlen); + == cpu_isa_traits_t::vlen); for (int ic1 = 0; ic1 < jcp.ic_block; ic1++) vmovups(ptr[reg_kernel + reg_tmp + ic1 * jcp.oc_block * jcp.typesize_out], @@ -3917,7 +3917,7 @@ status_t jit_avx512_common_conv_bwd_weights_kernel_f32::init_conf( jcp = zero(); - jcp.simd_w = cpu_isa_traits::vlen / typesize; + jcp.simd_w = cpu_isa_traits_t::vlen / typesize; jcp.nthr = jcp.aligned_threads = nthreads; jcp.ndims = ndims; jcp.prop_kind = cd.prop_kind; diff --git a/src/cpu/x64/jit_avx512_common_conv_kernel.hpp b/src/cpu/x64/jit_avx512_common_conv_kernel.hpp index f4bc66a0b52..3a4e0f6d7ab 100644 --- a/src/cpu/x64/jit_avx512_common_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_common_conv_kernel.hpp @@ -43,7 +43,7 @@ struct _jit_avx512_common_conv_fwd_kernel : public jit_generator { private: constexpr static int isa_simd_width_ - = cpu_isa_traits::vlen / sizeof(float); + = cpu_isa_traits_t::vlen / sizeof(float); using reg64_t = const Xbyak::Reg64; enum { typesize = sizeof(float), diff --git a/src/cpu/x64/jit_avx512_core_amx_1x1_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_amx_1x1_conv_kernel.hpp index 1fa1ba63d2a..8bc8cdb3cf8 100644 --- a/src/cpu/x64/jit_avx512_core_amx_1x1_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_amx_1x1_conv_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2022 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,7 +52,7 @@ struct jit_avx512_core_amx_1x1_fwd_kernel_t : public jit_generator { private: constexpr static int isa_simd_width_ - = cpu_isa_traits::vlen / sizeof(float); + = cpu_isa_traits_t::vlen / sizeof(float); std::unique_ptr> postops_injector_; diff --git a/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp index b7c37723ae9..c44760677f7 100644 --- a/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp @@ -269,7 +269,7 @@ struct jit_avx512_core_amx_fwd_kernel_t : public jit_generator { private: constexpr static int isa_simd_width_ - = cpu_isa_traits::vlen / sizeof(float); + = cpu_isa_traits_t::vlen / sizeof(float); std::unique_ptr> postops_injector_; std::unique_ptr copy_to_pbuffer_; diff --git a/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.cpp b/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.cpp index 65f80b9444b..85653c984ab 100644 --- a/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.cpp @@ -1193,7 +1193,7 @@ status_t jit_avx512_core_bf16_1x1_conv_kernel::init_conf( const memory_desc_wrapper &dst_d, primitive_attr_t &attr, int nthreads, bool reduce_src) { const bool with_groups = weights_d.ndims() == src_d.ndims() + 1; - const int simd_w = cpu_isa_traits::vlen / sizeof(float); + const int simd_w = cpu_isa_traits_t::vlen / sizeof(float); const int ndims = src_d.ndims(); // Big int (> INT_MAX) values are unsupported and jcp fields may overflow diff --git a/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.hpp index 45ca07b4bb7..384c474132f 100644 --- a/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.hpp @@ -49,7 +49,7 @@ struct jit_avx512_core_bf16_1x1_conv_kernel : public jit_generator { private: constexpr static int isa_simd_width_ - = cpu_isa_traits::vlen / sizeof(float); + = cpu_isa_traits_t::vlen / sizeof(float); std::unique_ptr> postops_injector_; diff --git a/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.cpp b/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.cpp index 716ac7f849a..503da652a16 100644 --- a/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.cpp @@ -911,7 +911,7 @@ status_t jit_avx512_core_bf16_fwd_kernel::init_conf(jit_conv_conf_t &jcp, const int regs = isa_has_bf16(jcp.isa) ? 31 /* expl_bcast case */ : 26; const bool ok_to_pad_channels = jcp.ngroups == 1 && !is_data_layout_nxc; - jcp.simd_w = cpu_isa_traits::vlen / sizeof(float); + jcp.simd_w = cpu_isa_traits_t::vlen / sizeof(float); const bool ok_to_try_lower_zmm = true && IMPLICATION(is_data_layout_nxc, @@ -1632,7 +1632,7 @@ status_t jit_avx512_core_bf16_bwd_data_kernel::init_conf(jit_conv_conf_t &jcp, bool ok_to_pad_channels = jcp.ngroups == 1 && !is_data_layout_nxc; - jcp.simd_w = cpu_isa_traits::vlen / sizeof(float); + jcp.simd_w = cpu_isa_traits_t::vlen / sizeof(float); const bool ok_to_try_lower_zmm = true && IMPLICATION(is_data_layout_nxc, @@ -3294,7 +3294,7 @@ void jit_avx512_core_bf16_conv_bwd_weights_kernel_f32::maybe_zero_kernel() { xor_(reg_tmp, reg_tmp); L(zeroing_loop); { - assert(get_kernel_offset(1, 0) == cpu_isa_traits::vlen); + assert(get_kernel_offset(1, 0) == cpu_isa_traits_t::vlen); for (int ic1 = 0; ic1 < jcp.ic_block; ic1++) vmovups(ptr[reg_kernel + reg_tmp + get_kernel_offset(ic1, 0)], zero); @@ -4137,7 +4137,7 @@ status_t jit_avx512_core_bf16_conv_bwd_weights_kernel_f32::init_conf( jit_conv_conf_t &jcp, const convolution_desc_t &cd, memory_desc_t &src_md, memory_desc_t &diff_weights_md, memory_desc_t &diff_bias_md, memory_desc_t &diff_dst_md, int nthreads) { - const int simd_w = cpu_isa_traits::vlen / sizeof(float); + const int simd_w = cpu_isa_traits_t::vlen / sizeof(float); const memory_desc_wrapper src_d(&src_md); const memory_desc_wrapper diff_weights_d(&diff_weights_md); diff --git a/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.hpp index 123e0f2df69..1acfbb7f4c2 100644 --- a/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.hpp @@ -43,7 +43,7 @@ struct _jit_avx512_core_bf16_fwd_kernel : public jit_generator { private: constexpr static int isa_simd_width_ - = cpu_isa_traits::vlen / sizeof(float); + = cpu_isa_traits_t::vlen / sizeof(float); using Vmm_down_t = typename utils::conditional::value, Xbyak::Ymm, Xbyak::Xmm>::type; diff --git a/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.cpp b/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.cpp index 4cd112b00cf..c03263eef75 100644 --- a/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,7 +43,7 @@ jit_avx512_dw_conv_fwd_kernel_bf16::jit_avx512_dw_conv_fwd_kernel_bf16( static constexpr size_t helper_vmm_idx = 31; static constexpr bool use_exact_tail_scalar_bcast = true; const size_t tail_size = jcp.oc_without_padding - % (cpu_isa_traits::vlen / sizeof(float)); + % (cpu_isa_traits_t::vlen / sizeof(float)); const rhs_arg_static_params_t rhs_arg_static_params {helper_vmm_idx, r14, r15, r12, preserve_gpr, preserve_vmm, diff --git a/src/cpu/x64/jit_avx512_core_fp8cvt.cpp b/src/cpu/x64/jit_avx512_core_fp8cvt.cpp index 9acd4c719ce..6b6c433f3eb 100644 --- a/src/cpu/x64/jit_avx512_core_fp8cvt.cpp +++ b/src/cpu/x64/jit_avx512_core_fp8cvt.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2024 Intel Corporation +* Copyright 2024-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -215,7 +215,7 @@ void fp8_emulation_e5m2_t::vcvt_f8_to_f16_vnni_block(int num_rows, prepare_f8_to_f16_vnni_masks(zmm_permute_idx); - const auto zmm_width_in_bytes = cpu_isa_traits::vlen; + const auto zmm_width_in_bytes = cpu_isa_traits_t::vlen; for (int r = 0; r < num_rows; r += 2) { perform_f8_to_f16_vnni_conversion( @@ -304,7 +304,7 @@ void fp8_emulation_e4m3_t::vcvt_f8_to_f16_vnni(const Xbyak::Zmm &zmm_out1, void fp8_emulation_e4m3_t::vcvt_f8_to_f16_vnni_block(int num_rows, const Xbyak::Reg64 ®_data_in, const Xbyak::Reg64 ®_stride_in, const Xbyak::Reg64 ®_data_out) { - const auto zmm_width_in_bytes = cpu_isa_traits::vlen; + const auto zmm_width_in_bytes = cpu_isa_traits_t::vlen; const Xbyak::Zmm zmm_out1(xmm_aux4_.getIdx()); const Xbyak::Zmm zmm_out2(xmm_aux5_.getIdx()); const Xbyak::Ymm ymm_out1(zmm_out1.getIdx()); diff --git a/src/cpu/x64/jit_avx512_core_resampling.cpp b/src/cpu/x64/jit_avx512_core_resampling.cpp index 079d3627672..0ced081dfa0 100644 --- a/src/cpu/x64/jit_avx512_core_resampling.cpp +++ b/src/cpu/x64/jit_avx512_core_resampling.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -752,7 +752,7 @@ struct jit_avx512_core_resampling_kernel_t } static constexpr std::size_t simd_w() { - return cpu_isa_traits::vlen / sizeof(float); + return cpu_isa_traits_t::vlen / sizeof(float); } Zmm zmm_src = Zmm(1); diff --git a/src/cpu/x64/jit_avx512_core_scale_precompute.hpp b/src/cpu/x64/jit_avx512_core_scale_precompute.hpp index a93fceeca65..8f74ad4dc4c 100644 --- a/src/cpu/x64/jit_avx512_core_scale_precompute.hpp +++ b/src/cpu/x64/jit_avx512_core_scale_precompute.hpp @@ -88,8 +88,8 @@ struct jit_avx512_core_scale_precompute_t : public jit_generator { private: constexpr static int simd_w_ - = cpu_isa_traits::vlen / sizeof(float); - using Vmm = typename cpu_isa_traits::Vmm; + = cpu_isa_traits_t::vlen / sizeof(float); + using Vmm = typename cpu_isa_traits_t::Vmm; const primitive_attr_t *attr_; const bool with_wei_scales_; diff --git a/src/cpu/x64/jit_avx512_core_x8s8s32x_1x1_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_x8s8s32x_1x1_conv_kernel.hpp index 6eb52072d8c..24b9bd63760 100644 --- a/src/cpu/x64/jit_avx512_core_x8s8s32x_1x1_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_x8s8s32x_1x1_conv_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2018-2023 Intel Corporation +* Copyright 2018-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,7 +42,7 @@ struct _jit_avx512_core_x8s8s32x_1x1_conv_kernel : public jit_generator { private: constexpr static int isa_simd_width_ - = cpu_isa_traits::vlen / sizeof(float); + = cpu_isa_traits_t::vlen / sizeof(float); using Vmm_down_t = typename utils::conditional::value, Xbyak::Ymm, Xbyak::Xmm>::type; diff --git a/src/cpu/x64/jit_avx512_core_x8s8s32x_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_x8s8s32x_conv_kernel.hpp index 2a0333317f3..32437a3315a 100644 --- a/src/cpu/x64/jit_avx512_core_x8s8s32x_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_x8s8s32x_conv_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2016-2023 Intel Corporation +* Copyright 2016-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,7 +45,7 @@ struct _jit_avx512_core_x8s8s32x_fwd_kernel : public jit_generator { private: constexpr static int isa_simd_width_ - = cpu_isa_traits::vlen / sizeof(float); + = cpu_isa_traits_t::vlen / sizeof(float); using Vmm_down_t = typename utils::conditional::value, Xbyak::Ymm, Xbyak::Xmm>::type; diff --git a/src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp b/src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp index ec8d52a5a76..0732c3d0ae1 100644 --- a/src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp +++ b/src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp @@ -233,7 +233,7 @@ struct brgemm_convolution_bwd_strided_t : public primitive_t { std::vector> kernels_po_; - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; std::unique_ptr> diff --git a/src/cpu/x64/jit_brgemm_conv_trans_kernel.cpp b/src/cpu/x64/jit_brgemm_conv_trans_kernel.cpp index 7e043de1137..930d8dc15e6 100644 --- a/src/cpu/x64/jit_brgemm_conv_trans_kernel.cpp +++ b/src/cpu/x64/jit_brgemm_conv_trans_kernel.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2021-2024 Intel Corporation +* Copyright 2021-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,7 +42,7 @@ jit_avx512_core_brgemm_conv_trans_kernel_t:: , iw_size(inp_dsz * jcp.ngroups * jcp.ic_without_padding) , dst_w_block(dst_w(jcp, jcp.ow_block)) , dst_stride(jcp.copy_block_only ? dst_w_block : jcp.iwp) - , VL(cpu_isa_traits::vlen) + , VL(cpu_isa_traits_t::vlen) , n_vec(ic_block_sz / jcp.simd_w) , n_tail_vec((jcp.ic_without_padding % ic_block_sz) / jcp.simd_w) { diff --git a/src/cpu/x64/jit_brgemm_inner_product.hpp b/src/cpu/x64/jit_brgemm_inner_product.hpp index 281909b59d0..cdd8da4b948 100644 --- a/src/cpu/x64/jit_brgemm_inner_product.hpp +++ b/src/cpu/x64/jit_brgemm_inner_product.hpp @@ -631,8 +631,8 @@ struct brgemm_inner_product_bwd_weights_t : public primitive_t { private: struct thread_info_t; - using ker_diff_bias_t - = jit_brgemm_kernel_diff_bias_t::Vmm>; + using ker_diff_bias_t = jit_brgemm_kernel_diff_bias_t< + typename cpu_isa_traits_t::Vmm>; std::unique_ptr kernels_db_[2][2]; std::unique_ptr brg_kernels_[brgemm_inner_product_utils::max_num_brg_kernels_ip]; diff --git a/src/cpu/x64/jit_gemm_inner_product_utils.cpp b/src/cpu/x64/jit_gemm_inner_product_utils.cpp index 782a7e0887a..8b8f30eb22d 100644 --- a/src/cpu/x64/jit_gemm_inner_product_utils.cpp +++ b/src/cpu/x64/jit_gemm_inner_product_utils.cpp @@ -178,7 +178,7 @@ struct jit_pp_kernel_t : public pp_kernel_t, public jit_generator { int compute_vreg_bias_shift_ = 0; int compute_vreg_prev_dst_shift_ = 0; - const size_t vlen = cpu_isa_traits::vlen / sizeof(float); + const size_t vlen = cpu_isa_traits_t::vlen / sizeof(float); static constexpr int reg64_size_ = sizeof(int64_t); static constexpr int reg_binary_post_op_oc_off_ = 0; static constexpr int reg_binary_post_op_offset_ = 1 * reg64_size_; diff --git a/src/cpu/x64/jit_gemm_x8s8s32x_convolution_utils.cpp b/src/cpu/x64/jit_gemm_x8s8s32x_convolution_utils.cpp index 9bb3d0e1687..36be819d8ff 100644 --- a/src/cpu/x64/jit_gemm_x8s8s32x_convolution_utils.cpp +++ b/src/cpu/x64/jit_gemm_x8s8s32x_convolution_utils.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -160,7 +160,7 @@ jit_pp_ker_t::jit_pp_ker_t( , zmm_step_(1u) , bias_step_factor_(jcp_.with_bias ? zmm_step_++ : 0u) , sum_step_factor_(jcp_.with_sum ? zmm_step_++ : 0) - , max_unroll_((cpu_isa_traits::n_vregs + , max_unroll_((cpu_isa_traits_t::n_vregs - number_of_reserved_zmm_regs_) / zmm_step_) , zp_pad_comp_helper_(jit_gemm_convolution_utils::padding_exists(jcp) @@ -348,7 +348,7 @@ void jit_pp_ker_t::generate() { using namespace Xbyak; using namespace utils; - size_t vlen = cpu_isa_traits::vlen / sizeof(float); + size_t vlen = cpu_isa_traits_t::vlen / sizeof(float); for (; vlen >= 1 && (jcp_.oc % vlen != 0); --vlen) {} preamble(); diff --git a/src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.hpp b/src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.hpp index 6bb4fc45c34..284406e3d80 100644 --- a/src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.hpp +++ b/src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2017-2023 Intel Corporation +* Copyright 2017-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,7 +45,8 @@ struct jit_sse41_1x1_conv_kernel_f32 : public jit_generator { const primitive_attr_t &attr_; private: - static constexpr auto simd_w_ = cpu_isa_traits::vlen / sizeof(float); + static constexpr auto simd_w_ + = cpu_isa_traits_t::vlen / sizeof(float); using reg64_t = const Xbyak::Reg64; using xmm_t = const Xbyak::Xmm; diff --git a/src/cpu/x64/jit_sse41_conv_kernel_f32.hpp b/src/cpu/x64/jit_sse41_conv_kernel_f32.hpp index 50f03d9e5aa..2534f41d899 100644 --- a/src/cpu/x64/jit_sse41_conv_kernel_f32.hpp +++ b/src/cpu/x64/jit_sse41_conv_kernel_f32.hpp @@ -44,7 +44,8 @@ struct jit_sse41_conv_fwd_kernel_f32 : public jit_generator { const primitive_attr_t &attr_; private: - static constexpr auto simd_w_ = cpu_isa_traits::vlen / sizeof(float); + static constexpr auto simd_w_ + = cpu_isa_traits_t::vlen / sizeof(float); using reg64_t = const Xbyak::Reg64; reg64_t reg_input = rax; reg64_t aux_reg_input = r8; diff --git a/src/cpu/x64/jit_uni_batch_normalization.cpp b/src/cpu/x64/jit_uni_batch_normalization.cpp index bd022ebeffd..693b34efacc 100644 --- a/src/cpu/x64/jit_uni_batch_normalization.cpp +++ b/src/cpu/x64/jit_uni_batch_normalization.cpp @@ -245,7 +245,7 @@ struct jit_bnorm_t : public jit_generator { : (isa == avx2) ? yword : zword; - const int vlen = isa == sse41 ? 32 : cpu_isa_traits::vlen; + const int vlen = isa == sse41 ? 32 : cpu_isa_traits_t::vlen; const batch_normalization_pd_t *pd_ = nullptr; const jit_bnorm_conf_t *jbp_ = nullptr; @@ -2313,7 +2313,7 @@ struct driver_t : public c_compatible { private: enum { simd_w = isa == sse41 ? 8 - : cpu_isa_traits::vlen + : cpu_isa_traits_t::vlen / sizeof(acc_data_t) // BF16 will expand to FP32 }; @@ -2420,7 +2420,7 @@ status_t jit_uni_batch_normalization_fwd_t::pd_t::init(engine_t *engine) { "bad padded dimensions for current isa"); // Only IC % simd_w == 0 is supported for now - const int simd_w = cpu_isa_traits::vlen / sizeof(acc_data_t); + const int simd_w = cpu_isa_traits_t::vlen / sizeof(acc_data_t); VDISPATCH_BNORM(!(src_d.matches_one_of_tag(nc, nwc, nhwc, ndhwc) && src_d.padded_dims()[1] % simd_w != 0), VERBOSE_UNSUPPORTED_PAD_FEATURE, diff --git a/src/cpu/x64/jit_uni_batch_normalization_s8.cpp b/src/cpu/x64/jit_uni_batch_normalization_s8.cpp index 5ceb6f59eed..1b21c77165e 100644 --- a/src/cpu/x64/jit_uni_batch_normalization_s8.cpp +++ b/src/cpu/x64/jit_uni_batch_normalization_s8.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -49,10 +49,10 @@ struct jit_bnorm_base_t : public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_bnorm_s8_t) - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; const AddressFrame &vmmword = (isa == sse41) ? xword : ((isa == avx2) ? yword : zword); - const int vlen = cpu_isa_traits::vlen; + const int vlen = cpu_isa_traits_t::vlen; const batch_normalization_pd_t *pd_; @@ -82,7 +82,7 @@ struct jit_bnorm_base_t : public jit_generator { Vmm vmm_aux = Vmm(isa == avx512_core ? 28 : 10); // shared with 'veps' Vmm vmm_mask = Vmm(0); // used for AVX2 and SSE41 - size_t simd_w_ = cpu_isa_traits::vlen / sizeof(float); + size_t simd_w_ = cpu_isa_traits_t::vlen / sizeof(float); size_t c_in_xmm_ = (isa == sse41) ? 8 : 16; size_t chan_data_offt_; size_t num_c_blocks_; diff --git a/src/cpu/x64/jit_uni_binary.cpp b/src/cpu/x64/jit_uni_binary.cpp index 47076bc0678..24a6e02eb06 100644 --- a/src/cpu/x64/jit_uni_binary.cpp +++ b/src/cpu/x64/jit_uni_binary.cpp @@ -512,8 +512,8 @@ bool jit_uni_binary_t::post_ops_ok(const primitive_attr_t *attr, } } - const int vlen = is_avx512_core ? cpu_isa_traits::vlen - : cpu_isa_traits::vlen; + const int vlen = is_avx512_core ? cpu_isa_traits_t::vlen + : cpu_isa_traits_t::vlen; const bool postops_per_oc_broadcast_exists = binary_injector::any_binary_postop_rhs_per_oc_broadcast( p, src0_d, supported_strategies); diff --git a/src/cpu/x64/jit_uni_convert_xf16.hpp b/src/cpu/x64/jit_uni_convert_xf16.hpp index 94fc8287758..17ca3c6aa61 100644 --- a/src/cpu/x64/jit_uni_convert_xf16.hpp +++ b/src/cpu/x64/jit_uni_convert_xf16.hpp @@ -70,8 +70,8 @@ struct jit_uni_cvt_ps_to_xf16_t : public jit_generator { const bool is_dynamic_size_; const int tail_size_; - constexpr static int simd_w_ = cpu_isa_traits::vlen / sizeof(float); - using Vmm = typename cpu_isa_traits::Vmm; + constexpr static int simd_w_ = cpu_isa_traits_t::vlen / sizeof(float); + using Vmm = typename cpu_isa_traits_t::Vmm; using Vmm_down_t = typename vreg_traits_t::Vmm_lower_t; const Vmm vmm_input = Vmm(0); @@ -174,8 +174,8 @@ struct jit_uni_cvt_xf16_to_ps_t : public jit_generator { protected: constexpr static int elem_granularity = isa == avx2_vnni_2 ? 2 : 1; - constexpr static int simd_w_ = cpu_isa_traits::vlen / sizeof(float); - using Vmm = typename cpu_isa_traits::Vmm; + constexpr static int simd_w_ = cpu_isa_traits_t::vlen / sizeof(float); + using Vmm = typename cpu_isa_traits_t::Vmm; using Vmm_down_t = typename vreg_traits_t::Vmm_lower_t; const impl::data_type_t input_dt_; diff --git a/src/cpu/x64/jit_uni_deconv_zp_pad_str_kernel.cpp b/src/cpu/x64/jit_uni_deconv_zp_pad_str_kernel.cpp index 4dc83eebaef..dc8571ac5e6 100644 --- a/src/cpu/x64/jit_uni_deconv_zp_pad_str_kernel.cpp +++ b/src/cpu/x64/jit_uni_deconv_zp_pad_str_kernel.cpp @@ -122,7 +122,7 @@ void jit_uni_deconv_zp_pad_str_kernel_t::init() { template Vmm jit_uni_deconv_zp_pad_str_kernel_t::get_next_vmm() { - static constexpr int max_v_regs = cpu_isa_traits::n_vregs; + static constexpr int max_v_regs = cpu_isa_traits_t::n_vregs; const Vmm vmm {static_cast(current_vmm_++)}; diff --git a/src/cpu/x64/jit_uni_dw_conv_kernel_f32.cpp b/src/cpu/x64/jit_uni_dw_conv_kernel_f32.cpp index 290cba2d4e2..682bca89025 100644 --- a/src/cpu/x64/jit_uni_dw_conv_kernel_f32.cpp +++ b/src/cpu/x64/jit_uni_dw_conv_kernel_f32.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -46,7 +46,7 @@ jit_uni_dw_conv_fwd_kernel_f32::jit_uni_dw_conv_fwd_kernel_f32( static constexpr size_t helper_vmm_idx = 31; static constexpr bool use_exact_tail_scalar_bcast = true; const size_t tail_size = jcp.oc_without_padding - % (cpu_isa_traits::vlen / sizeof(float)); + % (cpu_isa_traits_t::vlen / sizeof(float)); rhs_arg_static_params_t rhs_arg_static_params {helper_vmm_idx, r14, r15, r12, preserve_gpr, preserve_vmm, GET_OFF(post_ops_binary_rhs_arg_vec), GET_OFF(dst_orig), @@ -73,7 +73,7 @@ void jit_uni_dw_conv_fwd_kernel_f32::load_src( const auto ch_blk = jcp.ch_block; const auto ocb_stride = dst_layout_nxc ? ch_blk : jcp.oh * jcp.ow * ch_blk; const auto ow_stride = dst_layout_nxc ? jcp.ngroups : ch_blk; - const int vlen = cpu_isa_traits::vlen / sizeof(float); + const int vlen = cpu_isa_traits_t::vlen / sizeof(float); const int c_tail = jcp.oc % jcp.ch_block; const int repeats = max_repeats(); @@ -140,7 +140,7 @@ void jit_uni_dw_conv_fwd_kernel_f32::apply_filter_unrolled( const auto icb_stride = src_layout_nxc ? ch_blk : (jcp.is_fused_conv ? 1 : jcp.ih) * jcp.iw * ch_blk; - const int vlen = cpu_isa_traits::vlen / sizeof(float); + const int vlen = cpu_isa_traits_t::vlen / sizeof(float); auto get_input_spatial_index = [=](int oi, int ki) { return (ki * dilate_w + oi * stride_w - pad_l); @@ -290,7 +290,7 @@ void jit_uni_dw_conv_fwd_kernel_f32::apply_postops( [&](const int r, const int ch, const int ow, const bool mask_flag_blocked_layout) { const int vlen - = cpu_isa_traits::vlen / sizeof(float); + = cpu_isa_traits_t::vlen / sizeof(float); const bool is_tail_load = check_if_tail_load( is_ch_tail, c_tail, ch, ur_ch_blocks, vlen, r); if ((ch + 1 == ur_ch_blocks) && is_ch_tail @@ -408,7 +408,7 @@ void jit_uni_dw_conv_fwd_kernel_f32::store_dst( const auto ch_blk = jcp.ch_block; const auto ocb_stride = dst_layout_nxc ? ch_blk : jcp.oh * jcp.ow * ch_blk; const auto ow_stride = dst_layout_nxc ? jcp.ngroups : ch_blk; - const int vlen = cpu_isa_traits::vlen / sizeof(float); + const int vlen = cpu_isa_traits_t::vlen / sizeof(float); const int c_tail = jcp.oc_without_padding % jcp.ch_block; const int repeats = max_repeats(); diff --git a/src/cpu/x64/jit_uni_dw_conv_kernel_f32.hpp b/src/cpu/x64/jit_uni_dw_conv_kernel_f32.hpp index 1a152cec9e7..32609788e47 100644 --- a/src/cpu/x64/jit_uni_dw_conv_kernel_f32.hpp +++ b/src/cpu/x64/jit_uni_dw_conv_kernel_f32.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2022 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,7 +48,7 @@ struct jit_uni_dw_conv_fwd_kernel_f32 : public jit_generator { const Xbyak::AddressFrame &vmmword = (isa == sse41) ? xword : (isa == avx2) ? yword : zword; - const int vlen = cpu_isa_traits::vlen; + const int vlen = cpu_isa_traits_t::vlen; // dw convolution reg64_t reg_input = r8; @@ -137,7 +137,7 @@ struct jit_uni_dw_conv_bwd_data_kernel_f32 : public jit_generator { using Vmm = typename utils::conditional3::type; const int reg_repeats_ = (isa == sse41) ? 2 : 1; - const int simd_w_ = cpu_isa_traits::vlen / sizeof(float); + const int simd_w_ = cpu_isa_traits_t::vlen / sizeof(float); using reg64_t = const Xbyak::Reg64; inline Vmm get_ker_reg(int idx) { return Vmm(idx + 0); } @@ -203,7 +203,7 @@ struct jit_uni_dw_conv_bwd_weights_kernel_f32 : public jit_generator { using Vmm = typename utils::conditional3::type; - const int simd_w_ = cpu_isa_traits::vlen / sizeof(float); + const int simd_w_ = cpu_isa_traits_t::vlen / sizeof(float); const int reg_repeats_ = (isa == sse41) ? 2 : 1; const int req_aux_vmm = isa == sse41 ? 1 : 0; // used for FMA operand diff --git a/src/cpu/x64/jit_uni_eltwise.cpp b/src/cpu/x64/jit_uni_eltwise.cpp index 0e57fd1d016..1491c82f6af 100644 --- a/src/cpu/x64/jit_uni_eltwise.cpp +++ b/src/cpu/x64/jit_uni_eltwise.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2017-2024 Intel Corporation +* Copyright 2017-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -81,9 +81,9 @@ struct jit_uni_kernel_t : public jit_uni_eltwise_kernel { jit_uni_kernel_t(const eltwise_pd_t *pd) : jit_uni_eltwise_kernel(pd, jit_name(), isa) - , vlen_(is_bf16() || is_f16() ? cpu_isa_traits::vlen / 2 - : is_f8() ? cpu_isa_traits::vlen / 4 - : cpu_isa_traits::vlen) + , vlen_(is_bf16() || is_f16() ? cpu_isa_traits_t::vlen / 2 + : is_f8() ? cpu_isa_traits_t::vlen / 4 + : cpu_isa_traits_t::vlen) , simd_w_(vlen_ / dtype_size()) , is_fwd_(pd_->is_fwd()) { @@ -231,7 +231,7 @@ struct jit_uni_kernel_t : public jit_uni_eltwise_kernel { } private: - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; static constexpr cpu_isa_t injector_isa = isa == avx512_core_amx ? avx512_core : isa; diff --git a/src/cpu/x64/jit_uni_eltwise_int.cpp b/src/cpu/x64/jit_uni_eltwise_int.cpp index 5177d251954..b5eb5a6d4b6 100644 --- a/src/cpu/x64/jit_uni_eltwise_int.cpp +++ b/src/cpu/x64/jit_uni_eltwise_int.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -76,7 +76,7 @@ struct jit_uni_subkernel_int_t : public jit_uni_eltwise_int_kernel { void generate() override { Reg64 param = abi_param1; - const size_t vlen = cpu_isa_traits::vlen; + const size_t vlen = cpu_isa_traits_t::vlen; const size_t simd_w = vlen / sizeof(float); const size_t loop_dec[] = {simd_w, 1}; const size_t uf[] = {1, 1}; @@ -128,7 +128,7 @@ struct jit_uni_subkernel_int_t : public jit_uni_eltwise_int_kernel { } private: - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; using opmask_t = const Xbyak::Opmask; Reg64 reg_from = rax; diff --git a/src/cpu/x64/jit_uni_group_normalization.cpp b/src/cpu/x64/jit_uni_group_normalization.cpp index 9c6187f9172..37929887fc9 100644 --- a/src/cpu/x64/jit_uni_group_normalization.cpp +++ b/src/cpu/x64/jit_uni_group_normalization.cpp @@ -220,11 +220,11 @@ struct kernel_t : public jit_uni_group_normalization_fwd_t::kernel_base_t, } protected: - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; const Xbyak::AddressFrame &vmmword = (isa == sse41) ? xword : (isa == avx2) ? yword : zword; - const int vlen = cpu_isa_traits::vlen; + const int vlen = cpu_isa_traits_t::vlen; struct ker_args_t { const void *src; @@ -568,11 +568,11 @@ struct kernel_stat_t } protected: - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; const Xbyak::AddressFrame &vmmword = (isa == sse41) ? xword : (isa == avx2) ? yword : zword; - const int vlen = cpu_isa_traits::vlen; + const int vlen = cpu_isa_traits_t::vlen; struct ker_args_t { const void *src; diff --git a/src/cpu/x64/jit_uni_i8i8_pooling.cpp b/src/cpu/x64/jit_uni_i8i8_pooling.cpp index 5647066e22c..57302b9d517 100644 --- a/src/cpu/x64/jit_uni_i8i8_pooling.cpp +++ b/src/cpu/x64/jit_uni_i8i8_pooling.cpp @@ -69,7 +69,7 @@ template struct jit_uni_i8i8_pooling_fwd_ker_t : public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_i8i8_pooling_fwd_ker_t) - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; Xmm xreg(int idx) const { return Xmm(idx); } Ymm yreg(int idx) const { return Ymm(xreg(idx).getIdx()); } Vmm vreg(int idx) const { return Vmm(xreg(idx).getIdx()); } @@ -238,7 +238,7 @@ struct jit_uni_i8i8_pooling_fwd_ker_t : public jit_generator { if (jpp.with_postops) { - const int simd_w = cpu_isa_traits::vlen / sizeof(float); + const int simd_w = cpu_isa_traits_t::vlen / sizeof(float); const std::size_t c_tail_elems = jpp.c % simd_w; post_op_tail_opmask_idx_ = 0; if (c_tail_elems) { @@ -318,7 +318,7 @@ void jit_uni_i8i8_pooling_fwd_ker_t::load_src_max_op( // Example: idx=[31..0] // vreg_src = [x,x,x,x,.....,x,-,-,-,-,-] ; x => byte data // shift to transform vreg_src = [-,-,-,-,-,x,..,x,x,x,x,] - const uint8_t shift = cpu_isa_traits::vlen - jpp.c_tail; + const uint8_t shift = cpu_isa_traits_t::vlen - jpp.c_tail; if (jpp.safe_c_tail) { @@ -427,10 +427,10 @@ void jit_uni_i8i8_pooling_fwd_ker_t::load_src_avg_op( // vreg_src = [x,x,x,x,.....,x,-,-,-,-,-] ; x => byte data // shift to transform vreg_src = [-,-,-,-,-,x,..,x,x,x,x,] // Re-purposing vreg_zeros here. Set it back to zero immmediately. - const int msk_gran - = cpu_isa_traits::vlen / data_type_size(avg_proc_dt); + const int msk_gran = cpu_isa_traits_t::vlen + / data_type_size(avg_proc_dt); - const uint8_t shift = cpu_isa_traits::vlen + const uint8_t shift = cpu_isa_traits_t::vlen - (jpp.c_tail > (ll + 1) * msk_gran ? msk_gran : jpp.c_tail - (ll * msk_gran)); @@ -569,7 +569,7 @@ void jit_uni_i8i8_pooling_fwd_ker_t::store_dst_max_op( int c_block = jpp.c_block; const uint64_t low_mask = (1ULL << (c_block / 2)) - 1; - const uint8_t shift = cpu_isa_traits::vlen - jpp.c_tail; + const uint8_t shift = cpu_isa_traits_t::vlen - jpp.c_tail; if (masked) { switch (jpp.src_dt) { @@ -677,7 +677,7 @@ void jit_uni_i8i8_pooling_fwd_ker_t::store_dst_avg_op( const int copy_range = masked ? math::ilog2q(jpp.tail[ll] + 1) - : cpu_isa_traits::vlen / data_type_size(avg_proc_dt); + : cpu_isa_traits_t::vlen / data_type_size(avg_proc_dt); for (int i = 0; i < copy_range; i++) pextrb(ptr[reg_ptr_dst_i8 + offset + i], vr_dst, i); } else @@ -736,7 +736,7 @@ void jit_uni_i8i8_pooling_fwd_ker_t::store_dst_avg_op( // mmx_mask(ll) - ll-th mask of tail in non-zero-tail case const int msk_gran - = cpu_isa_traits::vlen / data_type_size(avg_proc_dt); + = cpu_isa_traits_t::vlen / data_type_size(avg_proc_dt); const int ll_end = (ll + 1) * msk_gran; // ((ll + 1) * 8) @@ -1070,7 +1070,7 @@ void jit_uni_i8i8_pooling_fwd_ker_t::init_mask() {} template <> void jit_uni_i8i8_pooling_fwd_ker_t::init_mask() { using namespace data_type; - using cpu_isa = cpu_isa_traits; + using cpu_isa = cpu_isa_traits_t; // AVX2 mask initialization: mask stored in Ymm-regs auto init = [&](uint64_t bit_mask, bool need_ymm_mask = true, @@ -1324,7 +1324,7 @@ status_t jit_uni_i8i8_pooling_fwd_ker_t::init_conf( // isa == sse41 : 16 bytes -> 16 for s8/u8, 4 for s32 // isa == avx2 : 32 bytes -> 32 for s8/u8, 8 for s32 // isa == avx512* : 64 bytes -> 64 for s8/u8, 16 for s32 - int simd_w = cpu_isa_traits::vlen / data_type_size(jpp.src_dt); + int simd_w = cpu_isa_traits_t::vlen / data_type_size(jpp.src_dt); /* Verify that vlen-sized memory access happens within the tensor's * size, otherwise load/store will always spill outside the memory @@ -1361,7 +1361,7 @@ status_t jit_uni_i8i8_pooling_fwd_ker_t::init_conf( // avg_proc_dt (s32) defines granularity (because u8/s8 processed as s32) // sse : 4, avx2 : 8, avx512 : 16 const size_t msk_gran - = cpu_isa_traits::vlen / data_type_size(avg_proc_dt); + = cpu_isa_traits_t::vlen / data_type_size(avg_proc_dt); const size_t msk_msk = (1ULL << msk_gran) - 1; size_t m = tail_mask; for (size_t ll = 0; ll < max_num_ll; ll++) { @@ -1452,11 +1452,11 @@ status_t jit_uni_i8i8_pooling_fwd_t::execute_forward( * boundary, if so, compute a safe memory access. */ const auto src_safe_access = reinterpret_cast( reinterpret_cast(src_i8 + src_d.size() - 1) - - (cpu_isa_traits::vlen - 1)); + - (cpu_isa_traits_t::vlen - 1)); const auto dst_safe_access = reinterpret_cast( reinterpret_cast(dst_i8 + dst_d.size() - 1) - - (cpu_isa_traits::vlen - 1)); + - (cpu_isa_traits_t::vlen - 1)); parallel_nd(jpp.mb, jpp.od, jpp.oh, jpp.ow, [&](dim_t n, dim_t od, dim_t oh, dim_t ow) { diff --git a/src/cpu/x64/jit_uni_instance_normalization.cpp b/src/cpu/x64/jit_uni_instance_normalization.cpp index 41f8e0b50bf..7b6274b0f29 100644 --- a/src/cpu/x64/jit_uni_instance_normalization.cpp +++ b/src/cpu/x64/jit_uni_instance_normalization.cpp @@ -210,11 +210,11 @@ struct kernel_t : public jit_uni_instance_normalization_fwd_t::kernel_base_t, } protected: - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; const Xbyak::AddressFrame &vmmword = (isa == sse41) ? xword : (isa == avx2) ? yword : zword; - const int vlen = cpu_isa_traits::vlen; + const int vlen = cpu_isa_traits_t::vlen; struct ker_args_t { const void *src; @@ -477,11 +477,11 @@ struct kernel_stat_t } protected: - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; const Xbyak::AddressFrame &vmmword = (isa == sse41) ? xword : (isa == avx2) ? yword : zword; - const int vlen = cpu_isa_traits::vlen; + const int vlen = cpu_isa_traits_t::vlen; struct ker_args_t { const void *src; diff --git a/src/cpu/x64/jit_uni_layer_normalization.cpp b/src/cpu/x64/jit_uni_layer_normalization.cpp index 058c6f338c9..ab1a28f20fb 100644 --- a/src/cpu/x64/jit_uni_layer_normalization.cpp +++ b/src/cpu/x64/jit_uni_layer_normalization.cpp @@ -150,11 +150,11 @@ struct jit_stat_and_data_base_kernel_t : stat_and_data_kernel_t, protected: static constexpr int unroll_factor_ = 4; - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; const AddressFrame &vmmword = (isa == sse41) ? xword : (isa == avx2) ? yword : zword; - const int vlen = cpu_isa_traits::vlen; + const int vlen = cpu_isa_traits_t::vlen; struct ker_args_t { const void *src; @@ -764,11 +764,11 @@ struct jit_diff_ss_kernel_t : diff_ss_kernel_t, public jit_generator { } protected: - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; const AddressFrame &vmmword = (isa == sse41) ? xword : (isa == avx2) ? yword : zword; - const int vlen = cpu_isa_traits::vlen; + const int vlen = cpu_isa_traits_t::vlen; struct ker_args_t { const void *src; @@ -962,11 +962,11 @@ struct jit_diff_data_base_kernel_t : diff_data_kernel_t, public jit_generator { protected: static constexpr int unroll_factor_ = 4; - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; const AddressFrame &vmmword = (isa == sse41) ? xword : (isa == avx2) ? yword : zword; - const int vlen = cpu_isa_traits::vlen; + const int vlen = cpu_isa_traits_t::vlen; struct ker_args_t { const void *src; diff --git a/src/cpu/x64/jit_uni_pool_kernel.cpp b/src/cpu/x64/jit_uni_pool_kernel.cpp index e7e1d50d43b..e762b481cca 100644 --- a/src/cpu/x64/jit_uni_pool_kernel.cpp +++ b/src/cpu/x64/jit_uni_pool_kernel.cpp @@ -35,7 +35,7 @@ using namespace alg_kind; #define GET_OFF(field) offsetof(jit_pool_call_s, field) constexpr int sse41_single_block_size - = cpu_isa_traits::vlen / sizeof(float); + = cpu_isa_traits_t::vlen / sizeof(float); static bcast_set_t get_supported_bcast_strategies() { return {broadcasting_strategy_t::scalar, broadcasting_strategy_t::per_oc, @@ -1303,7 +1303,7 @@ void jit_uni_pool_kernel::zero_diff_src( mov(aux_reg_zero_ih, reg_zero_ih); L(l_ih_loop); { - const auto vlen = cpu_isa_traits::vlen; + const auto vlen = cpu_isa_traits_t::vlen; const int step = c_off * dt_size; // TODO: maybe a big code generated here @@ -1362,7 +1362,7 @@ void jit_uni_pool_kernel::generate() { ? jpp.f32_accum_block_size : output_c_off; - int vlen = cpu_isa_traits::vlen; + int vlen = cpu_isa_traits_t::vlen; const size_t input_dt_size = jpp.needs_f32_accum_for_bf16 ? sizeof(float) : jpp.dt_size; diff --git a/src/cpu/x64/jit_uni_pool_kernel.hpp b/src/cpu/x64/jit_uni_pool_kernel.hpp index bf861af4c6c..56aa1f664af 100644 --- a/src/cpu/x64/jit_uni_pool_kernel.hpp +++ b/src/cpu/x64/jit_uni_pool_kernel.hpp @@ -59,7 +59,7 @@ struct jit_uni_pool_kernel : public jit_generator { using Reg32 = Xbyak::Reg32; using Reg64 = Xbyak::Reg64; - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; int vmm_idx_upper_bound() const noexcept { return is_superset(isa, avx512_core) ? 31 : 15; diff --git a/src/cpu/x64/jit_uni_reduction_kernel.hpp b/src/cpu/x64/jit_uni_reduction_kernel.hpp index c844e5d7935..ac0d320cdb9 100644 --- a/src/cpu/x64/jit_uni_reduction_kernel.hpp +++ b/src/cpu/x64/jit_uni_reduction_kernel.hpp @@ -49,7 +49,7 @@ struct jit_uni_reduction_kernel_base_t : public jit_generator { std::queue sum_scales_; }; -template ::Vmm> +template ::Vmm> struct jit_uni_reduction_kernel_t : public jit_uni_reduction_kernel_base_t { jit_uni_reduction_kernel_t( const jit_reduction_conf_t &conf, const memory_desc_t *dst_md); diff --git a/src/cpu/x64/jit_uni_resampling_kernel.hpp b/src/cpu/x64/jit_uni_resampling_kernel.hpp index d1b807e1665..23dd2134e63 100644 --- a/src/cpu/x64/jit_uni_resampling_kernel.hpp +++ b/src/cpu/x64/jit_uni_resampling_kernel.hpp @@ -70,7 +70,7 @@ struct jit_uni_resampling_kernel_t : public jit_uni_resampling_kernel_base_t { using c_oriented_generation_fn_t = std::function; constexpr int vmm_idx(int idx) const { - return (cpu_isa_traits::n_vregs - 1) - idx; + return (cpu_isa_traits_t::n_vregs - 1) - idx; } bool can_movntps_be_used() const; diff --git a/src/cpu/x64/jit_uni_softmax.cpp b/src/cpu/x64/jit_uni_softmax.cpp index d121ed36477..eb9e2e71f1d 100644 --- a/src/cpu/x64/jit_uni_softmax.cpp +++ b/src/cpu/x64/jit_uni_softmax.cpp @@ -54,12 +54,12 @@ struct jit_softmax_dense_kernel_t : jit_softmax_kernel_base_t, public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_softmax_dense_kernel_t) - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; const AddressFrame &vmmword = is_superset(isa, avx512_core) ? zword : is_superset(isa, avx) ? yword : xword; - static constexpr auto vlen = cpu_isa_traits::vlen; - static constexpr auto n_vregs = cpu_isa_traits::n_vregs; + static constexpr auto vlen = cpu_isa_traits_t::vlen; + static constexpr auto n_vregs = cpu_isa_traits_t::n_vregs; static constexpr auto simd_w_ = vlen / sizeof(float); // bf16 works on ymms const memory_desc_wrapper src_d_, dst_d_, diff_dst_d_; @@ -1000,11 +1000,11 @@ struct jit_softmax_strided_kernel_t : jit_softmax_kernel_base_t, public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_softmax_strided_kernel_t) - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; const AddressFrame &vmmword = is_superset(isa, avx512_core) ? zword : is_superset(isa, avx) ? yword : xword; - static constexpr auto vlen = cpu_isa_traits::vlen; + static constexpr auto vlen = cpu_isa_traits_t::vlen; static constexpr auto simd_w_ = vlen / sizeof(float); // bf16 works on ymms const memory_desc_wrapper src_d_, dst_d_; diff --git a/src/cpu/x64/jit_uni_tbb_batch_normalization.cpp b/src/cpu/x64/jit_uni_tbb_batch_normalization.cpp index bfee901b372..b5190f46484 100644 --- a/src/cpu/x64/jit_uni_tbb_batch_normalization.cpp +++ b/src/cpu/x64/jit_uni_tbb_batch_normalization.cpp @@ -56,7 +56,7 @@ template int get_vlen(jit_memory_tag_kind_t tag_kind) { return isa == sse41 && tag_kind == jit_memory_tag_kind_t::blocked ? 32 - : cpu_isa_traits::vlen; + : cpu_isa_traits_t::vlen; } template @@ -95,7 +95,7 @@ std::tuple get_data_strides( #define PARAM_ADDR(x) (reg_param_ + offsetof(call_params_t, x)) template struct jit_bnorm_process_tail_t { - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; jit_bnorm_process_tail_t(const batch_normalization_pd_t *pd, jit_generator *host, Reg64 reg_tmp, Reg64 reg_blk_has_tail, @@ -109,7 +109,7 @@ struct jit_bnorm_process_tail_t { const memory_desc_wrapper data_d(pd->src_md()); c_is_padded_ = pd->C() != data_d.padded_dims()[1]; - const int vlen = isa == sse41 ? 32 : cpu_isa_traits::vlen; + const int vlen = isa == sse41 ? 32 : cpu_isa_traits_t::vlen; tail_ = pd->C() % (int)(vlen / sizeof(float)); } @@ -198,7 +198,7 @@ struct jit_bnorm_process_tail_t { template struct jit_bnorm_process_relu_t { - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; jit_bnorm_process_relu_t(const batch_normalization_pd_t *pd, jit_generator *host, Reg64 reg_off_dat, Reg64 reg_tmp, @@ -369,7 +369,7 @@ struct jit_bnorm_process_relu_t { template struct helper_vmovups_data_t { - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; helper_vmovups_data_t(const batch_normalization_pd_t *pd, jit_generator *host, Zmm zmm_reserved_1, Zmm zmm_reserved_2, @@ -464,7 +464,7 @@ struct helper_vmovups_data_t { template struct jit_bnorm_fwd_statistics_t : public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_bnorm_fwd_statistics_t) - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; const AddressFrame &vmmword = (isa == sse41) ? xword : (isa == avx2) ? yword @@ -855,7 +855,7 @@ struct jit_bnorm_fwd_var_t : jit_bnorm_fwd_statistics_t { template struct jit_bnorm_fwd_t : public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_bnorm_fwd_t) - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; const AddressFrame &vmmword = (isa == sse41) ? xword : (isa == avx2) ? yword @@ -1230,7 +1230,7 @@ struct jit_bnorm_fwd_t : public jit_generator { template struct jit_bnorm_bwd_t : public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_bnorm_bwd_t) - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; const AddressFrame &vmmword = (isa == sse41) ? xword : (isa == avx2) ? yword @@ -1516,7 +1516,7 @@ struct jit_bnorm_bwd_t : public jit_generator { template struct jit_bnorm_bwd_diff_ss_t : public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_bnorm_bwd_diff_ss_t) - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; const AddressFrame &vmmword = (isa == sse41) ? xword : (isa == avx2) ? yword diff --git a/src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.hpp b/src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.hpp index 64f96c2a38d..9a7b9a0b85f 100644 --- a/src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.hpp +++ b/src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2023 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -42,7 +42,7 @@ struct _jit_uni_x8s8s32x_fwd_kernel : public jit_generator { private: constexpr static int isa_simd_width_ - = cpu_isa_traits::vlen / sizeof(float); + = cpu_isa_traits_t::vlen / sizeof(float); std::unique_ptr> postops_injector_; enum { diff --git a/src/cpu/x64/jit_uni_xf16_sum.cpp b/src/cpu/x64/jit_uni_xf16_sum.cpp index c7e6c02e9ad..cabc102e131 100644 --- a/src/cpu/x64/jit_uni_xf16_sum.cpp +++ b/src/cpu/x64/jit_uni_xf16_sum.cpp @@ -191,7 +191,7 @@ status_t jit_avx512_core_bf16_sum_kernel_t::init_conf( for (/*continue*/; jsp.loop_unroll < max_unroll; jsp.loop_unroll++) { const int num_regs = num_vregs_required(jsp.loop_unroll + 1, jsp.num_srcs); - if (num_regs > (cpu_isa_traits::n_vregs + if (num_regs > (cpu_isa_traits_t::n_vregs - (isa_has_bf16(jsp.isa) ? 1 : 6))) break; } diff --git a/src/cpu/x64/jit_uni_xf16_sum.hpp b/src/cpu/x64/jit_uni_xf16_sum.hpp index a45c4c1ea18..7fa54259e9b 100644 --- a/src/cpu/x64/jit_uni_xf16_sum.hpp +++ b/src/cpu/x64/jit_uni_xf16_sum.hpp @@ -103,7 +103,7 @@ struct jit_avx512_core_bf16_sum_kernel_t jit_avx512_core_bf16_sum_kernel_t(jit_sum_conf_t ajsp) : jit_uni_xf16_sum_kernel_t( ajsp, utils::div_up(ajsp.num_srcs, 2)) - , max_vregs_available(cpu_isa_traits::n_vregs + , max_vregs_available(cpu_isa_traits_t::n_vregs - (isa_has_bf16(jsp.isa) ? 1 : 6)) , bf16_emu_(nullptr) { if (!mayiuse(avx512_core_bf16)) diff --git a/src/cpu/x64/lrn/jit_uni_lrn_kernel.cpp b/src/cpu/x64/lrn/jit_uni_lrn_kernel.cpp index b3f9fefb9b2..1ea0628e09f 100644 --- a/src/cpu/x64/lrn/jit_uni_lrn_kernel.cpp +++ b/src/cpu/x64/lrn/jit_uni_lrn_kernel.cpp @@ -1308,7 +1308,7 @@ void jit_uni_lrn_fwd_kernel_t::generate( const Xbyak::Xmm &xe_lo = this->xmm14; const Xbyak::Xmm &xe_hi = this->xmm15; - const int vlen = cpu_isa_traits::vlen / sizeof(float); + const int vlen = cpu_isa_traits_t::vlen / sizeof(float); bool compute_tail = J.tail != 0; bool load_lo = J.tail == 0 || J.tail > 4; diff --git a/src/cpu/x64/lrn/jit_uni_lrn_kernel.hpp b/src/cpu/x64/lrn/jit_uni_lrn_kernel.hpp index e445518f636..830beab654e 100644 --- a/src/cpu/x64/lrn/jit_uni_lrn_kernel.hpp +++ b/src/cpu/x64/lrn/jit_uni_lrn_kernel.hpp @@ -95,11 +95,11 @@ class jit_uni_lrn_kernel_t> : public jit_generator { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_lrn_kernel_t); // TODO: why use double simd for sse41? static constexpr int VECTOR_LENGTH - = (cpu_isa_traits<(isa > sse41 ? isa : avx2)>::vlen + = (cpu_isa_traits_t<(isa > sse41 ? isa : avx2)>::vlen / sizeof(float)); protected: - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; void load_constant(float constant, const Vmm &v_constant, const Xbyak::Xmm &x_constant); @@ -119,7 +119,8 @@ class jit_uni_lrn_kernel_t> : public jit_generator { const Xbyak::Reg64 w_ = this->r10; const Xbyak::Reg64 imm_addr64_ = this->rbx; const Xbyak::Reg64 reg_tmp_ = this->rsi; - static constexpr size_t simd_w_ = cpu_isa_traits::vlen / sizeof(float); + static constexpr size_t simd_w_ + = cpu_isa_traits_t::vlen / sizeof(float); int single_pixel_offset_ = VECTOR_LENGTH * sizeof(typename prec_traits_t::type); diff --git a/src/cpu/x64/matmul/brgemm_matmul.hpp b/src/cpu/x64/matmul/brgemm_matmul.hpp index c8f7dc4baa1..ee5ecf600c4 100644 --- a/src/cpu/x64/matmul/brgemm_matmul.hpp +++ b/src/cpu/x64/matmul/brgemm_matmul.hpp @@ -166,7 +166,7 @@ struct brgemm_matmul_t : public primitive_t { std::unique_ptr jit_scale_precompute_; using reducer_t = x64::jit_brgemm_kernel_diff_bias_t< - typename cpu_isa_traits::Vmm>; + typename cpu_isa_traits_t::Vmm>; std::unique_ptr reducers_[2][2]; }; diff --git a/src/cpu/x64/matmul/brgemm_matmul_copy_utils.cpp b/src/cpu/x64/matmul/brgemm_matmul_copy_utils.cpp index 43ed3b8d858..f5f3c46925b 100644 --- a/src/cpu/x64/matmul/brgemm_matmul_copy_utils.cpp +++ b/src/cpu/x64/matmul/brgemm_matmul_copy_utils.cpp @@ -3681,7 +3681,7 @@ struct jit_brgemm_matmul_copy_b_transposed_t static constexpr bool is_ymm_ = std::is_same::value; static constexpr cpu_isa_t isa_ = is_ymm_ ? avx2 : avx512_core; - static constexpr int max_vmm_regs_ = cpu_isa_traits::n_vregs; + static constexpr int max_vmm_regs_ = cpu_isa_traits_t::n_vregs; static constexpr int vlen_ = vreg_traits_t::vlen; static constexpr int n_blk_step_ = is_ymm_ ? 8 : 16; static constexpr int req_cvt_bf16_k_blk_step_ = 16; diff --git a/src/cpu/x64/matmul/jit_uni_sparse_matmul.cpp b/src/cpu/x64/matmul/jit_uni_sparse_matmul.cpp index d9777435206..b58d87a22c4 100644 --- a/src/cpu/x64/matmul/jit_uni_sparse_matmul.cpp +++ b/src/cpu/x64/matmul/jit_uni_sparse_matmul.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2023-2024 Intel Corporation +* Copyright 2023-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -92,7 +92,7 @@ struct jit_uni_sparse_matmul_kernel_t : public sparse_matmul_kernel_t { using sparse_matmul_kernel_t::tail_size; using sparse_matmul_kernel_t::vlen; - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; Reg64 reg_param = abi_param1; @@ -300,7 +300,7 @@ struct jit_uni_sparse_matmul_kernel_t : public sparse_matmul_kernel_t { } jit_uni_sparse_matmul_kernel_t(const matmul_pd_t *pd) - : sparse_matmul_kernel_t(cpu_isa_traits::vlen, pd) {} + : sparse_matmul_kernel_t(cpu_isa_traits_t::vlen, pd) {} ~jit_uni_sparse_matmul_kernel_t() override = default; }; diff --git a/src/cpu/x64/prelu/jit_prelu_utils.cpp b/src/cpu/x64/prelu/jit_prelu_utils.cpp index d98a3b69274..e7d6720f44f 100644 --- a/src/cpu/x64/prelu/jit_prelu_utils.cpp +++ b/src/cpu/x64/prelu/jit_prelu_utils.cpp @@ -47,34 +47,34 @@ cpu_isa_t get_supported_isa() { static int get_vlen(const cpu_isa_t &isa) noexcept { if (isa == avx512_core_fp16) - return cpu_isa_traits::vlen; + return cpu_isa_traits_t::vlen; else if (isa == avx512_core_bf16) - return cpu_isa_traits::vlen; + return cpu_isa_traits_t::vlen; else if (isa == avx512_core) - return cpu_isa_traits::vlen; + return cpu_isa_traits_t::vlen; else if (isa == avx2_vnni_2) - return cpu_isa_traits::vlen; + return cpu_isa_traits_t::vlen; else if (isa == avx2) - return cpu_isa_traits::vlen; + return cpu_isa_traits_t::vlen; else if (isa == avx) - return cpu_isa_traits::vlen; - return cpu_isa_traits::vlen; + return cpu_isa_traits_t::vlen; + return cpu_isa_traits_t::vlen; } int get_n_vregs(const cpu_isa_t &isa) noexcept { if (isa == avx512_core_fp16) - return cpu_isa_traits::n_vregs; + return cpu_isa_traits_t::n_vregs; else if (isa == avx512_core_bf16) - return cpu_isa_traits::n_vregs; + return cpu_isa_traits_t::n_vregs; else if (isa == avx512_core) - return cpu_isa_traits::n_vregs; + return cpu_isa_traits_t::n_vregs; else if (isa == avx2_vnni_2) - return cpu_isa_traits::n_vregs; + return cpu_isa_traits_t::n_vregs; else if (isa == avx2) - return cpu_isa_traits::n_vregs; + return cpu_isa_traits_t::n_vregs; else if (isa == avx) - return cpu_isa_traits::n_vregs; - return cpu_isa_traits::n_vregs; + return cpu_isa_traits_t::n_vregs; + return cpu_isa_traits_t::n_vregs; } bool is_s8u8(const std::set &tensor_data_types) noexcept { diff --git a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_bwd.hpp b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_bwd.hpp index bc783e89c5e..3428319ad68 100644 --- a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_bwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_bwd.hpp @@ -42,8 +42,8 @@ struct jit_uni_gru_cell_postgemm_part1_bwd : public jit_uni_rnn_postgemm { protected: // register size in bytes - using Vmm = typename cpu_isa_traits::Vmm; - static constexpr size_t vlen = cpu_isa_traits::vlen; + using Vmm = typename cpu_isa_traits_t::Vmm; + static constexpr size_t vlen = cpu_isa_traits_t::vlen; const size_t vlen_scratch = vlen / (sizeof(float) / types::data_type_size(scratch_data_t)); static constexpr size_t hstate_dt_size = sizeof(float); @@ -197,14 +197,14 @@ struct jit_uni_gru_cell_postgemm_part1_bwd : public jit_uni_rnn_postgemm { // Reduce diff attention into XMM size. Otherwise accumulation // using XMM will zero high part of YMM/ZMM. - if (vlen >= cpu_isa_traits::vlen) { + if (vlen >= cpu_isa_traits_t::vlen) { Zmm diff_attn_acc(dattn_acc_idx); Ymm diff_attn_acc_high(tmp1_idx); Ymm diff_attn_acc_low(dattn_acc_idx); vextractf32x8(diff_attn_acc_high, diff_attn_acc, 1); vaddps(diff_attn_acc_low, diff_attn_acc_low, diff_attn_acc_high); } - if (vlen >= cpu_isa_traits::vlen) { + if (vlen >= cpu_isa_traits_t::vlen) { Ymm diff_attn_acc(dattn_acc_idx); Xmm diff_attn_acc_high(tmp1_idx); Xmm diff_attn_acc_low(dattn_acc_idx); diff --git a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_fwd.hpp b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_fwd.hpp index 3f339252946..2fa3920554f 100644 --- a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_fwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_fwd.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -56,7 +56,7 @@ struct jit_uni_gru_cell_postgemm_part1_fwd : public jit_uni_rnn_postgemm { // register size in bytes using Vmm = typename jit_uni_eltwise_injector::Vmm; - static constexpr size_t vlen = cpu_isa_traits::vlen; + static constexpr size_t vlen = cpu_isa_traits_t::vlen; static constexpr size_t qscale_dt_size = sizeof(float); const size_t vlen_dst = vlen / (sizeof(float) / types::data_type_size(src_data_t)); diff --git a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_bwd.hpp b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_bwd.hpp index dd89b5deab0..761c59db557 100644 --- a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_bwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_bwd.hpp @@ -42,8 +42,8 @@ struct jit_uni_gru_cell_postgemm_part2_bwd : public jit_uni_rnn_postgemm { protected: // register size in bytes - using Vmm = typename cpu_isa_traits::Vmm; - static constexpr size_t vlen = cpu_isa_traits::vlen; + using Vmm = typename cpu_isa_traits_t::Vmm; + static constexpr size_t vlen = cpu_isa_traits_t::vlen; static constexpr size_t hstate_dt_size = sizeof(float); const size_t vlen_scratch = vlen / (sizeof(float) / types::data_type_size(scratch_data_t)); diff --git a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_fwd.hpp b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_fwd.hpp index 49814da34eb..e314b5a6617 100644 --- a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_fwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_fwd.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -56,7 +56,7 @@ struct jit_uni_gru_cell_postgemm_part2_fwd : public jit_uni_rnn_postgemm { // register size in bytes using Vmm = typename jit_uni_eltwise_injector::Vmm; - static constexpr size_t vlen = cpu_isa_traits::vlen; + static constexpr size_t vlen = cpu_isa_traits_t::vlen; static constexpr size_t qscale_dt_size = sizeof(float); const size_t vlen_dst = vlen / (sizeof(float) / types::data_type_size(src_data_t)); diff --git a/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_bwd.hpp b/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_bwd.hpp index 9af12243f26..8d794fa046f 100644 --- a/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_bwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_bwd.hpp @@ -42,8 +42,8 @@ struct jit_uni_gru_lbr_cell_postgemm_bwd : public jit_uni_rnn_postgemm { protected: // register size in bytes - using Vmm = typename cpu_isa_traits::Vmm; - static constexpr size_t vlen = cpu_isa_traits::vlen; + using Vmm = typename cpu_isa_traits_t::Vmm; + static constexpr size_t vlen = cpu_isa_traits_t::vlen; static constexpr size_t hstate_dt_size = sizeof(float); const size_t vlen_scratch = vlen / (sizeof(float) / types::data_type_size(scratch_data_t)); @@ -236,14 +236,14 @@ struct jit_uni_gru_lbr_cell_postgemm_bwd : public jit_uni_rnn_postgemm { // Reduce diff attention into XMM size. Otherwise accumulation // using XMM will zero high part of YMM/ZMM. - if (vlen >= cpu_isa_traits::vlen) { + if (vlen >= cpu_isa_traits_t::vlen) { Zmm diff_attn_acc(dattn_acc_idx); Ymm diff_attn_acc_high(tmp1_idx); Ymm diff_attn_acc_low(dattn_acc_idx); vextractf32x8(diff_attn_acc_high, diff_attn_acc, 1); vaddps(diff_attn_acc_low, diff_attn_acc_low, diff_attn_acc_high); } - if (vlen >= cpu_isa_traits::vlen) { + if (vlen >= cpu_isa_traits_t::vlen) { Ymm diff_attn_acc(dattn_acc_idx); Xmm diff_attn_acc_high(tmp1_idx); Xmm diff_attn_acc_low(dattn_acc_idx); diff --git a/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_fwd.hpp b/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_fwd.hpp index 2215c4abbcd..87c5e0d13a4 100644 --- a/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_fwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_fwd.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -57,7 +57,7 @@ struct jit_uni_gru_lbr_cell_postgemm_fwd : public jit_uni_rnn_postgemm { // register size in bytes using Vmm = typename jit_uni_eltwise_injector::Vmm; - static constexpr size_t vlen = cpu_isa_traits::vlen; + static constexpr size_t vlen = cpu_isa_traits_t::vlen; const size_t vlen_dst = vlen / (sizeof(float) / types::data_type_size(src_data_t)); diff --git a/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm.hpp b/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm.hpp index 8c6a0bd6331..18cf5b44b46 100644 --- a/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm.hpp +++ b/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2021-2024 Intel Corporation +* Copyright 2021-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,15 +31,15 @@ struct jit_uni_lstm_cell_postgemm_t { jit_generator *host, int tmp_id_begin, bool use_bf16_emu) : host_(host) , min_allowed_tmp_vmm_idx_(0) - , max_allowed_tmp_vmm_idx_(cpu_isa_traits::n_vregs - 1 + , max_allowed_tmp_vmm_idx_(cpu_isa_traits_t::n_vregs - 1 - (is_superset(isa, avx512_core) && use_bf16_emu ? 4 : 0)) { reset_tmp_vmm_idx_range(tmp_id_begin, max_allowed_tmp_vmm_idx_); } protected: using injector_t = jit_uni_eltwise_injector; - using Vmm = typename cpu_isa_traits::Vmm; - const size_t vlen_ = cpu_isa_traits::vlen; + using Vmm = typename cpu_isa_traits_t::Vmm; + const size_t vlen_ = cpu_isa_traits_t::vlen; Vmm get_next_tmp_vmm() { const Vmm vmm {current_tmp_id_++}; diff --git a/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_bwd.hpp b/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_bwd.hpp index 6d7590a0c0a..d242df98187 100644 --- a/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_bwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_bwd.hpp @@ -64,7 +64,7 @@ struct jit_uni_lstm_cell_postgemm_bwd std::unique_ptr tanh_injector_; // register size in bytes - static constexpr size_t vlen_ = cpu_isa_traits::vlen; + static constexpr size_t vlen_ = cpu_isa_traits_t::vlen; const size_t vlen_c_states_ = vlen_ / (sizeof(float) / cstate_dt_size_); static constexpr size_t diff_cstate_dt_size_ = sizeof(float); diff --git a/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_fwd.hpp b/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_fwd.hpp index b61dc40bc4b..016eed84f66 100644 --- a/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_fwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm_fwd.hpp @@ -69,7 +69,7 @@ struct jit_uni_lstm_cell_postgemm_fwd std::unique_ptr tanh_injector_; // register size in bytes - static constexpr size_t vlen_ = cpu_isa_traits::vlen; + static constexpr size_t vlen_ = cpu_isa_traits_t::vlen; static constexpr size_t qscale_dt_size = sizeof(float); static constexpr size_t weights_peephole_dt_size_ = sizeof(float); const size_t vlen_dst_ diff --git a/src/cpu/x64/rnn/jit_uni_lstm_cell_projection_postgemm_fwd.hpp b/src/cpu/x64/rnn/jit_uni_lstm_cell_projection_postgemm_fwd.hpp index fa03c716d34..dcd890c0bed 100644 --- a/src/cpu/x64/rnn/jit_uni_lstm_cell_projection_postgemm_fwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_lstm_cell_projection_postgemm_fwd.hpp @@ -44,7 +44,7 @@ struct jit_uni_lstm_cell_projection_postgemm_fwd : public jit_uni_rnn_postgemm { protected: // register size in bytes using Vmm = typename jit_uni_eltwise_injector::Vmm; - static constexpr size_t vlen = cpu_isa_traits::vlen; + static constexpr size_t vlen = cpu_isa_traits_t::vlen; static constexpr size_t qscale_dt_size = sizeof(float); const size_t vlen_dst = vlen / (sizeof(float) / types::data_type_size(src_data_t)); diff --git a/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_bwd.hpp b/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_bwd.hpp index 73838aa8874..bf3fc2a24cc 100644 --- a/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_bwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_bwd.hpp @@ -42,8 +42,8 @@ struct jit_uni_rnn_cell_postgemm_bwd : public jit_uni_rnn_postgemm { protected: // register size in bytes - using Vmm = typename cpu_isa_traits::Vmm; - static constexpr size_t vlen = cpu_isa_traits::vlen; + using Vmm = typename cpu_isa_traits_t::Vmm; + static constexpr size_t vlen = cpu_isa_traits_t::vlen; static constexpr size_t hstate_dt_size = sizeof(float); const size_t vlen_scratch = vlen / (sizeof(float) / types::data_type_size(scratch_data_t)); diff --git a/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_fwd.hpp b/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_fwd.hpp index 6cb72cd75df..9e4b9a37c1d 100644 --- a/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_fwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_fwd.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -52,7 +52,7 @@ struct jit_uni_rnn_cell_postgemm_fwd : public jit_uni_rnn_postgemm { // register size in bytes using Vmm = typename jit_uni_eltwise_injector::Vmm; - static constexpr size_t vlen = cpu_isa_traits::vlen; + static constexpr size_t vlen = cpu_isa_traits_t::vlen; static constexpr size_t cstate_dt_size = sizeof(float); static constexpr size_t qscale_dt_size = sizeof(float); diff --git a/src/cpu/x64/rnn/jit_uni_rnn_common_postgemm.hpp b/src/cpu/x64/rnn/jit_uni_rnn_common_postgemm.hpp index 0df5757cd70..85a10cbfae0 100644 --- a/src/cpu/x64/rnn/jit_uni_rnn_common_postgemm.hpp +++ b/src/cpu/x64/rnn/jit_uni_rnn_common_postgemm.hpp @@ -431,7 +431,7 @@ struct jit_uni_rnn_postgemm : public jit_generator { dshift_off_addr = ptr[qtable + 3 * vlen]; ymm_perm_mask_addr = ptr[qtable + 4 * vlen]; zmm_perm_mask_addr - = ptr[qtable + 4 * vlen + cpu_isa_traits::vlen]; + = ptr[qtable + 4 * vlen + cpu_isa_traits_t::vlen]; break; } case data_type::f32: { diff --git a/src/cpu/x64/shuffle/jit_uni_shuffle.cpp b/src/cpu/x64/shuffle/jit_uni_shuffle.cpp index 6b2e2053734..0f5dd99b63f 100644 --- a/src/cpu/x64/shuffle/jit_uni_shuffle.cpp +++ b/src/cpu/x64/shuffle/jit_uni_shuffle.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -79,7 +79,7 @@ status_t jit_uni_shuffle_t::pd_t::init(engine_t *engine) { blocked_format != format_tag::undef, VERBOSE_UNSUPPORTED_TAG); conf_.blk_size = src_d.blocking_desc().strides[ndims() - 1]; - conf_.simd_w = cpu_isa_traits::vlen / sizeof(float); + conf_.simd_w = cpu_isa_traits_t::vlen / sizeof(float); const bool has_spatial = utils::one_of(ndims(), 3, 4, 5); const dim_t HW = H() * W(); diff --git a/src/cpu/x64/shuffle/jit_uni_shuffle_kernel.hpp b/src/cpu/x64/shuffle/jit_uni_shuffle_kernel.hpp index ea7126ca5b7..852403a2938 100644 --- a/src/cpu/x64/shuffle/jit_uni_shuffle_kernel.hpp +++ b/src/cpu/x64/shuffle/jit_uni_shuffle_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2021-2024 Intel Corporation +* Copyright 2021-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,10 +41,10 @@ struct jit_uni_shuffle_kernel_t : public jit_generator { jit_uni_shuffle_kernel_t(const jit_shuffle_conf_t &conf); - using Vmm = typename cpu_isa_traits::Vmm; + using Vmm = typename cpu_isa_traits_t::Vmm; constexpr int vmm_idx(int idx) const { - return (cpu_isa_traits::n_vregs - 1) - idx; + return (cpu_isa_traits_t::n_vregs - 1) - idx; } /* From 2450c044828833f3807d0b646961ae23baa4f4a3 Mon Sep 17 00:00:00 2001 From: Dmitrii Zarukin Date: Mon, 17 Mar 2025 14:17:40 -0700 Subject: [PATCH 08/16] cpu: x64: eltwise_injector -> eltwise_injector_t --- .../injectors/jit_uni_eltwise_injector.cpp | 150 +++++++++--------- .../injectors/jit_uni_eltwise_injector.hpp | 8 +- .../injectors/jit_uni_postops_injector.cpp | 4 +- .../injectors/jit_uni_postops_injector.hpp | 3 +- .../x64/jit_avx512_core_amx_conv_kernel.hpp | 7 +- src/cpu/x64/jit_uni_eltwise.cpp | 6 +- src/cpu/x64/jit_uni_softmax.cpp | 18 +-- .../rnn/jit_uni_gru_cell_postgemm_1_fwd.hpp | 6 +- .../rnn/jit_uni_gru_cell_postgemm_2_fwd.hpp | 6 +- .../rnn/jit_uni_gru_lbr_cell_postgemm_fwd.hpp | 6 +- .../x64/rnn/jit_uni_lstm_cell_postgemm.hpp | 2 +- ..._uni_lstm_cell_projection_postgemm_fwd.hpp | 2 +- .../x64/rnn/jit_uni_rnn_cell_postgemm_fwd.hpp | 6 +- 13 files changed, 112 insertions(+), 112 deletions(-) diff --git a/src/cpu/x64/injectors/jit_uni_eltwise_injector.cpp b/src/cpu/x64/injectors/jit_uni_eltwise_injector.cpp index 93792edcafa..0717ead1aa0 100644 --- a/src/cpu/x64/injectors/jit_uni_eltwise_injector.cpp +++ b/src/cpu/x64/injectors/jit_uni_eltwise_injector.cpp @@ -64,14 +64,14 @@ bool is_supported(cpu_isa_t isa, alg_kind_t alg, data_type_t dt) { using namespace Xbyak; template -size_t jit_uni_eltwise_injector::get_stack_vmm_space() { +size_t jit_uni_eltwise_injector_t::get_stack_vmm_space() { return (save_state_ * preserve_vmm_ * n_vregs_to_preserve_ + op_vecs_count(alg_, is_fwd_)) * vlen_; } template -void jit_uni_eltwise_injector::injector_preamble( +void jit_uni_eltwise_injector_t::injector_preamble( const injector_utils::vmm_index_set_t &vmm_compute_idxs, injector_utils::vmm_index_set_iterator_t &start_idx_tail_it, const injector_utils::vmm_index_set_t &vmm_aux_indices) { @@ -223,7 +223,7 @@ void jit_uni_eltwise_injector::injector_preamble( } template -void jit_uni_eltwise_injector::injector_preamble_tail( +void jit_uni_eltwise_injector_t::injector_preamble_tail( size_t n_vregs_not_preserved) { // There was enough vmm registers to compute everything in one round. if (n_vregs_not_preserved == 0) return; @@ -262,7 +262,7 @@ void jit_uni_eltwise_injector::injector_preamble_tail( } template -void jit_uni_eltwise_injector::injector_postamble() { +void jit_uni_eltwise_injector_t::injector_postamble() { using namespace Xbyak::util; const int stack_vmm_space = get_stack_vmm_space(); @@ -296,7 +296,7 @@ void jit_uni_eltwise_injector::injector_postamble() { } template -void jit_uni_eltwise_injector::assign_regs() { +void jit_uni_eltwise_injector_t::assign_regs() { vmm_mask_ = Vmm(preserved_vmm_tail_indices_[0]); // For avx we need a register to save the upper part of Ymm @@ -323,13 +323,13 @@ void jit_uni_eltwise_injector::assign_regs() { // initialized with stock values from the injector, or with external values // provided by the user. template -Wmm jit_uni_eltwise_injector::vmm_aux(size_t idx) { +Wmm jit_uni_eltwise_injector_t::vmm_aux(size_t idx) { assert(idx < (n_vregs_preserved_ - need_vmm_mask_register_)); return Vmm(preserved_vmm_indices_[idx]); } template -void jit_uni_eltwise_injector::vec_shift(const Vmm &vmm_dst, +void jit_uni_eltwise_injector_t::vec_shift(const Vmm &vmm_dst, const Vmm &vmm_src, bool shift_left, const int imm) { if (isa != avx) { if (shift_left) @@ -357,7 +357,7 @@ void jit_uni_eltwise_injector::vec_shift(const Vmm &vmm_dst, // Uses injector masks objects: k_mask_ (>= avx512_core) or vmm_mask_ (<= avx2). // Stores a mask by applying cmpps on two inputs w/ a given predicate. template -void jit_uni_eltwise_injector::compute_cmp_mask(const Vmm &vmm_src, +void jit_uni_eltwise_injector_t::compute_cmp_mask(const Vmm &vmm_src, const Xbyak::Operand &compare_operand, int cmp_predicate) { if (is_avx512_) { h->vcmpps(k_mask_, vmm_src, compare_operand, cmp_predicate); @@ -369,7 +369,7 @@ void jit_uni_eltwise_injector::compute_cmp_mask(const Vmm &vmm_src, // Uses injector masks objects: k_mask_ (>= avx512_core) or vmm_mask_ (<= avx2). // Blends a result of second input into a first input w/ a stored mask. template -void jit_uni_eltwise_injector::blend_with_mask( +void jit_uni_eltwise_injector_t::blend_with_mask( const Vmm &vmm_dst, const Xbyak::Operand &src) { if (is_avx512_) { h->vblendmps(vmm_dst | k_mask_, vmm_dst, src); @@ -382,7 +382,7 @@ void jit_uni_eltwise_injector::blend_with_mask( // Tests a mask for all zeros. If all zeroes occur, set ZF = 1. // Nicely combines with jump_if_zero (jz). template -void jit_uni_eltwise_injector::test_mask() { +void jit_uni_eltwise_injector_t::test_mask() { if (is_avx512_) { h->kortestw(k_mask_, k_mask_); } else { @@ -391,7 +391,7 @@ void jit_uni_eltwise_injector::test_mask() { } template -void jit_uni_eltwise_injector::exp_compute_vector_fwd( +void jit_uni_eltwise_injector_t::exp_compute_vector_fwd( const Vmm &vmm_src) { // exp(x) = // = exp(n * ln(2) + r) // divide x by ln(2) and get quot and rem @@ -457,7 +457,7 @@ void jit_uni_eltwise_injector::exp_compute_vector_fwd( } template -void jit_uni_eltwise_injector::relu_compute_vector_fwd( +void jit_uni_eltwise_injector_t::relu_compute_vector_fwd( const Vmm &vmm_src) { h->uni_vmovups(vmm_aux(0), vmm_src); compute_cmp_mask(vmm_src, table_val(zero), _cmp_gt_os); @@ -466,13 +466,13 @@ void jit_uni_eltwise_injector::relu_compute_vector_fwd( } template -void jit_uni_eltwise_injector::relu_zero_ns_compute_vector_fwd( +void jit_uni_eltwise_injector_t::relu_zero_ns_compute_vector_fwd( const Vmm &vmm_src) { h->uni_vmaxps(vmm_src, vmm_src, table_val(zero)); } template -void jit_uni_eltwise_injector::elu_compute_vector_fwd( +void jit_uni_eltwise_injector_t::elu_compute_vector_fwd( const Vmm &vmm_src) { // IMPORTANT: we use vmm_aux(2) for the mask as exp_compute does not use it. h->uni_vmovups(vmm_aux(2), vmm_src); @@ -489,7 +489,7 @@ void jit_uni_eltwise_injector::elu_compute_vector_fwd( } template -void jit_uni_eltwise_injector::tanh_compute_vector_fwd( +void jit_uni_eltwise_injector_t::tanh_compute_vector_fwd( const Vmm &vmm_src) { // we add a check as the avx2 code cannot be used for avx assert(IMPLICATION(isa == avx2, mayiuse(avx2))); @@ -695,7 +695,7 @@ void jit_uni_eltwise_injector::tanh_compute_vector_fwd( } template -void jit_uni_eltwise_injector::gelu_tanh_compute_vector_fwd( +void jit_uni_eltwise_injector_t::gelu_tanh_compute_vector_fwd( const Vmm &vmm_src) { h->uni_vmovups(vmm_aux(0), vmm_src); @@ -721,26 +721,26 @@ void jit_uni_eltwise_injector::gelu_tanh_compute_vector_fwd( } template -void jit_uni_eltwise_injector::square_compute_vector_fwd( +void jit_uni_eltwise_injector_t::square_compute_vector_fwd( const Vmm &vmm_src) { h->uni_vmulps(vmm_src, vmm_src, vmm_src); } template -void jit_uni_eltwise_injector::abs_compute_vector_fwd( +void jit_uni_eltwise_injector_t::abs_compute_vector_fwd( const Vmm &vmm_src) { // compute abs(x) = _mm_and_ps(x, 01111..111)); h->uni_vandps(vmm_src, vmm_src, table_val(positive_mask)); } template -void jit_uni_eltwise_injector::sqrt_compute_vector_fwd( +void jit_uni_eltwise_injector_t::sqrt_compute_vector_fwd( const Vmm &vmm_src) { h->uni_vsqrtps(vmm_src, vmm_src); } template -void jit_uni_eltwise_injector::linear_compute_vector_fwd( +void jit_uni_eltwise_injector_t::linear_compute_vector_fwd( const Vmm &vmm_src) { // compute x = alpha * x + beta; h->uni_vmovups(vmm_aux(0), table_val(alpha)); @@ -748,14 +748,14 @@ void jit_uni_eltwise_injector::linear_compute_vector_fwd( } template -void jit_uni_eltwise_injector::clip_compute_vector_fwd( +void jit_uni_eltwise_injector_t::clip_compute_vector_fwd( const Vmm &vmm_src) { h->uni_vmaxps(vmm_src, vmm_src, table_val(alpha)); h->uni_vminps(vmm_src, vmm_src, table_val(beta)); } template -void jit_uni_eltwise_injector::mish_compute_vector_fwd( +void jit_uni_eltwise_injector_t::mish_compute_vector_fwd( const Vmm &vmm_src) { // An equation other than mish(x) = x*tanh(srelu(x)) was used // to calculate mish, but it should be remembered that it is equivalent @@ -788,7 +788,7 @@ void jit_uni_eltwise_injector::mish_compute_vector_fwd( } template -void jit_uni_eltwise_injector::hardswish_compute_vector_fwd( +void jit_uni_eltwise_injector_t::hardswish_compute_vector_fwd( const Vmm &vmm_src) { // result = x * hardsigmoid(x) h->uni_vmovups(vmm_aux(0), vmm_src); @@ -797,7 +797,7 @@ void jit_uni_eltwise_injector::hardswish_compute_vector_fwd( } template -void jit_uni_eltwise_injector::hardsigmoid_compute_vector_fwd( +void jit_uni_eltwise_injector_t::hardsigmoid_compute_vector_fwd( const Vmm &vmm_src) { // result = max(0, min(1, alpha * x + beta)) h->uni_vmulps(vmm_src, vmm_src, table_val(alpha)); @@ -807,7 +807,7 @@ void jit_uni_eltwise_injector::hardsigmoid_compute_vector_fwd( } template -void jit_uni_eltwise_injector::soft_relu_compute_vector_fwd( +void jit_uni_eltwise_injector_t::soft_relu_compute_vector_fwd( const Vmm &vmm_src) { // alpha scaling h->uni_vmulps(vmm_src, vmm_src, table_val(alpha)); @@ -929,7 +929,7 @@ void jit_uni_eltwise_injector::soft_relu_compute_vector_fwd( } template -void jit_uni_eltwise_injector::logistic_compute_vector_fwd( +void jit_uni_eltwise_injector_t::logistic_compute_vector_fwd( const Vmm &vmm_src) { // To avoid exp(x) overflow happened at x > logf(FLT_MAX), negate positive, // compute exp(x), where x <= 0 to get 0 <= exp(x) <= 1 and restore value @@ -962,7 +962,7 @@ void jit_uni_eltwise_injector::logistic_compute_vector_fwd( } template -void jit_uni_eltwise_injector::swish_compute_vector_fwd( +void jit_uni_eltwise_injector_t::swish_compute_vector_fwd( const Vmm &vmm_src) { // Save src data for later usage h->uni_vmovups(vmm_aux(3), vmm_src); @@ -976,7 +976,7 @@ void jit_uni_eltwise_injector::swish_compute_vector_fwd( } template -void jit_uni_eltwise_injector::log_compute_vector_fwd( +void jit_uni_eltwise_injector_t::log_compute_vector_fwd( const Vmm &vmm_src) { // From J.-M. Muller and others, Handbook of Floating-Point Arithmetic, 2010 // Here is a brief mathematics to approximate log(x): @@ -1171,7 +1171,7 @@ void jit_uni_eltwise_injector::log_compute_vector_fwd( } template -void jit_uni_eltwise_injector::pow_compute_vector_fwd( +void jit_uni_eltwise_injector_t::pow_compute_vector_fwd( const Vmm &vmm_src) { // dispatch between special cases. if (beta_ == -1) { // alpha / x @@ -1284,7 +1284,7 @@ void jit_uni_eltwise_injector::pow_compute_vector_fwd( } template -void jit_uni_eltwise_injector::gelu_erf_minimax_approx_compute_vector_fwd(const Vmm &vmm_src) { using namespace Xbyak::util; @@ -1352,7 +1352,7 @@ void jit_uni_eltwise_injector -void jit_uni_eltwise_injector::gelu_erf_compute_vector_fwd( +void jit_uni_eltwise_injector_t::gelu_erf_compute_vector_fwd( const Vmm &vmm_src) { if (is_avx512_) { gelu_erf_minimax_approx_compute_vector_fwd(vmm_src); @@ -1421,7 +1421,7 @@ void jit_uni_eltwise_injector::gelu_erf_compute_vector_fwd( } template -void jit_uni_eltwise_injector::relu_compute_vector_bwd( +void jit_uni_eltwise_injector_t::relu_compute_vector_bwd( const Vmm &vmm_src) { // invariant to whether `s` or `d` is passed. // get mask of `s` > 0 @@ -1432,7 +1432,7 @@ void jit_uni_eltwise_injector::relu_compute_vector_bwd( } template -void jit_uni_eltwise_injector::elu_compute_vector_bwd( +void jit_uni_eltwise_injector_t::elu_compute_vector_bwd( const Vmm &vmm_src) { if (use_dst_) { // get mask of `d` > 0 @@ -1457,7 +1457,7 @@ void jit_uni_eltwise_injector::elu_compute_vector_bwd( } template -void jit_uni_eltwise_injector::tanh_compute_vector_bwd( +void jit_uni_eltwise_injector_t::tanh_compute_vector_bwd( const Vmm &vmm_src) { // res = 1 - d^2 = 1 - tanh^2(s) if (!use_dst_) tanh_compute_vector_fwd(vmm_src); @@ -1467,7 +1467,7 @@ void jit_uni_eltwise_injector::tanh_compute_vector_bwd( } template -void jit_uni_eltwise_injector::gelu_tanh_compute_vector_bwd( +void jit_uni_eltwise_injector_t::gelu_tanh_compute_vector_bwd( const Vmm &vmm_src) { h->uni_vmovups(vmm_aux(0), vmm_src); @@ -1514,14 +1514,14 @@ void jit_uni_eltwise_injector::gelu_tanh_compute_vector_bwd( } template -void jit_uni_eltwise_injector::square_compute_vector_bwd( +void jit_uni_eltwise_injector_t::square_compute_vector_bwd( const Vmm &vmm_src) { // res = 2 * s h->uni_vmulps(vmm_src, vmm_src, table_val(two)); } template -void jit_uni_eltwise_injector::abs_compute_vector_bwd( +void jit_uni_eltwise_injector_t::abs_compute_vector_bwd( const Vmm &vmm_src) { // replace positive values with 1.f compute_cmp_mask(vmm_src, table_val(zero), _cmp_gt_os); @@ -1532,7 +1532,7 @@ void jit_uni_eltwise_injector::abs_compute_vector_bwd( } template -void jit_uni_eltwise_injector::sqrt_compute_vector_bwd( +void jit_uni_eltwise_injector_t::sqrt_compute_vector_bwd( const Vmm &vmm_src) { // res = 0.5 / d = 0.5 / sqrt(s) if (!use_dst_) sqrt_compute_vector_fwd(vmm_src); @@ -1543,20 +1543,20 @@ void jit_uni_eltwise_injector::sqrt_compute_vector_bwd( } template -void jit_uni_eltwise_injector::linear_compute_vector_bwd( +void jit_uni_eltwise_injector_t::linear_compute_vector_bwd( const Vmm &vmm_src) { h->uni_vmovups(vmm_src, table_val(alpha)); } template -void jit_uni_eltwise_injector::soft_relu_compute_vector_bwd( +void jit_uni_eltwise_injector_t::soft_relu_compute_vector_bwd( const Vmm &vmm_src) { h->uni_vmulps(vmm_src, vmm_src, table_val(alpha)); logistic_compute_vector_fwd(vmm_src); } template -void jit_uni_eltwise_injector::mish_compute_vector_bwd( +void jit_uni_eltwise_injector_t::mish_compute_vector_bwd( const Vmm &vmm_src) { // IMPORTANT: we use vmm_aux(2) to save src as exp does not use it. h->uni_vmovups(vmm_aux(2), vmm_src); // vmm_aux(2) = x @@ -1597,7 +1597,7 @@ void jit_uni_eltwise_injector::mish_compute_vector_bwd( } template -void jit_uni_eltwise_injector::logistic_compute_vector_bwd( +void jit_uni_eltwise_injector_t::logistic_compute_vector_bwd( const Vmm &vmm_src) { // res = d * (1 - d) = d - d * d; d = logistic(s) if (!use_dst_) logistic_compute_vector_fwd(vmm_src); @@ -1608,13 +1608,13 @@ void jit_uni_eltwise_injector::logistic_compute_vector_bwd( } template -void jit_uni_eltwise_injector::exp_compute_vector_bwd( +void jit_uni_eltwise_injector_t::exp_compute_vector_bwd( const Vmm &vmm_src) { if (!use_dst_) exp_compute_vector_fwd(vmm_src); } template -void jit_uni_eltwise_injector::swish_compute_vector_bwd( +void jit_uni_eltwise_injector_t::swish_compute_vector_bwd( const Vmm &vmm_src) { // R = alpha * s h->uni_vmulps(vmm_src, vmm_src, table_val(alpha)); @@ -1638,7 +1638,7 @@ void jit_uni_eltwise_injector::swish_compute_vector_bwd( } template -void jit_uni_eltwise_injector::log_compute_vector_bwd( +void jit_uni_eltwise_injector_t::log_compute_vector_bwd( const Vmm &vmm_src) { // res = 1 / s h->uni_vmovups(vmm_aux(0), table_val(one)); @@ -1648,7 +1648,7 @@ void jit_uni_eltwise_injector::log_compute_vector_bwd( } template -void jit_uni_eltwise_injector::clip_compute_vector_bwd( +void jit_uni_eltwise_injector_t::clip_compute_vector_bwd( const Vmm &vmm_src) { // set result with 1.f h->uni_vmovups(vmm_aux(0), table_val(one)); @@ -1664,7 +1664,7 @@ void jit_uni_eltwise_injector::clip_compute_vector_bwd( } template -void jit_uni_eltwise_injector::pow_compute_vector_bwd( +void jit_uni_eltwise_injector_t::pow_compute_vector_bwd( const Vmm &vmm_src) { // dispatch some special cases. if (beta_ == 0) { // zero @@ -1695,7 +1695,7 @@ void jit_uni_eltwise_injector::pow_compute_vector_bwd( } template -void jit_uni_eltwise_injector::gelu_erf_compute_vector_bwd( +void jit_uni_eltwise_injector_t::gelu_erf_compute_vector_bwd( const Vmm &vmm_src) { // R = s / sqrt(2) h->uni_vmulps(vmm_src, vmm_src, @@ -1759,7 +1759,7 @@ void jit_uni_eltwise_injector::gelu_erf_compute_vector_bwd( } template -void jit_uni_eltwise_injector::hardswish_compute_vector_bwd( +void jit_uni_eltwise_injector_t::hardswish_compute_vector_bwd( const Vmm &vmm_src) { // Get mask for 0 < alpha * x + beta < 1 h->uni_vmovups(vmm_aux(0), vmm_src); @@ -1776,7 +1776,7 @@ void jit_uni_eltwise_injector::hardswish_compute_vector_bwd( } template -void jit_uni_eltwise_injector::hardsigmoid_compute_vector_bwd( +void jit_uni_eltwise_injector_t::hardsigmoid_compute_vector_bwd( const Vmm &vmm_src) { // Get mask for 0 < alpha * x + beta < 1 // Zero rest values. @@ -1793,13 +1793,13 @@ void jit_uni_eltwise_injector::hardsigmoid_compute_vector_bwd( } template -void jit_uni_eltwise_injector::round_compute_vector_fwd( +void jit_uni_eltwise_injector_t::round_compute_vector_fwd( const Vmm &vmm_src) { h->uni_vroundps(vmm_src, vmm_src, _op_mxcsr); } template -size_t jit_uni_eltwise_injector::aux_gprs_count( +size_t jit_uni_eltwise_injector_t::aux_gprs_count( alg_kind_t alg, bool is_fwd, float alpha) { using namespace alg_kind; int ret = 0; @@ -1813,13 +1813,13 @@ size_t jit_uni_eltwise_injector::aux_gprs_count( }; template -bool jit_uni_eltwise_injector::need_vmm_stack_ptr( +bool jit_uni_eltwise_injector_t::need_vmm_stack_ptr( alg_kind_t alg, bool is_fwd, float alpha) { return op_vecs_count(alg, is_fwd) + aux_vecs_count(alg, is_fwd, alpha); } template -size_t jit_uni_eltwise_injector::op_vecs_count( +size_t jit_uni_eltwise_injector_t::op_vecs_count( alg_kind_t alg, bool is_fwd) { using namespace alg_kind; int ret = 0; @@ -1841,7 +1841,7 @@ size_t jit_uni_eltwise_injector::op_vecs_count( } template -size_t jit_uni_eltwise_injector::aux_vecs_count( +size_t jit_uni_eltwise_injector_t::aux_vecs_count( alg_kind_t alg, bool is_fwd, float alpha) { // For avx we need a register to save the upper part of Ymm const bool extra_avx_vmm = isa == avx; @@ -1923,7 +1923,7 @@ size_t jit_uni_eltwise_injector::aux_vecs_count( } template -bool jit_uni_eltwise_injector::need_mask_register( +bool jit_uni_eltwise_injector_t::need_mask_register( alg_kind_t alg, bool is_fwd, float alpha) { if (is_superset(isa, avx512_core)) return false; @@ -1997,7 +1997,7 @@ bool jit_uni_eltwise_injector::need_mask_register( } template -void jit_uni_eltwise_injector::compute_body( +void jit_uni_eltwise_injector_t::compute_body( const injector_utils::vmm_index_set_iterator_t &start_idx_it, const injector_utils::vmm_index_set_iterator_t &end_idx_it) { using namespace alg_kind; @@ -2102,7 +2102,7 @@ void jit_uni_eltwise_injector::compute_body( } template -void jit_uni_eltwise_injector::compute_vector_range( +void jit_uni_eltwise_injector_t::compute_vector_range( size_t start_compute_idx, size_t end_compute_idx, const injector_utils::vmm_index_set_t &vmm_aux_indices) { injector_utils::vmm_index_set_t vmm_compute_idxs; @@ -2112,7 +2112,7 @@ void jit_uni_eltwise_injector::compute_vector_range( } template -void jit_uni_eltwise_injector::compute_vector_range( +void jit_uni_eltwise_injector_t::compute_vector_range( const injector_utils::vmm_index_set_t &vmm_compute_idxs, const injector_utils::vmm_index_set_t &vmm_aux_indices) { if (vmm_compute_idxs.empty()) return; @@ -2135,7 +2135,7 @@ void jit_uni_eltwise_injector::compute_vector_range( } template -void jit_uni_eltwise_injector::prepare_table(bool gen_table) { +void jit_uni_eltwise_injector_t::prepare_table(bool gen_table) { if (!gen_table) return; h->align(64); @@ -2177,7 +2177,7 @@ void jit_uni_eltwise_injector::prepare_table(bool gen_table) { } template -void jit_uni_eltwise_injector::register_table_entries() { +void jit_uni_eltwise_injector_t::register_table_entries() { // This function is responsible to pick all necessary constants // for a given algorithm, compute right offset for them to be used // in table_val() and save the hexadecimal value of them, which @@ -2923,20 +2923,20 @@ void jit_uni_eltwise_injector::register_table_entries() { } } -template struct jit_uni_eltwise_injector; -template struct jit_uni_eltwise_injector; -template struct jit_uni_eltwise_injector; -template struct jit_uni_eltwise_injector; -template struct jit_uni_eltwise_injector; -template struct jit_uni_eltwise_injector; -template struct jit_uni_eltwise_injector; -template struct jit_uni_eltwise_injector; -template struct jit_uni_eltwise_injector; -template struct jit_uni_eltwise_injector; -template struct jit_uni_eltwise_injector; -template struct jit_uni_eltwise_injector; -template struct jit_uni_eltwise_injector; -template struct jit_uni_eltwise_injector; +template struct jit_uni_eltwise_injector_t; +template struct jit_uni_eltwise_injector_t; +template struct jit_uni_eltwise_injector_t; +template struct jit_uni_eltwise_injector_t; +template struct jit_uni_eltwise_injector_t; +template struct jit_uni_eltwise_injector_t; +template struct jit_uni_eltwise_injector_t; +template struct jit_uni_eltwise_injector_t; +template struct jit_uni_eltwise_injector_t; +template struct jit_uni_eltwise_injector_t; +template struct jit_uni_eltwise_injector_t; +template struct jit_uni_eltwise_injector_t; +template struct jit_uni_eltwise_injector_t; +template struct jit_uni_eltwise_injector_t; } // namespace x64 } // namespace cpu diff --git a/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp b/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp index 924611c74c0..d44602eac4b 100644 --- a/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp +++ b/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp @@ -76,7 +76,7 @@ bool is_supported(cpu_isa_t isa, alg_kind_t alg, data_type_t dt); } // namespace eltwise_injector template ::Vmm> -struct jit_uni_eltwise_injector { +struct jit_uni_eltwise_injector_t { using Vmm = Wmm; // Arguments description: @@ -91,7 +91,7 @@ struct jit_uni_eltwise_injector { // - algorithm derivative. // use_dst - defines whether source or destination point is passed to alg // code. Depends on algorithm. See `_use_dst_for_bwd` algs definition. - jit_uni_eltwise_injector(jit_generator *host, alg_kind_t alg, float alpha, + jit_uni_eltwise_injector_t(jit_generator *host, alg_kind_t alg, float alpha, float beta, float scale, data_type_t dt = data_type::f32, bool save_state = true, Xbyak::Reg64 p_table = Xbyak::Reg64(Xbyak::Operand::RAX), @@ -117,14 +117,14 @@ struct jit_uni_eltwise_injector { register_table_entries(); } - jit_uni_eltwise_injector(jit_generator *host, + jit_uni_eltwise_injector_t(jit_generator *host, const post_ops_t::entry_t::eltwise_t &eltwise, data_type_t dt = data_type::f32, bool save_state = true, Xbyak::Reg64 p_table = Xbyak::Reg64(Xbyak::Operand::RAX), Xbyak::Opmask k_mask = Xbyak::Opmask(1), bool is_fwd = true, bool use_dst = false, bool preserve_vmm = true, bool preserve_p_table = true) - : jit_uni_eltwise_injector(host, eltwise.alg, eltwise.alpha, + : jit_uni_eltwise_injector_t(host, eltwise.alg, eltwise.alpha, eltwise.beta, eltwise.scale, dt, save_state, p_table, k_mask, is_fwd, use_dst, preserve_vmm, preserve_p_table) {} diff --git a/src/cpu/x64/injectors/jit_uni_postops_injector.cpp b/src/cpu/x64/injectors/jit_uni_postops_injector.cpp index 99e71861a1f..02ad28a44fb 100644 --- a/src/cpu/x64/injectors/jit_uni_postops_injector.cpp +++ b/src/cpu/x64/injectors/jit_uni_postops_injector.cpp @@ -31,7 +31,7 @@ size_t aux_vec_count(const post_ops_t &post_ops, cpu_isa_t isa, bool is_fwd) { #define CASE_ELTWISE_SUPERSET(_isa) \ if (is_superset(isa, _isa)) { \ res = nstl::max(res, \ - jit_uni_eltwise_injector<_isa>::aux_vecs_count( \ + jit_uni_eltwise_injector_t<_isa>::aux_vecs_count( \ post_op.eltwise.alg, is_fwd, post_op.eltwise.alpha)); \ continue; \ } @@ -75,7 +75,7 @@ jit_uni_postops_injector_t::jit_uni_postops_injector_t( // moment. Once the use case show up, add the argument to the // top-level ctor and propagate its value. alg_to_eltwise_injector_.emplace(i, - jit_uni_eltwise_injector(host_, post_op.eltwise, + jit_uni_eltwise_injector_t(host_, post_op.eltwise, data_type::f32, esp.save_state, esp.p_table_, esp.k_mask_, esp.is_fwd, esp.use_dst, esp.preserve_vmm, esp.preserve_p_table)); diff --git a/src/cpu/x64/injectors/jit_uni_postops_injector.hpp b/src/cpu/x64/injectors/jit_uni_postops_injector.hpp index 6fedc8a01c6..3c96e9922e2 100644 --- a/src/cpu/x64/injectors/jit_uni_postops_injector.hpp +++ b/src/cpu/x64/injectors/jit_uni_postops_injector.hpp @@ -173,7 +173,8 @@ class jit_uni_postops_injector_t : public jit_uni_postops_injector_base_t { post_ops_t post_ops_; jit_generator *host_; // Key is a numerical order of a post-op in attributes. - std::map> alg_to_eltwise_injector_; + std::map> + alg_to_eltwise_injector_; std::unique_ptr> binary_injector_; lambda_jit_injectors_t lambda_jit_injectors_; diff --git a/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp index c44760677f7..de19a179d1f 100644 --- a/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp @@ -477,9 +477,8 @@ struct jit_avx512_core_amx_bwd_data_kernel_t : public jit_generator { , eltwise_injector_(nullptr) , bwd_data_copy_kernel_(nullptr) { if (jcp.with_eltwise) - eltwise_injector_ - = utils::make_unique>( - this, jcp.eltwise); + eltwise_injector_ = utils::make_unique< + jit_uni_eltwise_injector_t>(this, jcp.eltwise); bwd_data_copy_kernel_ = utils::make_unique< jit_avx512_core_amx_bwd_data_copy_kernel_t>(jcp); } @@ -511,7 +510,7 @@ struct jit_avx512_core_amx_bwd_data_kernel_t : public jit_generator { } private: - std::unique_ptr> eltwise_injector_; + std::unique_ptr> eltwise_injector_; std::unique_ptr bwd_data_copy_kernel_; diff --git a/src/cpu/x64/jit_uni_eltwise.cpp b/src/cpu/x64/jit_uni_eltwise.cpp index 1491c82f6af..56e6056c0b0 100644 --- a/src/cpu/x64/jit_uni_eltwise.cpp +++ b/src/cpu/x64/jit_uni_eltwise.cpp @@ -92,8 +92,8 @@ struct jit_uni_kernel_t : public jit_uni_eltwise_kernel { // using the first 7 vregs can be considered volatile during the call // to eltwise injector const bool save_state = is_fwd_ ? false : true; - eltwise_injector_.reset(new jit_uni_eltwise_injector(this, - desc.alg_kind, desc.alpha, desc.beta, 1.f, data_type::f32, + eltwise_injector_.reset(new jit_uni_eltwise_injector_t( + this, desc.alg_kind, desc.alpha, desc.beta, 1.f, data_type::f32, save_state, reg_injector_table, injector_mask, is_fwd_, pd_->use_dst())); io::io_conf_t io_conf; @@ -261,7 +261,7 @@ struct jit_uni_kernel_t : public jit_uni_eltwise_kernel { Vmm vmm_src_odd = Vmm(8); Vmm vmm_diff_dst_even = vmm_diff_dst; Vmm vmm_diff_dst_odd = Vmm(9); - std::unique_ptr> eltwise_injector_; + std::unique_ptr> eltwise_injector_; io::jit_io_multi_dt_helper_t io_; /* bf16 and fp8 support */ diff --git a/src/cpu/x64/jit_uni_softmax.cpp b/src/cpu/x64/jit_uni_softmax.cpp index eb9e2e71f1d..231b5c6ae17 100644 --- a/src/cpu/x64/jit_uni_softmax.cpp +++ b/src/cpu/x64/jit_uni_softmax.cpp @@ -65,8 +65,8 @@ struct jit_softmax_dense_kernel_t : jit_softmax_kernel_base_t, const memory_desc_wrapper src_d_, dst_d_, diff_dst_d_; io::jit_io_multi_dt_helper_t io_; - std::unique_ptr> exp_injector_; - std::unique_ptr> log_injector_; + std::unique_ptr> exp_injector_; + std::unique_ptr> log_injector_; std::unique_ptr> postops_injector_; @@ -609,7 +609,7 @@ struct jit_softmax_dense_kernel_t : jit_softmax_kernel_base_t, // Prepare indices for exp aux vmms. injector_utils::vmm_index_set_t exp_aux_indices; const auto exp_vmm_aux_count - = jit_uni_eltwise_injector::aux_vecs_count( + = jit_uni_eltwise_injector_t::aux_vecs_count( alg_kind::eltwise_exp, pd_->is_fwd(), 0.f); for (size_t j = 0; j < exp_vmm_aux_count; j++) { // Insert the next idx starting after `vreg_tmp_sum`. @@ -899,11 +899,11 @@ struct jit_softmax_dense_kernel_t : jit_softmax_kernel_base_t, // initialization. void generate() override { if (pd_->is_fwd() || is_logsoftmax_) - exp_injector_.reset(new jit_uni_eltwise_injector(this, + exp_injector_.reset(new jit_uni_eltwise_injector_t(this, alg_kind::eltwise_exp, 0.0f, 0.0f, 1.0f, data_type::f32, !use_ext_aux_vmms_, reg_exp_injector_table, injector_mask)); if (pd_->is_fwd() && is_logsoftmax_) { - log_injector_.reset(new jit_uni_eltwise_injector(this, + log_injector_.reset(new jit_uni_eltwise_injector_t(this, alg_kind::eltwise_log, 0.0f, 0.0f, 1.0f, data_type::f32, true, reg_log_injector_table, injector_mask)); } @@ -1010,8 +1010,8 @@ struct jit_softmax_strided_kernel_t : jit_softmax_kernel_base_t, const memory_desc_wrapper src_d_, dst_d_; io::jit_io_multi_dt_helper_t io_; - std::unique_ptr> exp_injector_; - std::unique_ptr> log_injector_; + std::unique_ptr> exp_injector_; + std::unique_ptr> log_injector_; std::unique_ptr> postops_injector_; @@ -1453,11 +1453,11 @@ struct jit_softmax_strided_kernel_t : jit_softmax_kernel_base_t, void generate() override { if (pd_->is_fwd() || is_logsoftmax_) - exp_injector_.reset(new jit_uni_eltwise_injector(this, + exp_injector_.reset(new jit_uni_eltwise_injector_t(this, alg_kind::eltwise_exp, 0.0f, 0.0f, 1.0f, data_type::f32, true, reg_exp_injector_table, injector_mask)); if (pd_->is_fwd() && is_logsoftmax_) { - log_injector_.reset(new jit_uni_eltwise_injector(this, + log_injector_.reset(new jit_uni_eltwise_injector_t(this, alg_kind::eltwise_log, 0.0f, 0.0f, 1.0f, data_type::f32, true, reg_log_injector_table, injector_mask)); } diff --git a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_fwd.hpp b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_fwd.hpp index 2fa3920554f..4135d1baa95 100644 --- a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_fwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_1_fwd.hpp @@ -31,8 +31,8 @@ struct jit_uni_gru_cell_postgemm_part1_fwd : public jit_uni_rnn_postgemm { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_gru_cell_postgemm_part1_fwd) using injector_t = typename utils::conditional, - jit_uni_eltwise_injector>::type; + jit_uni_eltwise_injector_t, + jit_uni_eltwise_injector_t>::type; jit_uni_gru_cell_postgemm_part1_fwd( const rnn_utils::rnn_conf_t &rnn, const rnn_pd_t *pd) @@ -55,7 +55,7 @@ struct jit_uni_gru_cell_postgemm_part1_fwd : public jit_uni_rnn_postgemm { std::unique_ptr sigmoid_injector_; // register size in bytes - using Vmm = typename jit_uni_eltwise_injector::Vmm; + using Vmm = typename jit_uni_eltwise_injector_t::Vmm; static constexpr size_t vlen = cpu_isa_traits_t::vlen; static constexpr size_t qscale_dt_size = sizeof(float); const size_t vlen_dst diff --git a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_fwd.hpp b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_fwd.hpp index e314b5a6617..86fb3cb8165 100644 --- a/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_fwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_gru_cell_postgemm_2_fwd.hpp @@ -30,8 +30,8 @@ struct jit_uni_gru_cell_postgemm_part2_fwd : public jit_uni_rnn_postgemm { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_gru_cell_postgemm_part2_fwd) using injector_t = typename utils::conditional, - jit_uni_eltwise_injector>::type; + jit_uni_eltwise_injector_t, + jit_uni_eltwise_injector_t>::type; jit_uni_gru_cell_postgemm_part2_fwd( const rnn_utils::rnn_conf_t &rnn, const rnn_pd_t *pd) @@ -55,7 +55,7 @@ struct jit_uni_gru_cell_postgemm_part2_fwd : public jit_uni_rnn_postgemm { std::unique_ptr tanh_injector_; // register size in bytes - using Vmm = typename jit_uni_eltwise_injector::Vmm; + using Vmm = typename jit_uni_eltwise_injector_t::Vmm; static constexpr size_t vlen = cpu_isa_traits_t::vlen; static constexpr size_t qscale_dt_size = sizeof(float); const size_t vlen_dst diff --git a/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_fwd.hpp b/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_fwd.hpp index 87c5e0d13a4..24663441828 100644 --- a/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_fwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_gru_lbr_cell_postgemm_fwd.hpp @@ -31,8 +31,8 @@ struct jit_uni_gru_lbr_cell_postgemm_fwd : public jit_uni_rnn_postgemm { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_gru_lbr_cell_postgemm_fwd) using injector_t = typename utils::conditional, - jit_uni_eltwise_injector>::type; + jit_uni_eltwise_injector_t, + jit_uni_eltwise_injector_t>::type; jit_uni_gru_lbr_cell_postgemm_fwd( const rnn_utils::rnn_conf_t &rnn, const rnn_pd_t *pd) @@ -56,7 +56,7 @@ struct jit_uni_gru_lbr_cell_postgemm_fwd : public jit_uni_rnn_postgemm { std::unique_ptr tanh_injector_; // register size in bytes - using Vmm = typename jit_uni_eltwise_injector::Vmm; + using Vmm = typename jit_uni_eltwise_injector_t::Vmm; static constexpr size_t vlen = cpu_isa_traits_t::vlen; const size_t vlen_dst diff --git a/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm.hpp b/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm.hpp index 18cf5b44b46..b9ad23ffa4c 100644 --- a/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm.hpp +++ b/src/cpu/x64/rnn/jit_uni_lstm_cell_postgemm.hpp @@ -37,7 +37,7 @@ struct jit_uni_lstm_cell_postgemm_t { } protected: - using injector_t = jit_uni_eltwise_injector; + using injector_t = jit_uni_eltwise_injector_t; using Vmm = typename cpu_isa_traits_t::Vmm; const size_t vlen_ = cpu_isa_traits_t::vlen; diff --git a/src/cpu/x64/rnn/jit_uni_lstm_cell_projection_postgemm_fwd.hpp b/src/cpu/x64/rnn/jit_uni_lstm_cell_projection_postgemm_fwd.hpp index dcd890c0bed..125fa9826b0 100644 --- a/src/cpu/x64/rnn/jit_uni_lstm_cell_projection_postgemm_fwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_lstm_cell_projection_postgemm_fwd.hpp @@ -43,7 +43,7 @@ struct jit_uni_lstm_cell_projection_postgemm_fwd : public jit_uni_rnn_postgemm { protected: // register size in bytes - using Vmm = typename jit_uni_eltwise_injector::Vmm; + using Vmm = typename jit_uni_eltwise_injector_t::Vmm; static constexpr size_t vlen = cpu_isa_traits_t::vlen; static constexpr size_t qscale_dt_size = sizeof(float); const size_t vlen_dst diff --git a/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_fwd.hpp b/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_fwd.hpp index 9e4b9a37c1d..f064e268d0f 100644 --- a/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_fwd.hpp +++ b/src/cpu/x64/rnn/jit_uni_rnn_cell_postgemm_fwd.hpp @@ -31,8 +31,8 @@ struct jit_uni_rnn_cell_postgemm_fwd : public jit_uni_rnn_postgemm { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_rnn_cell_postgemm_fwd) using injector_t = typename utils::conditional, - jit_uni_eltwise_injector>::type; + jit_uni_eltwise_injector_t, + jit_uni_eltwise_injector_t>::type; jit_uni_rnn_cell_postgemm_fwd( const rnn_utils::rnn_conf_t &rnn, const rnn_pd_t *pd) @@ -51,7 +51,7 @@ struct jit_uni_rnn_cell_postgemm_fwd : public jit_uni_rnn_postgemm { std::unique_ptr injector_; // register size in bytes - using Vmm = typename jit_uni_eltwise_injector::Vmm; + using Vmm = typename jit_uni_eltwise_injector_t::Vmm; static constexpr size_t vlen = cpu_isa_traits_t::vlen; static constexpr size_t cstate_dt_size = sizeof(float); static constexpr size_t qscale_dt_size = sizeof(float); From 4a5ff650b6a14e1d8bccf5054bd684256fe0281b Mon Sep 17 00:00:00 2001 From: Dmitrii Zarukin Date: Mon, 17 Mar 2025 14:21:30 -0700 Subject: [PATCH 09/16] cpu: jit_generator -> jit_generator_t --- src/cpu/x64/amx_tile_configure.cpp | 8 +- src/cpu/x64/brgemm/brgemm_types.hpp | 14 +- src/cpu/x64/brgemm/jit_brdgmm_kernel.cpp | 2 +- src/cpu/x64/brgemm/jit_brgemm_amx_uker.cpp | 2 +- src/cpu/x64/brgemm/jit_brgemm_kernel.cpp | 2 +- src/cpu/x64/cpu_barrier.cpp | 8 +- src/cpu/x64/cpu_barrier.hpp | 7 +- src/cpu/x64/cpu_reducer.cpp | 6 +- .../amx/jit_avx512_core_amx_copy_kern.cpp | 4 +- .../amx/jit_avx512_core_amx_copy_kern.hpp | 2 +- .../amx/jit_avx512_core_amx_gemm_kern.cpp | 4 +- .../amx/jit_avx512_core_amx_gemm_kern.hpp | 2 +- src/cpu/x64/gemm/bf16/common_s16.hpp | 16 +- .../jit_avx512_core_gemm_bf16bf16f32_kern.cpp | 4 +- .../jit_avx512_core_gemm_bf16bf16f32_kern.hpp | 2 +- .../jit_avx512_core_gemv_bf16bf16f32_kern.cpp | 4 +- .../jit_avx512_core_gemv_bf16bf16f32_kern.hpp | 2 +- ...512_core_s16_24x8_copy_an_kern_autogen.cpp | 4 +- ...512_core_s16_24x8_copy_at_kern_autogen.cpp | 4 +- ...512_core_s16_24x8_copy_bn_kern_autogen.cpp | 4 +- ...512_core_s16_24x8_copy_bt_kern_autogen.cpp | 4 +- ...512_core_s16_48x8_copy_an_kern_autogen.cpp | 4 +- ...512_core_s16_48x8_copy_at_kern_autogen.cpp | 4 +- ...512_core_s16_48x8_copy_bn_kern_autogen.cpp | 4 +- ...512_core_s16_48x8_copy_bt_kern_autogen.cpp | 4 +- src/cpu/x64/gemm/f32/common_f32.hpp | 40 ++--- .../f32/jit_avx2_f32_copy_an_kern_autogen.cpp | 4 +- .../f32/jit_avx2_f32_copy_at_kern_autogen.cpp | 4 +- .../f32/jit_avx2_f32_copy_bn_kern_autogen.cpp | 4 +- .../f32/jit_avx2_f32_copy_bt_kern_autogen.cpp | 4 +- .../gemm/f32/jit_avx2_kernel_sgemm_kern.cpp | 4 +- .../gemm/f32/jit_avx2_kernel_sgemm_kern.hpp | 2 +- .../gemm/f32/jit_avx512_common_gemm_f32.cpp | 4 +- ...t_avx512_core_f32_copy_an_kern_autogen.cpp | 4 +- ...12_core_f32_copy_at_kern_part1_autogen.cpp | 4 +- ...t_avx512_core_f32_copy_bn_kern_autogen.cpp | 4 +- ...t_avx512_core_f32_copy_bt_kern_autogen.cpp | 4 +- ...it_avx512_core_gemm_smalln_tn_f32_kern.cpp | 4 +- .../f32/jit_avx_f32_copy_an_kern_autogen.cpp | 4 +- .../f32/jit_avx_f32_copy_at_kern_autogen.cpp | 4 +- .../f32/jit_avx_f32_copy_bn_kern_autogen.cpp | 4 +- .../f32/jit_avx_f32_copy_bt_kern_autogen.cpp | 4 +- src/cpu/x64/gemm/f32/jit_avx_gemm_f32.cpp | 4 +- .../x64/gemm/f32/jit_avx_gemv_t_f32_kern.cpp | 4 +- .../x64/gemm/f32/jit_avx_gemv_t_f32_kern.hpp | 2 +- ...avx_kernel_b0_sgemm_kern_part1_autogen.cpp | 4 +- ...it_avx_kernel_sgemm_kern_part1_autogen.cpp | 4 +- .../jit_sse41_f32_copy_an_kern_autogen.cpp | 4 +- .../jit_sse41_f32_copy_at_kern_autogen.cpp | 4 +- .../jit_sse41_f32_copy_bn_kern_autogen.cpp | 4 +- .../jit_sse41_f32_copy_bt_kern_autogen.cpp | 4 +- .../gemm/f32/jit_sse41_gemv_n_f32_kern.cpp | 4 +- .../gemm/f32/jit_sse41_gemv_n_f32_kern.hpp | 2 +- .../gemm/f32/jit_sse41_gemv_t_f32_kern.cpp | 4 +- .../gemm/f32/jit_sse41_gemv_t_f32_kern.hpp | 2 +- ...jit_sse41_kernel_b0_sgemm_kern_autogen.cpp | 4 +- .../jit_sse41_kernel_sgemm_kern_autogen.cpp | 4 +- src/cpu/x64/gemm/gemm_info.cpp | 14 +- src/cpu/x64/gemm/s8x8s32/common_u8.hpp | 112 +++++++------- .../s8x8s32/jit_avx2_gemm_s8u8s32_kern.cpp | 4 +- .../s8x8s32/jit_avx2_gemm_s8u8s32_kern.hpp | 2 +- .../jit_avx2_u8_copy_an_kern_autogen.cpp | 4 +- .../jit_avx2_u8_copy_at_kern_autogen.cpp | 4 +- .../jit_avx2_u8_copy_bn_kern_autogen.cpp | 4 +- .../jit_avx2_u8_copy_bt_kern_autogen.cpp | 4 +- .../jit_avx2_u8_copy_sum_an_kern_autogen.cpp | 4 +- .../jit_avx2_u8_copy_sum_at_kern_autogen.cpp | 4 +- .../jit_avx2_u8_copy_sum_bn_kern_autogen.cpp | 4 +- .../jit_avx2_u8_copy_sum_bt_kern_autogen.cpp | 4 +- .../jit_avx2_vnni_u8_copy_an_kern_autogen.cpp | 4 +- .../jit_avx2_vnni_u8_copy_at_kern_autogen.cpp | 4 +- .../jit_avx2_vnni_u8_copy_bn_kern_autogen.cpp | 4 +- .../jit_avx2_vnni_u8_copy_bt_kern_autogen.cpp | 4 +- ..._avx2_vnni_u8_copy_sum_an_kern_autogen.cpp | 4 +- ..._avx2_vnni_u8_copy_sum_at_kern_autogen.cpp | 4 +- ..._avx2_vnni_u8_copy_sum_bn_kern_autogen.cpp | 4 +- ..._avx2_vnni_u8_copy_sum_bt_kern_autogen.cpp | 4 +- .../jit_avx512_core_gemm_s8u8s32_kern.cpp | 4 +- .../jit_avx512_core_gemm_s8u8s32_kern.hpp | 2 +- ...t_avx512_core_kernel_gemv_s8x8s32_kern.hpp | 4 +- ...it_avx512_core_u8_copy_an_kern_autogen.cpp | 4 +- ...it_avx512_core_u8_copy_at_kern_autogen.cpp | 4 +- ...it_avx512_core_u8_copy_bn_kern_autogen.cpp | 4 +- ...it_avx512_core_u8_copy_bt_kern_autogen.cpp | 4 +- ...vx512_core_u8_copy_sum_an_kern_autogen.cpp | 4 +- ...vx512_core_u8_copy_sum_at_kern_autogen.cpp | 4 +- ...vx512_core_u8_copy_sum_bn_kern_autogen.cpp | 4 +- ...vx512_core_u8_copy_sum_bt_kern_autogen.cpp | 4 +- ..._kernel_b0_b_gemm_s8u8s32_kern_autogen.cpp | 4 +- ..._kernel_b0_c_gemm_s8u8s32_kern_autogen.cpp | 4 +- ...vx_kernel_b0_gemm_s8u8s32_kern_autogen.cpp | 4 +- ..._kernel_b0_r_gemm_s8u8s32_kern_autogen.cpp | 4 +- ...avx_kernel_b_gemm_s8u8s32_kern_autogen.cpp | 4 +- ...avx_kernel_c_gemm_s8u8s32_kern_autogen.cpp | 4 +- ...t_avx_kernel_gemm_s8u8s32_kern_autogen.cpp | 4 +- ...avx_kernel_r_gemm_s8u8s32_kern_autogen.cpp | 4 +- .../jit_avx_u8_copy_an_kern_autogen.cpp | 4 +- .../jit_avx_u8_copy_at_kern_autogen.cpp | 4 +- .../jit_avx_u8_copy_bn_kern_autogen.cpp | 4 +- .../jit_avx_u8_copy_bt_kern_autogen.cpp | 4 +- .../jit_avx_u8_copy_sum_an_kern_autogen.cpp | 4 +- .../jit_avx_u8_copy_sum_at_kern_autogen.cpp | 4 +- .../jit_avx_u8_copy_sum_bn_kern_autogen.cpp | 4 +- .../jit_avx_u8_copy_sum_bt_kern_autogen.cpp | 4 +- ..._kernel_b0_b_gemm_s8u8s32_kern_autogen.cpp | 4 +- ..._kernel_b0_c_gemm_s8u8s32_kern_autogen.cpp | 4 +- ...41_kernel_b0_gemm_s8u8s32_kern_autogen.cpp | 4 +- ..._kernel_b0_r_gemm_s8u8s32_kern_autogen.cpp | 4 +- ...e41_kernel_b_gemm_s8u8s32_kern_autogen.cpp | 4 +- ...e41_kernel_c_gemm_s8u8s32_kern_autogen.cpp | 4 +- ...sse41_kernel_gemm_s8u8s32_kern_autogen.cpp | 4 +- ...e41_kernel_r_gemm_s8u8s32_kern_autogen.cpp | 4 +- .../jit_sse41_u8_copy_an_kern_autogen.cpp | 4 +- .../jit_sse41_u8_copy_at_kern_autogen.cpp | 4 +- .../jit_sse41_u8_copy_bn_kern_autogen.cpp | 4 +- .../jit_sse41_u8_copy_bt_kern_autogen.cpp | 4 +- .../jit_sse41_u8_copy_sum_an_kern_autogen.cpp | 4 +- .../jit_sse41_u8_copy_sum_at_kern_autogen.cpp | 4 +- .../jit_sse41_u8_copy_sum_bn_kern_autogen.cpp | 4 +- .../jit_sse41_u8_copy_sum_bt_kern_autogen.cpp | 4 +- src/cpu/x64/gemm_bf16_convolution.cpp | 6 +- src/cpu/x64/gemm_bf16_convolution.hpp | 2 +- src/cpu/x64/injectors/injector_utils.cpp | 6 +- src/cpu/x64/injectors/injector_utils.hpp | 8 +- .../x64/injectors/jit_uni_binary_injector.cpp | 58 +++---- .../x64/injectors/jit_uni_binary_injector.hpp | 6 +- .../injectors/jit_uni_eltwise_injector.hpp | 28 ++-- .../injectors/jit_uni_postops_injector.cpp | 18 +-- .../injectors/jit_uni_postops_injector.hpp | 18 ++- src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.cpp | 2 +- src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.hpp | 2 +- src/cpu/x64/jit_avx2_1x1_convolution.cpp | 4 +- src/cpu/x64/jit_avx2_conv_kernel_f32.cpp | 2 +- src/cpu/x64/jit_avx2_conv_kernel_f32.hpp | 10 +- .../x64/jit_avx512_common_1x1_conv_kernel.cpp | 2 +- .../x64/jit_avx512_common_1x1_conv_kernel.hpp | 2 +- src/cpu/x64/jit_avx512_common_conv_kernel.cpp | 2 +- src/cpu/x64/jit_avx512_common_conv_kernel.hpp | 14 +- .../jit_avx512_core_amx_1x1_conv_kernel.cpp | 2 +- .../jit_avx512_core_amx_1x1_conv_kernel.hpp | 2 +- .../x64/jit_avx512_core_amx_conv_kernel.cpp | 4 +- .../x64/jit_avx512_core_amx_conv_kernel.hpp | 32 ++-- .../jit_avx512_core_bf16_1x1_conv_kernel.cpp | 2 +- .../jit_avx512_core_bf16_1x1_conv_kernel.hpp | 2 +- .../x64/jit_avx512_core_bf16_conv_kernel.cpp | 2 +- .../x64/jit_avx512_core_bf16_conv_kernel.hpp | 15 +- .../jit_avx512_core_bf16_dw_conv_kernel.cpp | 2 +- .../jit_avx512_core_bf16_dw_conv_kernel.hpp | 10 +- src/cpu/x64/jit_avx512_core_bf16cvt.hpp | 24 +-- src/cpu/x64/jit_avx512_core_fp16cvt.hpp | 8 +- src/cpu/x64/jit_avx512_core_fp8cvt.cpp | 2 +- src/cpu/x64/jit_avx512_core_fp8cvt.hpp | 12 +- src/cpu/x64/jit_avx512_core_resampling.cpp | 2 +- src/cpu/x64/jit_avx512_core_resampling.hpp | 2 +- .../x64/jit_avx512_core_scale_precompute.hpp | 6 +- ...t_avx512_core_x8s8s32x_1x1_conv_kernel.cpp | 2 +- ...t_avx512_core_x8s8s32x_1x1_conv_kernel.hpp | 4 +- .../jit_avx512_core_x8s8s32x_conv_kernel.cpp | 2 +- .../jit_avx512_core_x8s8s32x_conv_kernel.hpp | 4 +- ...jit_avx512_core_x8s8s32x_deconvolution.cpp | 2 +- ...jit_avx512_core_x8s8s32x_deconvolution.hpp | 4 +- .../jit_avx512_sparse_decompress_kernel.hpp | 8 +- src/cpu/x64/jit_brgemm_conv.hpp | 2 +- .../x64/jit_brgemm_conv_bwd_copy_kernel.cpp | 2 +- .../x64/jit_brgemm_conv_bwd_copy_kernel.hpp | 4 +- src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp | 2 +- .../x64/jit_brgemm_conv_bwd_trans_kernel.cpp | 2 +- .../x64/jit_brgemm_conv_bwd_trans_kernel.hpp | 4 +- .../x64/jit_brgemm_conv_comp_pad_kernel.cpp | 4 +- .../x64/jit_brgemm_conv_comp_pad_kernel.hpp | 4 +- src/cpu/x64/jit_brgemm_conv_trans_kernel.cpp | 2 +- src/cpu/x64/jit_brgemm_conv_trans_kernel.hpp | 4 +- src/cpu/x64/jit_brgemm_post_ops.cpp | 6 +- src/cpu/x64/jit_brgemm_post_ops.hpp | 10 +- src/cpu/x64/jit_brgemm_transpose_utils.cpp | 142 ++++++++++-------- src/cpu/x64/jit_brgemm_transpose_utils.hpp | 14 +- src/cpu/x64/jit_gemm_inner_product_utils.cpp | 10 +- ...jit_gemm_x8s8s32x_conv_zp_src_pad_comp.cpp | 16 +- ...jit_gemm_x8s8s32x_conv_zp_src_pad_comp.hpp | 8 +- .../jit_gemm_x8s8s32x_convolution_utils.cpp | 10 +- src/cpu/x64/jit_generator.cpp | 2 +- src/cpu/x64/jit_generator.hpp | 14 +- src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.cpp | 2 +- src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.hpp | 2 +- src/cpu/x64/jit_sse41_conv_kernel_f32.cpp | 2 +- src/cpu/x64/jit_sse41_conv_kernel_f32.hpp | 2 +- src/cpu/x64/jit_transpose_utils.cpp | 44 +++--- src/cpu/x64/jit_transpose_utils.hpp | 12 +- src/cpu/x64/jit_uni_1x1_conv_utils.hpp | 4 +- src/cpu/x64/jit_uni_batch_normalization.cpp | 6 +- .../x64/jit_uni_batch_normalization_s8.cpp | 4 +- src/cpu/x64/jit_uni_binary_kernel.cpp | 2 +- src/cpu/x64/jit_uni_binary_kernel.hpp | 6 +- src/cpu/x64/jit_uni_convert_xf16.cpp | 4 +- src/cpu/x64/jit_uni_convert_xf16.hpp | 12 +- .../x64/jit_uni_deconv_zp_pad_str_kernel.cpp | 6 +- .../x64/jit_uni_deconv_zp_pad_str_kernel.hpp | 6 +- src/cpu/x64/jit_uni_dw_conv_kernel_f32.cpp | 2 +- src/cpu/x64/jit_uni_dw_conv_kernel_f32.hpp | 10 +- src/cpu/x64/jit_uni_dw_conv_kernel_utils.hpp | 4 +- src/cpu/x64/jit_uni_eltwise.cpp | 6 +- src/cpu/x64/jit_uni_eltwise_int.cpp | 6 +- src/cpu/x64/jit_uni_group_normalization.cpp | 22 +-- src/cpu/x64/jit_uni_i8i8_pooling.cpp | 4 +- .../x64/jit_uni_instance_normalization.cpp | 22 +-- src/cpu/x64/jit_uni_layer_normalization.cpp | 31 ++-- src/cpu/x64/jit_uni_pool_kernel.cpp | 2 +- src/cpu/x64/jit_uni_pool_kernel.hpp | 2 +- src/cpu/x64/jit_uni_reduction_kernel.hpp | 4 +- src/cpu/x64/jit_uni_reorder.cpp | 16 +- src/cpu/x64/jit_uni_reorder_direct_copy.cpp | 10 +- src/cpu/x64/jit_uni_resampling_kernel.hpp | 4 +- src/cpu/x64/jit_uni_softmax.cpp | 20 ++- .../x64/jit_uni_tbb_batch_normalization.cpp | 40 ++--- .../x64/jit_uni_x8s8s32x_1x1_conv_kernel.cpp | 2 +- .../x64/jit_uni_x8s8s32x_1x1_conv_kernel.hpp | 6 +- src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.cpp | 2 +- src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.hpp | 4 +- .../x64/jit_uni_x8s8s32x_deconvolution.cpp | 2 +- .../x64/jit_uni_x8s8s32x_deconvolution.hpp | 4 +- src/cpu/x64/jit_uni_xf16_sum.hpp | 6 +- .../lrn/jit_avx512_common_lrn_bwd_base.cpp | 6 +- .../lrn/jit_avx512_common_lrn_bwd_base.hpp | 2 +- .../lrn/jit_avx512_common_lrn_fwd_base.cpp | 6 +- .../lrn/jit_avx512_common_lrn_fwd_base.hpp | 2 +- src/cpu/x64/lrn/jit_uni_lrn_kernel.cpp | 12 +- src/cpu/x64/lrn/jit_uni_lrn_kernel.hpp | 2 +- .../x64/matmul/brgemm_matmul_copy_utils.cpp | 126 +++++++++------- src/cpu/x64/matmul/jit_uni_sparse_matmul.cpp | 6 +- src/cpu/x64/prelu/jit_prelu_base_kernel.cpp | 4 +- src/cpu/x64/prelu/jit_prelu_base_kernel.hpp | 4 +- .../x64/prelu/jit_prelu_reduction_kernel.cpp | 2 +- .../x64/prelu/jit_prelu_reduction_kernel.hpp | 6 +- src/cpu/x64/prelu/jit_prelu_utils.cpp | 2 +- src/cpu/x64/prelu/jit_prelu_utils.hpp | 4 +- .../prelu/jit_uni_prelu_backward_kernel.cpp | 6 +- .../prelu/jit_uni_prelu_backward_kernel.hpp | 4 +- .../prelu/jit_uni_prelu_forward_kernel.hpp | 6 +- .../rnn/jit_brgemm_transpose_single_row.cpp | 4 +- .../rnn/jit_brgemm_transpose_single_row.hpp | 6 +- src/cpu/x64/rnn/jit_diff_weights_peephole.cpp | 4 +- src/cpu/x64/rnn/jit_diff_weights_peephole.hpp | 6 +- src/cpu/x64/rnn/jit_gates_reduction.cpp | 2 +- src/cpu/x64/rnn/jit_gates_reduction.hpp | 6 +- .../x64/rnn/jit_uni_lstm_cell_postgemm.hpp | 4 +- .../x64/rnn/jit_uni_rnn_common_postgemm.hpp | 4 +- .../x64/shuffle/jit_uni_shuffle_kernel.cpp | 4 +- .../x64/shuffle/jit_uni_shuffle_kernel.hpp | 2 +- src/cpu/x64/utils/jit_io_helper.cpp | 13 +- src/cpu/x64/utils/jit_io_helper.hpp | 6 +- 250 files changed, 967 insertions(+), 889 deletions(-) diff --git a/src/cpu/x64/amx_tile_configure.cpp b/src/cpu/x64/amx_tile_configure.cpp index 410ea05bc91..9464c604617 100644 --- a/src/cpu/x64/amx_tile_configure.cpp +++ b/src/cpu/x64/amx_tile_configure.cpp @@ -22,12 +22,12 @@ namespace impl { namespace cpu { namespace x64 { -struct jit_amx_tilecfg_t : public jit_generator { +struct jit_amx_tilecfg_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_amx_tilecfg_t) // TODO: Need to check status jit_amx_tilecfg_t(bool lazy = false) - : jit_generator(jit_name(), avx512_core_amx), is_lazy_(lazy) { + : jit_generator_t(jit_name(), avx512_core_amx), is_lazy_(lazy) { create_kernel(); } @@ -72,11 +72,11 @@ struct jit_amx_tilecfg_t : public jit_generator { } }; -struct jit_amx_tilerelease_t : public jit_generator { +struct jit_amx_tilerelease_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_amx_tilerelease_t) // TODO: Need to check status - jit_amx_tilerelease_t() : jit_generator(jit_name(), avx512_core_amx) { + jit_amx_tilerelease_t() : jit_generator_t(jit_name(), avx512_core_amx) { create_kernel(); } diff --git a/src/cpu/x64/brgemm/brgemm_types.hpp b/src/cpu/x64/brgemm/brgemm_types.hpp index 809b541e52d..2d252230706 100644 --- a/src/cpu/x64/brgemm/brgemm_types.hpp +++ b/src/cpu/x64/brgemm/brgemm_types.hpp @@ -569,20 +569,20 @@ struct jit_brgemm_kernel_t; struct jit_brgemm_amx_uker_base_t; template struct jit_brdgmm_kernel_base_t; -class jit_generator; +class jit_generator_t; struct brgemm_kernel_t { brgemm_kernel_t() = default; virtual ~brgemm_kernel_t() = default; virtual status_t create_kernel() = 0; virtual void operator()(brgemm_kernel_params_t *) const = 0; - virtual const jit_generator *get_jit_generator() const = 0; + virtual const jit_generator_t *get_jit_generator() const = 0; virtual const brgemm_desc_t &get_brg() const = 0; }; -struct jit_base_brgemm_kernel_t : public jit_generator { +struct jit_base_brgemm_kernel_t : public jit_generator_t { jit_base_brgemm_kernel_t(const char *impl_name, cpu_isa_t isa_impl) - : jit_generator(impl_name, isa_impl) {} + : jit_generator_t(impl_name, isa_impl) {} virtual const brgemm_desc_t &get_brg() const = 0; }; @@ -593,7 +593,7 @@ struct brgemm_kernel_common_t : public brgemm_kernel_t { status_t create_kernel() override; void operator()(brgemm_kernel_params_t *) const override; - const jit_generator *get_jit_generator() const override; + const jit_generator_t *get_jit_generator() const override; const brgemm_desc_t &get_brg() const override { return ((jit_base_brgemm_kernel_t *)brgemm_kernel_)->get_brg(); } @@ -610,7 +610,7 @@ struct brgemm_amx_uker_t : public brgemm_kernel_t { status_t create_kernel() override; void operator()(brgemm_kernel_params_t *) const override; - const jit_generator *get_jit_generator() const override; + const jit_generator_t *get_jit_generator() const override; const brgemm_desc_t &get_brg() const override { return ((jit_base_brgemm_kernel_t *)brgemm_kernel_)->get_brg(); } @@ -628,7 +628,7 @@ struct brdgmm_kernel_t : public brgemm_kernel_t { status_t create_kernel() override; void operator()(brgemm_kernel_params_t *) const override; - const jit_generator *get_jit_generator() const override; + const jit_generator_t *get_jit_generator() const override; const brgemm_desc_t &get_brg() const override { return ((jit_base_brgemm_kernel_t *)brgemm_kernel_)->get_brg(); } diff --git a/src/cpu/x64/brgemm/jit_brdgmm_kernel.cpp b/src/cpu/x64/brgemm/jit_brdgmm_kernel.cpp index f94533ce98f..7a8aaf66445 100644 --- a/src/cpu/x64/brgemm/jit_brdgmm_kernel.cpp +++ b/src/cpu/x64/brgemm/jit_brdgmm_kernel.cpp @@ -1439,7 +1439,7 @@ void brdgmm_kernel_t::operator()(brgemm_kernel_params_t *params) const { } template -const jit_generator *brdgmm_kernel_t::get_jit_generator() const { +const jit_generator_t *brdgmm_kernel_t::get_jit_generator() const { return brgemm_kernel_; } diff --git a/src/cpu/x64/brgemm/jit_brgemm_amx_uker.cpp b/src/cpu/x64/brgemm/jit_brgemm_amx_uker.cpp index f270c82d347..a8641db38ed 100644 --- a/src/cpu/x64/brgemm/jit_brgemm_amx_uker.cpp +++ b/src/cpu/x64/brgemm/jit_brgemm_amx_uker.cpp @@ -2747,7 +2747,7 @@ void brgemm_amx_uker_t::operator()(brgemm_kernel_params_t *params) const { (*brgemm_kernel_)(params); } -const jit_generator *brgemm_amx_uker_t::get_jit_generator() const { +const jit_generator_t *brgemm_amx_uker_t::get_jit_generator() const { return brgemm_kernel_; } diff --git a/src/cpu/x64/brgemm/jit_brgemm_kernel.cpp b/src/cpu/x64/brgemm/jit_brgemm_kernel.cpp index 21ea0f25533..79eb9f0e40c 100644 --- a/src/cpu/x64/brgemm/jit_brgemm_kernel.cpp +++ b/src/cpu/x64/brgemm/jit_brgemm_kernel.cpp @@ -2948,7 +2948,7 @@ void brgemm_kernel_common_t::operator()( } template -const jit_generator *brgemm_kernel_common_t::get_jit_generator() const { +const jit_generator_t *brgemm_kernel_common_t::get_jit_generator() const { return brgemm_kernel_; } diff --git a/src/cpu/x64/cpu_barrier.cpp b/src/cpu/x64/cpu_barrier.cpp index 24ab6515b02..2ab3bb5c4a5 100644 --- a/src/cpu/x64/cpu_barrier.cpp +++ b/src/cpu/x64/cpu_barrier.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2017-2022 Intel Corporation +* Copyright 2017-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ namespace x64 { namespace simple_barrier { void generate( - jit_generator &code, Xbyak::Reg64 reg_ctx, Xbyak::Reg64 reg_nthr) { + jit_generator_t &code, Xbyak::Reg64 reg_ctx, Xbyak::Reg64 reg_nthr) { #define BAR_CTR_OFF offsetof(ctx_t, ctr) #define BAR_SENSE_OFF offsetof(ctx_t, sense) using namespace Xbyak; @@ -81,7 +81,7 @@ void generate( } /** jit barrier generator */ -struct jit_t : public jit_generator { +struct jit_t : public jit_generator_t { void generate() override { simple_barrier::generate(*this, abi_param1, abi_param2); @@ -89,7 +89,7 @@ struct jit_t : public jit_generator { } // TODO: Need to check status - jit_t() : jit_generator(jit_name()) { create_kernel(); } + jit_t() : jit_generator_t(jit_name()) { create_kernel(); } DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_t) }; diff --git a/src/cpu/x64/cpu_barrier.hpp b/src/cpu/x64/cpu_barrier.hpp index c76d57911af..f5cd7966ac9 100644 --- a/src/cpu/x64/cpu_barrier.hpp +++ b/src/cpu/x64/cpu_barrier.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2017-2020 Intel Corporation +* Copyright 2017-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -68,11 +68,12 @@ void barrier(ctx_t *ctx, int nthr); /** injects actual barrier implementation into another jitted code * @params: - * code -- jit_generator object where the barrier is to be injected + * code -- jit_generator_t object where the barrier is to be injected * reg_ctx -- read-only register with pointer to the barrier context * reg_nnthr -- read-only register with the # of synchronizing threads */ -void generate(jit_generator &code, Xbyak::Reg64 reg_ctx, Xbyak::Reg64 reg_nthr); +void generate( + jit_generator_t &code, Xbyak::Reg64 reg_ctx, Xbyak::Reg64 reg_nthr); } // namespace simple_barrier diff --git a/src/cpu/x64/cpu_reducer.cpp b/src/cpu/x64/cpu_reducer.cpp index 28beab84a9e..86397f0881c 100644 --- a/src/cpu/x64/cpu_reducer.cpp +++ b/src/cpu/x64/cpu_reducer.cpp @@ -97,12 +97,12 @@ void reduce_balancer_t::balance() { using namespace Xbyak; template -struct reducer_2d_driver_t : public jit_generator { +struct reducer_2d_driver_t : public jit_generator_t { using data_t = typename prec_traits_t::type; reducer_2d_driver_t(int n_src, size_t src_ld, size_t src_step, size_t dst_step, bool nullify_dst, const char *name) - : jit_generator(name) + : jit_generator_t(name) , n_src_(n_src) , src_ld_(src_ld) , src_step_(src_step) @@ -126,7 +126,7 @@ struct reducer_2d_driver_f_s_32_t : public reducer_2d_driver_t { void operator()( data_t *dst, const data_t *srcs, size_t ny, size_t nx) override { - jit_generator::operator()(dst, srcs, ny, nx); + jit_generator_t::operator()(dst, srcs, ny, nx); } /* cpu specific part */ diff --git a/src/cpu/x64/gemm/amx/jit_avx512_core_amx_copy_kern.cpp b/src/cpu/x64/gemm/amx/jit_avx512_core_amx_copy_kern.cpp index 7f9b09824d2..31f2f355419 100644 --- a/src/cpu/x64/gemm/amx/jit_avx512_core_amx_copy_kern.cpp +++ b/src/cpu/x64/gemm/amx/jit_avx512_core_amx_copy_kern.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -496,7 +496,7 @@ void jit_avx512_core_amx_copy_kern::generate() { jit_avx512_core_amx_copy_kern::jit_avx512_core_amx_copy_kern( bool is_a, bool is_trans, int isize) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , is_a_(is_a) , is_trans_(is_trans) , size_(isize) diff --git a/src/cpu/x64/gemm/amx/jit_avx512_core_amx_copy_kern.hpp b/src/cpu/x64/gemm/amx/jit_avx512_core_amx_copy_kern.hpp index 1e51de69e43..94ddade59c4 100644 --- a/src/cpu/x64/gemm/amx/jit_avx512_core_amx_copy_kern.hpp +++ b/src/cpu/x64/gemm/amx/jit_avx512_core_amx_copy_kern.hpp @@ -24,7 +24,7 @@ namespace impl { namespace cpu { namespace x64 { -class jit_avx512_core_amx_copy_kern : public jit_generator { +class jit_avx512_core_amx_copy_kern : public jit_generator_t { public: jit_avx512_core_amx_copy_kern(bool is_a, bool is_trans, int isize); DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_amx_copy_kern); diff --git a/src/cpu/x64/gemm/amx/jit_avx512_core_amx_gemm_kern.cpp b/src/cpu/x64/gemm/amx/jit_avx512_core_amx_gemm_kern.cpp index f9005d6ea6e..21a5c601a70 100644 --- a/src/cpu/x64/gemm/amx/jit_avx512_core_amx_gemm_kern.cpp +++ b/src/cpu/x64/gemm/amx/jit_avx512_core_amx_gemm_kern.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -457,7 +457,7 @@ void jit_avx512_core_amx_gemm_kern::generate() { jit_avx512_core_amx_gemm_kern::jit_avx512_core_amx_gemm_kern( int typea, int typeb, int typec, int betaZero) - : jit_generator(jit_name(), avx512_core_amx) + : jit_generator_t(jit_name(), avx512_core_amx) , typea(typea) , typeb(typeb) , typec(typec) diff --git a/src/cpu/x64/gemm/amx/jit_avx512_core_amx_gemm_kern.hpp b/src/cpu/x64/gemm/amx/jit_avx512_core_amx_gemm_kern.hpp index 5df4ac140c9..7ab92fa1a95 100644 --- a/src/cpu/x64/gemm/amx/jit_avx512_core_amx_gemm_kern.hpp +++ b/src/cpu/x64/gemm/amx/jit_avx512_core_amx_gemm_kern.hpp @@ -24,7 +24,7 @@ namespace impl { namespace cpu { namespace x64 { -class jit_avx512_core_amx_gemm_kern : public jit_generator { +class jit_avx512_core_amx_gemm_kern : public jit_generator_t { public: jit_avx512_core_amx_gemm_kern( int typea, int typeb, int typec, int betaZero); diff --git a/src/cpu/x64/gemm/bf16/common_s16.hpp b/src/cpu/x64/gemm/bf16/common_s16.hpp index 5a9d221b48f..b7fdc60d2e3 100644 --- a/src/cpu/x64/gemm/bf16/common_s16.hpp +++ b/src/cpu/x64/gemm/bf16/common_s16.hpp @@ -24,7 +24,7 @@ namespace impl { namespace cpu { namespace x64 { -class jit_avx512_core_s16_48x8_copy_an_kern : public jit_generator { +class jit_avx512_core_s16_48x8_copy_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_s16_48x8_copy_an_kern); void generate() override; @@ -32,7 +32,7 @@ class jit_avx512_core_s16_48x8_copy_an_kern : public jit_generator { jit_avx512_core_s16_48x8_copy_an_kern(); }; -class jit_avx512_core_s16_48x8_copy_at_kern : public jit_generator { +class jit_avx512_core_s16_48x8_copy_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_s16_48x8_copy_at_kern); void generate() override; @@ -40,7 +40,7 @@ class jit_avx512_core_s16_48x8_copy_at_kern : public jit_generator { jit_avx512_core_s16_48x8_copy_at_kern(); }; -class jit_avx512_core_s16_48x8_copy_bn_kern : public jit_generator { +class jit_avx512_core_s16_48x8_copy_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_s16_48x8_copy_bn_kern); void generate() override; @@ -48,7 +48,7 @@ class jit_avx512_core_s16_48x8_copy_bn_kern : public jit_generator { jit_avx512_core_s16_48x8_copy_bn_kern(); }; -class jit_avx512_core_s16_48x8_copy_bt_kern : public jit_generator { +class jit_avx512_core_s16_48x8_copy_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_s16_48x8_copy_bt_kern); void generate() override; @@ -56,7 +56,7 @@ class jit_avx512_core_s16_48x8_copy_bt_kern : public jit_generator { jit_avx512_core_s16_48x8_copy_bt_kern(); }; -class jit_avx512_core_s16_24x8_copy_an_kern : public jit_generator { +class jit_avx512_core_s16_24x8_copy_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_s16_24x8_copy_an_kern); void generate() override; @@ -64,7 +64,7 @@ class jit_avx512_core_s16_24x8_copy_an_kern : public jit_generator { jit_avx512_core_s16_24x8_copy_an_kern(); }; -class jit_avx512_core_s16_24x8_copy_at_kern : public jit_generator { +class jit_avx512_core_s16_24x8_copy_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_s16_24x8_copy_at_kern); void generate() override; @@ -72,7 +72,7 @@ class jit_avx512_core_s16_24x8_copy_at_kern : public jit_generator { jit_avx512_core_s16_24x8_copy_at_kern(); }; -class jit_avx512_core_s16_24x8_copy_bn_kern : public jit_generator { +class jit_avx512_core_s16_24x8_copy_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_s16_24x8_copy_bn_kern); void generate() override; @@ -80,7 +80,7 @@ class jit_avx512_core_s16_24x8_copy_bn_kern : public jit_generator { jit_avx512_core_s16_24x8_copy_bn_kern(); }; -class jit_avx512_core_s16_24x8_copy_bt_kern : public jit_generator { +class jit_avx512_core_s16_24x8_copy_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_s16_24x8_copy_bt_kern); void generate() override; diff --git a/src/cpu/x64/gemm/bf16/jit_avx512_core_gemm_bf16bf16f32_kern.cpp b/src/cpu/x64/gemm/bf16/jit_avx512_core_gemm_bf16bf16f32_kern.cpp index 069dae9d902..099fbe0c24a 100644 --- a/src/cpu/x64/gemm/bf16/jit_avx512_core_gemm_bf16bf16f32_kern.cpp +++ b/src/cpu/x64/gemm/bf16/jit_avx512_core_gemm_bf16bf16f32_kern.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -425,7 +425,7 @@ void jit_avx512_core_gemm_bf16bf16f32_kern::generate() { jit_avx512_core_gemm_bf16bf16f32_kern::jit_avx512_core_gemm_bf16bf16f32_kern( bool beta_zero, bool alpha_one, bool use_zmm) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , beta_zero_(beta_zero) , alpha_one_(alpha_one) , bfloat16_(mayiuse(avx512_core_bf16)) diff --git a/src/cpu/x64/gemm/bf16/jit_avx512_core_gemm_bf16bf16f32_kern.hpp b/src/cpu/x64/gemm/bf16/jit_avx512_core_gemm_bf16bf16f32_kern.hpp index 9490a0c6e89..1c3872af1a2 100644 --- a/src/cpu/x64/gemm/bf16/jit_avx512_core_gemm_bf16bf16f32_kern.hpp +++ b/src/cpu/x64/gemm/bf16/jit_avx512_core_gemm_bf16bf16f32_kern.hpp @@ -25,7 +25,7 @@ namespace impl { namespace cpu { namespace x64 { -class jit_avx512_core_gemm_bf16bf16f32_kern : public jit_generator { +class jit_avx512_core_gemm_bf16bf16f32_kern : public jit_generator_t { public: jit_avx512_core_gemm_bf16bf16f32_kern( bool beta_zero, bool alpha_one, bool use_zmm); diff --git a/src/cpu/x64/gemm/bf16/jit_avx512_core_gemv_bf16bf16f32_kern.cpp b/src/cpu/x64/gemm/bf16/jit_avx512_core_gemv_bf16bf16f32_kern.cpp index 4d77805f9ff..7ca21048642 100644 --- a/src/cpu/x64/gemm/bf16/jit_avx512_core_gemv_bf16bf16f32_kern.cpp +++ b/src/cpu/x64/gemm/bf16/jit_avx512_core_gemv_bf16bf16f32_kern.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -515,7 +515,7 @@ void jit_avx512_core_gemv_bf16bf16f32_kern::generate() { // Function signature: gemv(*m, *n, *alpha, *a, *lda, *x, *incx, *y, *incy) jit_avx512_core_gemv_bf16bf16f32_kern::jit_avx512_core_gemv_bf16bf16f32_kern( bool trans) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , trans_(trans) , bfloat16_(mayiuse(avx512_core_bf16)) , arg_lda_(0) diff --git a/src/cpu/x64/gemm/bf16/jit_avx512_core_gemv_bf16bf16f32_kern.hpp b/src/cpu/x64/gemm/bf16/jit_avx512_core_gemv_bf16bf16f32_kern.hpp index a375172924c..c3fef6ab778 100644 --- a/src/cpu/x64/gemm/bf16/jit_avx512_core_gemv_bf16bf16f32_kern.hpp +++ b/src/cpu/x64/gemm/bf16/jit_avx512_core_gemv_bf16bf16f32_kern.hpp @@ -25,7 +25,7 @@ namespace impl { namespace cpu { namespace x64 { -class jit_avx512_core_gemv_bf16bf16f32_kern : public jit_generator { +class jit_avx512_core_gemv_bf16bf16f32_kern : public jit_generator_t { public: jit_avx512_core_gemv_bf16bf16f32_kern(bool trans); ~jit_avx512_core_gemv_bf16bf16f32_kern() override; diff --git a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_an_kern_autogen.cpp b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_an_kern_autogen.cpp index 22f089dc8b0..6fd9705397a 100644 --- a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_s16_24x8_copy_an_kern::jit_avx512_core_s16_24x8_copy_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_s16_24x8_copy_an_kern::generate() { diff --git a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_at_kern_autogen.cpp b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_at_kern_autogen.cpp index 9a6032745f7..9d4eecd8494 100644 --- a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_s16_24x8_copy_at_kern::jit_avx512_core_s16_24x8_copy_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_s16_24x8_copy_at_kern::generate() { diff --git a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_bn_kern_autogen.cpp b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_bn_kern_autogen.cpp index be61df11e29..943df25a67d 100644 --- a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_s16_24x8_copy_bn_kern::jit_avx512_core_s16_24x8_copy_bn_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_s16_24x8_copy_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_bt_kern_autogen.cpp b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_bt_kern_autogen.cpp index cd62ed88dbd..95c1fb60e0f 100644 --- a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_24x8_copy_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_s16_24x8_copy_bt_kern::jit_avx512_core_s16_24x8_copy_bt_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_s16_24x8_copy_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_an_kern_autogen.cpp b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_an_kern_autogen.cpp index 3a936e6a280..d1983c0e54e 100644 --- a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_s16_48x8_copy_an_kern::jit_avx512_core_s16_48x8_copy_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_s16_48x8_copy_an_kern::generate() { diff --git a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_at_kern_autogen.cpp b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_at_kern_autogen.cpp index ced7abdd837..26b573de3ab 100644 --- a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_s16_48x8_copy_at_kern::jit_avx512_core_s16_48x8_copy_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_s16_48x8_copy_at_kern::generate() { diff --git a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_bn_kern_autogen.cpp b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_bn_kern_autogen.cpp index 196039ad816..eaf588d2385 100644 --- a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_s16_48x8_copy_bn_kern::jit_avx512_core_s16_48x8_copy_bn_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_s16_48x8_copy_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_bt_kern_autogen.cpp b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_bt_kern_autogen.cpp index d448a2e121a..807fad124b6 100644 --- a/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/bf16/jit_avx512_core_s16_48x8_copy_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_s16_48x8_copy_bt_kern::jit_avx512_core_s16_48x8_copy_bt_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_s16_48x8_copy_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/common_f32.hpp b/src/cpu/x64/gemm/f32/common_f32.hpp index 03e07360f89..8b44f62f669 100644 --- a/src/cpu/x64/gemm/f32/common_f32.hpp +++ b/src/cpu/x64/gemm/f32/common_f32.hpp @@ -24,7 +24,7 @@ namespace impl { namespace cpu { namespace x64 { -class jit_avx512_core_f32_copy_an_kern : public jit_generator { +class jit_avx512_core_f32_copy_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_f32_copy_an_kern); void generate() override; @@ -32,7 +32,7 @@ class jit_avx512_core_f32_copy_an_kern : public jit_generator { jit_avx512_core_f32_copy_an_kern(); }; -class jit_avx512_core_f32_copy_at_kern : public jit_generator { +class jit_avx512_core_f32_copy_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_f32_copy_at_kern); void generate() override; void generate_part1(const Xbyak::Label &, const Xbyak::Label &, @@ -43,7 +43,7 @@ class jit_avx512_core_f32_copy_at_kern : public jit_generator { jit_avx512_core_f32_copy_at_kern(); }; -class jit_avx512_core_f32_copy_bn_kern : public jit_generator { +class jit_avx512_core_f32_copy_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_f32_copy_bn_kern); void generate() override; @@ -51,7 +51,7 @@ class jit_avx512_core_f32_copy_bn_kern : public jit_generator { jit_avx512_core_f32_copy_bn_kern(); }; -class jit_avx512_core_f32_copy_bt_kern : public jit_generator { +class jit_avx512_core_f32_copy_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_f32_copy_bt_kern); void generate() override; @@ -59,7 +59,7 @@ class jit_avx512_core_f32_copy_bt_kern : public jit_generator { jit_avx512_core_f32_copy_bt_kern(); }; -class jit_avx2_f32_copy_an_kern : public jit_generator { +class jit_avx2_f32_copy_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_f32_copy_an_kern); void generate() override; @@ -67,7 +67,7 @@ class jit_avx2_f32_copy_an_kern : public jit_generator { jit_avx2_f32_copy_an_kern(); }; -class jit_avx2_f32_copy_at_kern : public jit_generator { +class jit_avx2_f32_copy_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_f32_copy_at_kern); void generate() override; @@ -75,7 +75,7 @@ class jit_avx2_f32_copy_at_kern : public jit_generator { jit_avx2_f32_copy_at_kern(); }; -class jit_avx2_f32_copy_bn_kern : public jit_generator { +class jit_avx2_f32_copy_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_f32_copy_bn_kern); void generate() override; @@ -83,7 +83,7 @@ class jit_avx2_f32_copy_bn_kern : public jit_generator { jit_avx2_f32_copy_bn_kern(); }; -class jit_avx2_f32_copy_bt_kern : public jit_generator { +class jit_avx2_f32_copy_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_f32_copy_bt_kern); void generate() override; @@ -91,7 +91,7 @@ class jit_avx2_f32_copy_bt_kern : public jit_generator { jit_avx2_f32_copy_bt_kern(); }; -class jit_avx_f32_copy_an_kern : public jit_generator { +class jit_avx_f32_copy_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_f32_copy_an_kern); void generate() override; @@ -99,7 +99,7 @@ class jit_avx_f32_copy_an_kern : public jit_generator { jit_avx_f32_copy_an_kern(); }; -class jit_avx_f32_copy_at_kern : public jit_generator { +class jit_avx_f32_copy_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_f32_copy_at_kern); void generate() override; @@ -107,7 +107,7 @@ class jit_avx_f32_copy_at_kern : public jit_generator { jit_avx_f32_copy_at_kern(); }; -class jit_avx_f32_copy_bn_kern : public jit_generator { +class jit_avx_f32_copy_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_f32_copy_bn_kern); void generate() override; @@ -115,7 +115,7 @@ class jit_avx_f32_copy_bn_kern : public jit_generator { jit_avx_f32_copy_bn_kern(); }; -class jit_avx_f32_copy_bt_kern : public jit_generator { +class jit_avx_f32_copy_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_f32_copy_bt_kern); void generate() override; @@ -123,7 +123,7 @@ class jit_avx_f32_copy_bt_kern : public jit_generator { jit_avx_f32_copy_bt_kern(); }; -class jit_avx_kernel_b0_sgemm_kern : public jit_generator { +class jit_avx_kernel_b0_sgemm_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_kernel_b0_sgemm_kern); void generate() override; void generate_part1(const Xbyak::Label &, const Xbyak::Label &, @@ -134,7 +134,7 @@ class jit_avx_kernel_b0_sgemm_kern : public jit_generator { jit_avx_kernel_b0_sgemm_kern(); }; -class jit_avx_kernel_sgemm_kern : public jit_generator { +class jit_avx_kernel_sgemm_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_kernel_sgemm_kern); void generate() override; void generate_part1( @@ -145,7 +145,7 @@ class jit_avx_kernel_sgemm_kern : public jit_generator { jit_avx_kernel_sgemm_kern(); }; -class jit_sse41_f32_copy_an_kern : public jit_generator { +class jit_sse41_f32_copy_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_f32_copy_an_kern); void generate() override; @@ -153,7 +153,7 @@ class jit_sse41_f32_copy_an_kern : public jit_generator { jit_sse41_f32_copy_an_kern(); }; -class jit_sse41_f32_copy_at_kern : public jit_generator { +class jit_sse41_f32_copy_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_f32_copy_at_kern); void generate() override; @@ -161,7 +161,7 @@ class jit_sse41_f32_copy_at_kern : public jit_generator { jit_sse41_f32_copy_at_kern(); }; -class jit_sse41_f32_copy_bn_kern : public jit_generator { +class jit_sse41_f32_copy_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_f32_copy_bn_kern); void generate() override; @@ -169,7 +169,7 @@ class jit_sse41_f32_copy_bn_kern : public jit_generator { jit_sse41_f32_copy_bn_kern(); }; -class jit_sse41_f32_copy_bt_kern : public jit_generator { +class jit_sse41_f32_copy_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_f32_copy_bt_kern); void generate() override; @@ -177,7 +177,7 @@ class jit_sse41_f32_copy_bt_kern : public jit_generator { jit_sse41_f32_copy_bt_kern(); }; -class jit_sse41_kernel_b0_sgemm_kern : public jit_generator { +class jit_sse41_kernel_b0_sgemm_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_kernel_b0_sgemm_kern); void generate() override; @@ -185,7 +185,7 @@ class jit_sse41_kernel_b0_sgemm_kern : public jit_generator { jit_sse41_kernel_b0_sgemm_kern(); }; -class jit_sse41_kernel_sgemm_kern : public jit_generator { +class jit_sse41_kernel_sgemm_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_kernel_sgemm_kern); void generate() override; diff --git a/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_an_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_an_kern_autogen.cpp index 3b14fe68440..24ddf199bb8 100644 --- a/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_f32_copy_an_kern::jit_avx2_f32_copy_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_f32_copy_an_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_at_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_at_kern_autogen.cpp index 8f9205dfca5..4c8a2db1152 100644 --- a/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_f32_copy_at_kern::jit_avx2_f32_copy_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_f32_copy_at_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_bn_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_bn_kern_autogen.cpp index 1b086a5e4de..aede2f0ff7d 100644 --- a/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_f32_copy_bn_kern::jit_avx2_f32_copy_bn_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_f32_copy_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_bt_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_bt_kern_autogen.cpp index 9fd7218234b..4d82f62c52a 100644 --- a/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx2_f32_copy_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_f32_copy_bt_kern::jit_avx2_f32_copy_bt_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_f32_copy_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_avx2_kernel_sgemm_kern.cpp b/src/cpu/x64/gemm/f32/jit_avx2_kernel_sgemm_kern.cpp index d0fb52fa6c3..eef9c7acba3 100644 --- a/src/cpu/x64/gemm/f32/jit_avx2_kernel_sgemm_kern.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx2_kernel_sgemm_kern.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -436,7 +436,7 @@ void jit_avx2_kernel_sgemm_kern::generate() { } jit_avx2_kernel_sgemm_kern::jit_avx2_kernel_sgemm_kern(bool beta_zero) - : jit_generator(jit_name()), beta_zero_(beta_zero) {} + : jit_generator_t(jit_name()), beta_zero_(beta_zero) {} } // namespace x64 } // namespace cpu diff --git a/src/cpu/x64/gemm/f32/jit_avx2_kernel_sgemm_kern.hpp b/src/cpu/x64/gemm/f32/jit_avx2_kernel_sgemm_kern.hpp index b18b34686d5..ab26922fd62 100644 --- a/src/cpu/x64/gemm/f32/jit_avx2_kernel_sgemm_kern.hpp +++ b/src/cpu/x64/gemm/f32/jit_avx2_kernel_sgemm_kern.hpp @@ -29,7 +29,7 @@ namespace impl { namespace cpu { namespace x64 { -class jit_avx2_kernel_sgemm_kern : public jit_generator { +class jit_avx2_kernel_sgemm_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_kernel_sgemm_kern); const int elt_size_ = 4; diff --git a/src/cpu/x64/gemm/f32/jit_avx512_common_gemm_f32.cpp b/src/cpu/x64/gemm/f32/jit_avx512_common_gemm_f32.cpp index 7fd8e81bdb4..85e9e4aec69 100644 --- a/src/cpu/x64/gemm/f32/jit_avx512_common_gemm_f32.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx512_common_gemm_f32.cpp @@ -59,11 +59,11 @@ namespace x64 { namespace avx512_common_gemm_f32 { using namespace gemm_utils; -struct xbyak_gemm_t : public jit_generator { +struct xbyak_gemm_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_common_gemm_f32_xbyak_gemm) xbyak_gemm_t(char isTransA, char isTransB, float beta, bool hasBias = false) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , isTransA(isTransA) , isTransB(isTransB) , beta(beta) diff --git a/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_an_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_an_kern_autogen.cpp index bca29715498..3c9adacbd3c 100644 --- a/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_f32_copy_an_kern::jit_avx512_core_f32_copy_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_f32_copy_an_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_at_kern_part1_autogen.cpp b/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_at_kern_part1_autogen.cpp index 63bb212c563..73726efc52f 100644 --- a/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_at_kern_part1_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_at_kern_part1_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ namespace cpu { namespace x64 { jit_avx512_core_f32_copy_at_kern::jit_avx512_core_f32_copy_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_f32_copy_at_kern::generate() { Xbyak::Label l1f80; diff --git a/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_bn_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_bn_kern_autogen.cpp index c49dbb2f743..00ce423f2b3 100644 --- a/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_f32_copy_bn_kern::jit_avx512_core_f32_copy_bn_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_f32_copy_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_bt_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_bt_kern_autogen.cpp index 24d3145349f..a60c2e7a08c 100644 --- a/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx512_core_f32_copy_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_f32_copy_bt_kern::jit_avx512_core_f32_copy_bt_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_f32_copy_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_avx512_core_gemm_smalln_tn_f32_kern.cpp b/src/cpu/x64/gemm/f32/jit_avx512_core_gemm_smalln_tn_f32_kern.cpp index 8f1f9e299c3..3787430d2bf 100644 --- a/src/cpu/x64/gemm/f32/jit_avx512_core_gemm_smalln_tn_f32_kern.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx512_core_gemm_smalln_tn_f32_kern.cpp @@ -43,11 +43,11 @@ static inline Xbyak::Ymm make_ymm(const Xbyak::Zmm &v) { namespace avx512_core_gemm_smalln_tn_f32 { -struct xbyak_gemm_smalln_tn_t : public jit_generator { +struct xbyak_gemm_smalln_tn_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_gemm_smalln_tn_xbyak_gemm) xbyak_gemm_smalln_tn_t(int N, float beta, float alpha) - : jit_generator(jit_name()), N(N), beta(beta), alpha(alpha) {} + : jit_generator_t(jit_name()), N(N), beta(beta), alpha(alpha) {} void generate() override { using namespace Xbyak; diff --git a/src/cpu/x64/gemm/f32/jit_avx_f32_copy_an_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_avx_f32_copy_an_kern_autogen.cpp index 117de225946..0a9face3aff 100644 --- a/src/cpu/x64/gemm/f32/jit_avx_f32_copy_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx_f32_copy_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_f32_copy_an_kern::jit_avx_f32_copy_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_f32_copy_an_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_avx_f32_copy_at_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_avx_f32_copy_at_kern_autogen.cpp index 20e8c67d6be..95010bc3ff0 100644 --- a/src/cpu/x64/gemm/f32/jit_avx_f32_copy_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx_f32_copy_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_f32_copy_at_kern::jit_avx_f32_copy_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_f32_copy_at_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_avx_f32_copy_bn_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_avx_f32_copy_bn_kern_autogen.cpp index 277144c5fbd..985dedc5ef2 100644 --- a/src/cpu/x64/gemm/f32/jit_avx_f32_copy_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx_f32_copy_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_f32_copy_bn_kern::jit_avx_f32_copy_bn_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_f32_copy_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_avx_f32_copy_bt_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_avx_f32_copy_bt_kern_autogen.cpp index a7d9fe4fa04..ac3877bcdf3 100644 --- a/src/cpu/x64/gemm/f32/jit_avx_f32_copy_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx_f32_copy_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_f32_copy_bt_kern::jit_avx_f32_copy_bt_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_f32_copy_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_avx_gemm_f32.cpp b/src/cpu/x64/gemm/f32/jit_avx_gemm_f32.cpp index 8fe514cc0d0..38a01ce662e 100644 --- a/src/cpu/x64/gemm/f32/jit_avx_gemm_f32.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx_gemm_f32.cpp @@ -58,11 +58,11 @@ namespace avx_gemm_f32 { using namespace gemm_utils; using namespace Xbyak; -struct xbyak_gemm_t : public jit_generator { +struct xbyak_gemm_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_gemm_f32_xbyak_gemm) xbyak_gemm_t(char isTransA, char isTransB, float beta, bool hasBias = false) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , isTransA(isTransA) , isTransB(isTransB) , hasBias(hasBias) diff --git a/src/cpu/x64/gemm/f32/jit_avx_gemv_t_f32_kern.cpp b/src/cpu/x64/gemm/f32/jit_avx_gemv_t_f32_kern.cpp index d85f65fb581..c0e1d1adfad 100644 --- a/src/cpu/x64/gemm/f32/jit_avx_gemv_t_f32_kern.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx_gemv_t_f32_kern.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -302,7 +302,7 @@ void jit_avx_gemv_t_f32_kern::generate() { // Function signature: gemv(*m, *n, *alpha, *a, *lda, *x, *incx, *y, *incy) jit_avx_gemv_t_f32_kern::jit_avx_gemv_t_f32_kern() - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , is_avx2_(mayiuse(avx2)) , LDA_(is_windows ? rdi : r8) , X_(is_windows ? rsi : r9) diff --git a/src/cpu/x64/gemm/f32/jit_avx_gemv_t_f32_kern.hpp b/src/cpu/x64/gemm/f32/jit_avx_gemv_t_f32_kern.hpp index 3b9bf70b462..a97b6b9b106 100644 --- a/src/cpu/x64/gemm/f32/jit_avx_gemv_t_f32_kern.hpp +++ b/src/cpu/x64/gemm/f32/jit_avx_gemv_t_f32_kern.hpp @@ -24,7 +24,7 @@ namespace impl { namespace cpu { namespace x64 { -class jit_avx_gemv_t_f32_kern : public jit_generator { +class jit_avx_gemv_t_f32_kern : public jit_generator_t { public: jit_avx_gemv_t_f32_kern(void); DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_gemv_t_f32_kern); diff --git a/src/cpu/x64/gemm/f32/jit_avx_kernel_b0_sgemm_kern_part1_autogen.cpp b/src/cpu/x64/gemm/f32/jit_avx_kernel_b0_sgemm_kern_part1_autogen.cpp index 52fccd21619..90666d80512 100644 --- a/src/cpu/x64/gemm/f32/jit_avx_kernel_b0_sgemm_kern_part1_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx_kernel_b0_sgemm_kern_part1_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ namespace cpu { namespace x64 { jit_avx_kernel_b0_sgemm_kern::jit_avx_kernel_b0_sgemm_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_kernel_b0_sgemm_kern::generate() { Xbyak::Label l259c; diff --git a/src/cpu/x64/gemm/f32/jit_avx_kernel_sgemm_kern_part1_autogen.cpp b/src/cpu/x64/gemm/f32/jit_avx_kernel_sgemm_kern_part1_autogen.cpp index 8ea5bd9a729..ad8104bedb7 100644 --- a/src/cpu/x64/gemm/f32/jit_avx_kernel_sgemm_kern_part1_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_avx_kernel_sgemm_kern_part1_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_kernel_sgemm_kern::jit_avx_kernel_sgemm_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_kernel_sgemm_kern::generate() { Xbyak::Label l1efc; diff --git a/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_an_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_an_kern_autogen.cpp index 57039cba5b0..c1ce936c967 100644 --- a/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_f32_copy_an_kern::jit_sse41_f32_copy_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_f32_copy_an_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_at_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_at_kern_autogen.cpp index b1381469d1b..fa8b332c166 100644 --- a/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_f32_copy_at_kern::jit_sse41_f32_copy_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_f32_copy_at_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_bn_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_bn_kern_autogen.cpp index f095bf750e9..ef9f427c13c 100644 --- a/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_f32_copy_bn_kern::jit_sse41_f32_copy_bn_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_f32_copy_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_bt_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_bt_kern_autogen.cpp index 3f509e5dcef..8fc16c37590 100644 --- a/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_sse41_f32_copy_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_f32_copy_bt_kern::jit_sse41_f32_copy_bt_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_f32_copy_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_sse41_gemv_n_f32_kern.cpp b/src/cpu/x64/gemm/f32/jit_sse41_gemv_n_f32_kern.cpp index cb195f55006..c8c1e61f66e 100644 --- a/src/cpu/x64/gemm/f32/jit_sse41_gemv_n_f32_kern.cpp +++ b/src/cpu/x64/gemm/f32/jit_sse41_gemv_n_f32_kern.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2021-2024 Intel Corporation +* Copyright 2021-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -314,7 +314,7 @@ void jit_sse41_gemv_n_f32_kern::generate() { // Function signature: gemv(*m, *n, *alpha, *a, *lda, *x, *incx, *y, *incy) jit_sse41_gemv_n_f32_kern::jit_sse41_gemv_n_f32_kern(void) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , has_avx512_(mayiuse(avx512_core) && __BUILD_GEMM_AVX512) , has_avx2_(mayiuse(avx2) && __BUILD_GEMM_AVX2) , has_avx_(mayiuse(avx) && __BUILD_GEMM_AVX2) diff --git a/src/cpu/x64/gemm/f32/jit_sse41_gemv_n_f32_kern.hpp b/src/cpu/x64/gemm/f32/jit_sse41_gemv_n_f32_kern.hpp index 1a3b60ac2b3..606e4747a13 100644 --- a/src/cpu/x64/gemm/f32/jit_sse41_gemv_n_f32_kern.hpp +++ b/src/cpu/x64/gemm/f32/jit_sse41_gemv_n_f32_kern.hpp @@ -24,7 +24,7 @@ namespace impl { namespace cpu { namespace x64 { -class jit_sse41_gemv_n_f32_kern : public jit_generator { +class jit_sse41_gemv_n_f32_kern : public jit_generator_t { public: jit_sse41_gemv_n_f32_kern(); DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_gemv_n_f32_kern); diff --git a/src/cpu/x64/gemm/f32/jit_sse41_gemv_t_f32_kern.cpp b/src/cpu/x64/gemm/f32/jit_sse41_gemv_t_f32_kern.cpp index b3b578975fc..c005c282d5e 100644 --- a/src/cpu/x64/gemm/f32/jit_sse41_gemv_t_f32_kern.cpp +++ b/src/cpu/x64/gemm/f32/jit_sse41_gemv_t_f32_kern.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -273,7 +273,7 @@ void jit_sse41_gemv_t_f32_kern::generate() { // Function signature: gemv(*m, *n, *alpha, *a, *lda, *x, *incx, *y, *incy) jit_sse41_gemv_t_f32_kern::jit_sse41_gemv_t_f32_kern() - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , LDA_(is_windows ? rdi : r8) , X_(is_windows ? rsi : r9) , INCY_(r10) diff --git a/src/cpu/x64/gemm/f32/jit_sse41_gemv_t_f32_kern.hpp b/src/cpu/x64/gemm/f32/jit_sse41_gemv_t_f32_kern.hpp index 36fadc0d4bb..7f2f37a98bc 100644 --- a/src/cpu/x64/gemm/f32/jit_sse41_gemv_t_f32_kern.hpp +++ b/src/cpu/x64/gemm/f32/jit_sse41_gemv_t_f32_kern.hpp @@ -24,7 +24,7 @@ namespace impl { namespace cpu { namespace x64 { -class jit_sse41_gemv_t_f32_kern : public jit_generator { +class jit_sse41_gemv_t_f32_kern : public jit_generator_t { public: jit_sse41_gemv_t_f32_kern(void); DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_gemv_t_f32_kern); diff --git a/src/cpu/x64/gemm/f32/jit_sse41_kernel_b0_sgemm_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_sse41_kernel_b0_sgemm_kern_autogen.cpp index ae734d720b7..60c1b28bb7b 100644 --- a/src/cpu/x64/gemm/f32/jit_sse41_kernel_b0_sgemm_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_sse41_kernel_b0_sgemm_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_kernel_b0_sgemm_kern::jit_sse41_kernel_b0_sgemm_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_kernel_b0_sgemm_kern::generate() { diff --git a/src/cpu/x64/gemm/f32/jit_sse41_kernel_sgemm_kern_autogen.cpp b/src/cpu/x64/gemm/f32/jit_sse41_kernel_sgemm_kern_autogen.cpp index ba6a36882ed..21e23224ffc 100644 --- a/src/cpu/x64/gemm/f32/jit_sse41_kernel_sgemm_kern_autogen.cpp +++ b/src/cpu/x64/gemm/f32/jit_sse41_kernel_sgemm_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_kernel_sgemm_kern::jit_sse41_kernel_sgemm_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_kernel_sgemm_kern::generate() { diff --git a/src/cpu/x64/gemm/gemm_info.cpp b/src/cpu/x64/gemm/gemm_info.cpp index 6489bfca792..2aec1011282 100644 --- a/src/cpu/x64/gemm/gemm_info.cpp +++ b/src/cpu/x64/gemm/gemm_info.cpp @@ -404,8 +404,8 @@ void gemm_info_t::jit_init(void) { = is_bf16 && mayiuse(avx512_core_amx) && __BUILD_GEMM_AMX; bool is_amx = is_int8_amx || is_bf16_amx; - static maybe_unique_ptr copy_a[2][2] = {{nullptr}}; - static maybe_unique_ptr copy_b[2][2] = {{nullptr}}; + static maybe_unique_ptr copy_a[2][2] = {{nullptr}}; + static maybe_unique_ptr copy_b[2][2] = {{nullptr}}; switch (data_traits_t::data_type) { case data_type::s8: @@ -643,7 +643,7 @@ void gemm_info_t::jit_init(void) { UNUSED(is_b_s8); UNUSED(is_c_s32); - static maybe_unique_ptr kernel[2][2][2][2] + static maybe_unique_ptr kernel[2][2][2][2] = {{{{nullptr}}}}; switch (data_traits_t::data_type) { case data_type::s8: @@ -774,10 +774,10 @@ void gemm_info_t::jit_init(void) { default: break; } - static maybe_unique_ptr gemv_kernel[2] = {nullptr}; - static maybe_unique_ptr gemv_s8s8s32_kernel = nullptr; - static maybe_unique_ptr gemv_s8u8s32_kernel = nullptr; - static maybe_unique_ptr gemv_u8s8s32_kernel = nullptr; + static maybe_unique_ptr gemv_kernel[2] = {nullptr}; + static maybe_unique_ptr gemv_s8s8s32_kernel = nullptr; + static maybe_unique_ptr gemv_s8u8s32_kernel = nullptr; + static maybe_unique_ptr gemv_u8s8s32_kernel = nullptr; switch (data_traits_t::data_type) { case data_type::s8: if (false) { diff --git a/src/cpu/x64/gemm/s8x8s32/common_u8.hpp b/src/cpu/x64/gemm/s8x8s32/common_u8.hpp index 6d5f8edbc95..2ffbf023c1f 100644 --- a/src/cpu/x64/gemm/s8x8s32/common_u8.hpp +++ b/src/cpu/x64/gemm/s8x8s32/common_u8.hpp @@ -30,7 +30,7 @@ namespace impl { namespace cpu { namespace x64 { -class jit_avx512_core_u8_copy_an_kern : public jit_generator { +class jit_avx512_core_u8_copy_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_u8_copy_an_kern); void generate() override; @@ -38,7 +38,7 @@ class jit_avx512_core_u8_copy_an_kern : public jit_generator { jit_avx512_core_u8_copy_an_kern(); }; -class jit_avx512_core_u8_copy_at_kern : public jit_generator { +class jit_avx512_core_u8_copy_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_u8_copy_at_kern); void generate() override; @@ -46,7 +46,7 @@ class jit_avx512_core_u8_copy_at_kern : public jit_generator { jit_avx512_core_u8_copy_at_kern(); }; -class jit_avx512_core_u8_copy_bn_kern : public jit_generator { +class jit_avx512_core_u8_copy_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_u8_copy_bn_kern); void generate() override; bool s8_case; @@ -55,7 +55,7 @@ class jit_avx512_core_u8_copy_bn_kern : public jit_generator { jit_avx512_core_u8_copy_bn_kern(bool s8 = false); }; -class jit_avx512_core_u8_copy_bt_kern : public jit_generator { +class jit_avx512_core_u8_copy_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_u8_copy_bt_kern); void generate() override; bool s8_case; @@ -64,7 +64,7 @@ class jit_avx512_core_u8_copy_bt_kern : public jit_generator { jit_avx512_core_u8_copy_bt_kern(bool s8 = false); }; -class jit_avx512_core_u8_copy_sum_an_kern : public jit_generator { +class jit_avx512_core_u8_copy_sum_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_u8_copy_sum_an_kern); void generate() override; @@ -72,7 +72,7 @@ class jit_avx512_core_u8_copy_sum_an_kern : public jit_generator { jit_avx512_core_u8_copy_sum_an_kern(); }; -class jit_avx512_core_u8_copy_sum_at_kern : public jit_generator { +class jit_avx512_core_u8_copy_sum_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_u8_copy_sum_at_kern); void generate() override; @@ -80,7 +80,7 @@ class jit_avx512_core_u8_copy_sum_at_kern : public jit_generator { jit_avx512_core_u8_copy_sum_at_kern(); }; -class jit_avx512_core_u8_copy_sum_bn_kern : public jit_generator { +class jit_avx512_core_u8_copy_sum_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_u8_copy_sum_bn_kern); void generate() override; bool s8_case; @@ -89,7 +89,7 @@ class jit_avx512_core_u8_copy_sum_bn_kern : public jit_generator { jit_avx512_core_u8_copy_sum_bn_kern(bool s8 = false); }; -class jit_avx512_core_u8_copy_sum_bt_kern : public jit_generator { +class jit_avx512_core_u8_copy_sum_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_u8_copy_sum_bt_kern); void generate() override; bool s8_case; @@ -98,7 +98,7 @@ class jit_avx512_core_u8_copy_sum_bt_kern : public jit_generator { jit_avx512_core_u8_copy_sum_bt_kern(bool s8 = false); }; -class jit_avx2_vnni_u8_copy_an_kern : public jit_generator { +class jit_avx2_vnni_u8_copy_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_vnni_u8_copy_an_kern); void generate() override; @@ -106,7 +106,7 @@ class jit_avx2_vnni_u8_copy_an_kern : public jit_generator { jit_avx2_vnni_u8_copy_an_kern(); }; -class jit_avx2_vnni_u8_copy_at_kern : public jit_generator { +class jit_avx2_vnni_u8_copy_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_vnni_u8_copy_at_kern); void generate() override; @@ -114,7 +114,7 @@ class jit_avx2_vnni_u8_copy_at_kern : public jit_generator { jit_avx2_vnni_u8_copy_at_kern(); }; -class jit_avx2_vnni_u8_copy_bn_kern : public jit_generator { +class jit_avx2_vnni_u8_copy_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_vnni_u8_copy_bn_kern); void generate() override; @@ -122,7 +122,7 @@ class jit_avx2_vnni_u8_copy_bn_kern : public jit_generator { jit_avx2_vnni_u8_copy_bn_kern(); }; -class jit_avx2_vnni_u8_copy_bt_kern : public jit_generator { +class jit_avx2_vnni_u8_copy_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_vnni_u8_copy_bt_kern); void generate() override; @@ -130,7 +130,7 @@ class jit_avx2_vnni_u8_copy_bt_kern : public jit_generator { jit_avx2_vnni_u8_copy_bt_kern(); }; -class jit_avx2_vnni_u8_copy_sum_an_kern : public jit_generator { +class jit_avx2_vnni_u8_copy_sum_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_vnni_u8_copy_sum_an_kern); void generate() override; @@ -138,7 +138,7 @@ class jit_avx2_vnni_u8_copy_sum_an_kern : public jit_generator { jit_avx2_vnni_u8_copy_sum_an_kern(); }; -class jit_avx2_vnni_u8_copy_sum_at_kern : public jit_generator { +class jit_avx2_vnni_u8_copy_sum_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_vnni_u8_copy_sum_at_kern); void generate() override; @@ -146,7 +146,7 @@ class jit_avx2_vnni_u8_copy_sum_at_kern : public jit_generator { jit_avx2_vnni_u8_copy_sum_at_kern(); }; -class jit_avx2_vnni_u8_copy_sum_bn_kern : public jit_generator { +class jit_avx2_vnni_u8_copy_sum_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_vnni_u8_copy_sum_bn_kern); void generate() override; @@ -154,7 +154,7 @@ class jit_avx2_vnni_u8_copy_sum_bn_kern : public jit_generator { jit_avx2_vnni_u8_copy_sum_bn_kern(); }; -class jit_avx2_vnni_u8_copy_sum_bt_kern : public jit_generator { +class jit_avx2_vnni_u8_copy_sum_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_vnni_u8_copy_sum_bt_kern); void generate() override; @@ -162,7 +162,7 @@ class jit_avx2_vnni_u8_copy_sum_bt_kern : public jit_generator { jit_avx2_vnni_u8_copy_sum_bt_kern(); }; -class jit_avx2_u8_copy_an_kern : public jit_generator { +class jit_avx2_u8_copy_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_u8_copy_an_kern); void generate() override; @@ -170,7 +170,7 @@ class jit_avx2_u8_copy_an_kern : public jit_generator { jit_avx2_u8_copy_an_kern(); }; -class jit_avx2_u8_copy_at_kern : public jit_generator { +class jit_avx2_u8_copy_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_u8_copy_at_kern); void generate() override; @@ -178,7 +178,7 @@ class jit_avx2_u8_copy_at_kern : public jit_generator { jit_avx2_u8_copy_at_kern(); }; -class jit_avx2_u8_copy_bn_kern : public jit_generator { +class jit_avx2_u8_copy_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_u8_copy_bn_kern); void generate() override; @@ -186,7 +186,7 @@ class jit_avx2_u8_copy_bn_kern : public jit_generator { jit_avx2_u8_copy_bn_kern(); }; -class jit_avx2_u8_copy_bt_kern : public jit_generator { +class jit_avx2_u8_copy_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_u8_copy_bt_kern); void generate() override; @@ -194,7 +194,7 @@ class jit_avx2_u8_copy_bt_kern : public jit_generator { jit_avx2_u8_copy_bt_kern(); }; -class jit_avx2_u8_copy_sum_an_kern : public jit_generator { +class jit_avx2_u8_copy_sum_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_u8_copy_sum_an_kern); void generate() override; @@ -202,7 +202,7 @@ class jit_avx2_u8_copy_sum_an_kern : public jit_generator { jit_avx2_u8_copy_sum_an_kern(); }; -class jit_avx2_u8_copy_sum_at_kern : public jit_generator { +class jit_avx2_u8_copy_sum_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_u8_copy_sum_at_kern); void generate() override; @@ -210,7 +210,7 @@ class jit_avx2_u8_copy_sum_at_kern : public jit_generator { jit_avx2_u8_copy_sum_at_kern(); }; -class jit_avx2_u8_copy_sum_bn_kern : public jit_generator { +class jit_avx2_u8_copy_sum_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_u8_copy_sum_bn_kern); void generate() override; @@ -218,7 +218,7 @@ class jit_avx2_u8_copy_sum_bn_kern : public jit_generator { jit_avx2_u8_copy_sum_bn_kern(); }; -class jit_avx2_u8_copy_sum_bt_kern : public jit_generator { +class jit_avx2_u8_copy_sum_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_u8_copy_sum_bt_kern); void generate() override; @@ -226,7 +226,7 @@ class jit_avx2_u8_copy_sum_bt_kern : public jit_generator { jit_avx2_u8_copy_sum_bt_kern(); }; -class jit_avx_u8_copy_an_kern : public jit_generator { +class jit_avx_u8_copy_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_u8_copy_an_kern); void generate() override; @@ -234,7 +234,7 @@ class jit_avx_u8_copy_an_kern : public jit_generator { jit_avx_u8_copy_an_kern(); }; -class jit_avx_u8_copy_at_kern : public jit_generator { +class jit_avx_u8_copy_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_u8_copy_at_kern); void generate() override; @@ -242,7 +242,7 @@ class jit_avx_u8_copy_at_kern : public jit_generator { jit_avx_u8_copy_at_kern(); }; -class jit_avx_u8_copy_bn_kern : public jit_generator { +class jit_avx_u8_copy_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_u8_copy_bn_kern); void generate() override; @@ -250,7 +250,7 @@ class jit_avx_u8_copy_bn_kern : public jit_generator { jit_avx_u8_copy_bn_kern(); }; -class jit_avx_u8_copy_bt_kern : public jit_generator { +class jit_avx_u8_copy_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_u8_copy_bt_kern); void generate() override; @@ -258,7 +258,7 @@ class jit_avx_u8_copy_bt_kern : public jit_generator { jit_avx_u8_copy_bt_kern(); }; -class jit_avx_u8_copy_sum_an_kern : public jit_generator { +class jit_avx_u8_copy_sum_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_u8_copy_sum_an_kern); void generate() override; @@ -266,7 +266,7 @@ class jit_avx_u8_copy_sum_an_kern : public jit_generator { jit_avx_u8_copy_sum_an_kern(); }; -class jit_avx_u8_copy_sum_at_kern : public jit_generator { +class jit_avx_u8_copy_sum_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_u8_copy_sum_at_kern); void generate() override; @@ -274,7 +274,7 @@ class jit_avx_u8_copy_sum_at_kern : public jit_generator { jit_avx_u8_copy_sum_at_kern(); }; -class jit_avx_u8_copy_sum_bn_kern : public jit_generator { +class jit_avx_u8_copy_sum_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_u8_copy_sum_bn_kern); void generate() override; @@ -282,7 +282,7 @@ class jit_avx_u8_copy_sum_bn_kern : public jit_generator { jit_avx_u8_copy_sum_bn_kern(); }; -class jit_avx_u8_copy_sum_bt_kern : public jit_generator { +class jit_avx_u8_copy_sum_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_u8_copy_sum_bt_kern); void generate() override; @@ -290,7 +290,7 @@ class jit_avx_u8_copy_sum_bt_kern : public jit_generator { jit_avx_u8_copy_sum_bt_kern(); }; -class jit_avx_kernel_b0_gemm_s8u8s32_kern : public jit_generator { +class jit_avx_kernel_b0_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_kernel_b0_gemm_s8u8s32_kern); void generate() override; @@ -298,7 +298,7 @@ class jit_avx_kernel_b0_gemm_s8u8s32_kern : public jit_generator { jit_avx_kernel_b0_gemm_s8u8s32_kern(); }; -class jit_avx_kernel_b0_b_gemm_s8u8s32_kern : public jit_generator { +class jit_avx_kernel_b0_b_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_kernel_b0_b_gemm_s8u8s32_kern); void generate() override; @@ -306,7 +306,7 @@ class jit_avx_kernel_b0_b_gemm_s8u8s32_kern : public jit_generator { jit_avx_kernel_b0_b_gemm_s8u8s32_kern(); }; -class jit_avx_kernel_b0_r_gemm_s8u8s32_kern : public jit_generator { +class jit_avx_kernel_b0_r_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_kernel_b0_r_gemm_s8u8s32_kern); void generate() override; @@ -314,7 +314,7 @@ class jit_avx_kernel_b0_r_gemm_s8u8s32_kern : public jit_generator { jit_avx_kernel_b0_r_gemm_s8u8s32_kern(); }; -class jit_avx_kernel_b0_c_gemm_s8u8s32_kern : public jit_generator { +class jit_avx_kernel_b0_c_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_kernel_b0_c_gemm_s8u8s32_kern); void generate() override; @@ -322,7 +322,7 @@ class jit_avx_kernel_b0_c_gemm_s8u8s32_kern : public jit_generator { jit_avx_kernel_b0_c_gemm_s8u8s32_kern(); }; -class jit_avx_kernel_gemm_s8u8s32_kern : public jit_generator { +class jit_avx_kernel_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_kernel_gemm_s8u8s32_kern); void generate() override; @@ -330,7 +330,7 @@ class jit_avx_kernel_gemm_s8u8s32_kern : public jit_generator { jit_avx_kernel_gemm_s8u8s32_kern(); }; -class jit_avx_kernel_b_gemm_s8u8s32_kern : public jit_generator { +class jit_avx_kernel_b_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_kernel_b_gemm_s8u8s32_kern); void generate() override; @@ -338,7 +338,7 @@ class jit_avx_kernel_b_gemm_s8u8s32_kern : public jit_generator { jit_avx_kernel_b_gemm_s8u8s32_kern(); }; -class jit_avx_kernel_r_gemm_s8u8s32_kern : public jit_generator { +class jit_avx_kernel_r_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_kernel_r_gemm_s8u8s32_kern); void generate() override; @@ -346,7 +346,7 @@ class jit_avx_kernel_r_gemm_s8u8s32_kern : public jit_generator { jit_avx_kernel_r_gemm_s8u8s32_kern(); }; -class jit_avx_kernel_c_gemm_s8u8s32_kern : public jit_generator { +class jit_avx_kernel_c_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx_kernel_c_gemm_s8u8s32_kern); void generate() override; @@ -354,7 +354,7 @@ class jit_avx_kernel_c_gemm_s8u8s32_kern : public jit_generator { jit_avx_kernel_c_gemm_s8u8s32_kern(); }; -class jit_sse41_u8_copy_an_kern : public jit_generator { +class jit_sse41_u8_copy_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_u8_copy_an_kern); void generate() override; @@ -362,7 +362,7 @@ class jit_sse41_u8_copy_an_kern : public jit_generator { jit_sse41_u8_copy_an_kern(); }; -class jit_sse41_u8_copy_at_kern : public jit_generator { +class jit_sse41_u8_copy_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_u8_copy_at_kern); void generate() override; @@ -370,7 +370,7 @@ class jit_sse41_u8_copy_at_kern : public jit_generator { jit_sse41_u8_copy_at_kern(); }; -class jit_sse41_u8_copy_bn_kern : public jit_generator { +class jit_sse41_u8_copy_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_u8_copy_bn_kern); void generate() override; @@ -378,7 +378,7 @@ class jit_sse41_u8_copy_bn_kern : public jit_generator { jit_sse41_u8_copy_bn_kern(); }; -class jit_sse41_u8_copy_bt_kern : public jit_generator { +class jit_sse41_u8_copy_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_u8_copy_bt_kern); void generate() override; @@ -386,7 +386,7 @@ class jit_sse41_u8_copy_bt_kern : public jit_generator { jit_sse41_u8_copy_bt_kern(); }; -class jit_sse41_u8_copy_sum_an_kern : public jit_generator { +class jit_sse41_u8_copy_sum_an_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_u8_copy_sum_an_kern); void generate() override; @@ -394,7 +394,7 @@ class jit_sse41_u8_copy_sum_an_kern : public jit_generator { jit_sse41_u8_copy_sum_an_kern(); }; -class jit_sse41_u8_copy_sum_at_kern : public jit_generator { +class jit_sse41_u8_copy_sum_at_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_u8_copy_sum_at_kern); void generate() override; @@ -402,7 +402,7 @@ class jit_sse41_u8_copy_sum_at_kern : public jit_generator { jit_sse41_u8_copy_sum_at_kern(); }; -class jit_sse41_u8_copy_sum_bn_kern : public jit_generator { +class jit_sse41_u8_copy_sum_bn_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_u8_copy_sum_bn_kern); void generate() override; @@ -410,7 +410,7 @@ class jit_sse41_u8_copy_sum_bn_kern : public jit_generator { jit_sse41_u8_copy_sum_bn_kern(); }; -class jit_sse41_u8_copy_sum_bt_kern : public jit_generator { +class jit_sse41_u8_copy_sum_bt_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_u8_copy_sum_bt_kern); void generate() override; @@ -418,7 +418,7 @@ class jit_sse41_u8_copy_sum_bt_kern : public jit_generator { jit_sse41_u8_copy_sum_bt_kern(); }; -class jit_sse41_kernel_b0_gemm_s8u8s32_kern : public jit_generator { +class jit_sse41_kernel_b0_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_kernel_b0_gemm_s8u8s32_kern); void generate() override; @@ -426,7 +426,7 @@ class jit_sse41_kernel_b0_gemm_s8u8s32_kern : public jit_generator { jit_sse41_kernel_b0_gemm_s8u8s32_kern(); }; -class jit_sse41_kernel_b0_b_gemm_s8u8s32_kern : public jit_generator { +class jit_sse41_kernel_b0_b_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_kernel_b0_b_gemm_s8u8s32_kern); void generate() override; @@ -434,7 +434,7 @@ class jit_sse41_kernel_b0_b_gemm_s8u8s32_kern : public jit_generator { jit_sse41_kernel_b0_b_gemm_s8u8s32_kern(); }; -class jit_sse41_kernel_b0_r_gemm_s8u8s32_kern : public jit_generator { +class jit_sse41_kernel_b0_r_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_kernel_b0_r_gemm_s8u8s32_kern); void generate() override; @@ -442,7 +442,7 @@ class jit_sse41_kernel_b0_r_gemm_s8u8s32_kern : public jit_generator { jit_sse41_kernel_b0_r_gemm_s8u8s32_kern(); }; -class jit_sse41_kernel_b0_c_gemm_s8u8s32_kern : public jit_generator { +class jit_sse41_kernel_b0_c_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_kernel_b0_c_gemm_s8u8s32_kern); void generate() override; @@ -450,7 +450,7 @@ class jit_sse41_kernel_b0_c_gemm_s8u8s32_kern : public jit_generator { jit_sse41_kernel_b0_c_gemm_s8u8s32_kern(); }; -class jit_sse41_kernel_gemm_s8u8s32_kern : public jit_generator { +class jit_sse41_kernel_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_kernel_gemm_s8u8s32_kern); void generate() override; @@ -458,7 +458,7 @@ class jit_sse41_kernel_gemm_s8u8s32_kern : public jit_generator { jit_sse41_kernel_gemm_s8u8s32_kern(); }; -class jit_sse41_kernel_b_gemm_s8u8s32_kern : public jit_generator { +class jit_sse41_kernel_b_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_kernel_b_gemm_s8u8s32_kern); void generate() override; @@ -466,7 +466,7 @@ class jit_sse41_kernel_b_gemm_s8u8s32_kern : public jit_generator { jit_sse41_kernel_b_gemm_s8u8s32_kern(); }; -class jit_sse41_kernel_r_gemm_s8u8s32_kern : public jit_generator { +class jit_sse41_kernel_r_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_kernel_r_gemm_s8u8s32_kern); void generate() override; @@ -474,7 +474,7 @@ class jit_sse41_kernel_r_gemm_s8u8s32_kern : public jit_generator { jit_sse41_kernel_r_gemm_s8u8s32_kern(); }; -class jit_sse41_kernel_c_gemm_s8u8s32_kern : public jit_generator { +class jit_sse41_kernel_c_gemm_s8u8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_sse41_kernel_c_gemm_s8u8s32_kern); void generate() override; diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_gemm_s8u8s32_kern.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_gemm_s8u8s32_kern.cpp index d15e3cc71b6..095ad417f0b 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_gemm_s8u8s32_kern.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_gemm_s8u8s32_kern.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2018-2024 Intel Corporation +* Copyright 2018-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -454,7 +454,7 @@ void jit_avx2_gemm_s8u8s32_kern::generate() { jit_avx2_gemm_s8u8s32_kern::jit_avx2_gemm_s8u8s32_kern(bool beta_zero, bool enable_offset_c, bool enable_offset_r, int unroll_m) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , beta_zero_(beta_zero) , enable_offset_c_(enable_offset_c) , enable_offset_r_(enable_offset_r) diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_gemm_s8u8s32_kern.hpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_gemm_s8u8s32_kern.hpp index 845974dc978..6141b193e1d 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_gemm_s8u8s32_kern.hpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_gemm_s8u8s32_kern.hpp @@ -24,7 +24,7 @@ namespace impl { namespace cpu { namespace x64 { -class jit_avx2_gemm_s8u8s32_kern : public jit_generator { +class jit_avx2_gemm_s8u8s32_kern : public jit_generator_t { public: jit_avx2_gemm_s8u8s32_kern(bool beta_zero, bool enable_offset_c, bool enable_offset_r, int unroll_m); diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_an_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_an_kern_autogen.cpp index 2b3b0481ae4..dfb7b597d90 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_u8_copy_an_kern::jit_avx2_u8_copy_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_u8_copy_an_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_at_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_at_kern_autogen.cpp index eba5bc7aa69..dc788f2174a 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_u8_copy_at_kern::jit_avx2_u8_copy_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_u8_copy_at_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_bn_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_bn_kern_autogen.cpp index c7693955be9..66281997ef4 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_u8_copy_bn_kern::jit_avx2_u8_copy_bn_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_u8_copy_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_bt_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_bt_kern_autogen.cpp index 457ca33123d..016e8c7ba55 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_u8_copy_bt_kern::jit_avx2_u8_copy_bt_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_u8_copy_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_an_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_an_kern_autogen.cpp index f0d95ff0a98..b31037588dc 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_u8_copy_sum_an_kern::jit_avx2_u8_copy_sum_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_u8_copy_sum_an_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_at_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_at_kern_autogen.cpp index c2b3c495035..0467f1a31a3 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_u8_copy_sum_at_kern::jit_avx2_u8_copy_sum_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_u8_copy_sum_at_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_bn_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_bn_kern_autogen.cpp index a5cc7eaa55b..918060fc329 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_u8_copy_sum_bn_kern::jit_avx2_u8_copy_sum_bn_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_u8_copy_sum_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_bt_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_bt_kern_autogen.cpp index f1b7931da6d..80b6a7c87fb 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_u8_copy_sum_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2024 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_u8_copy_sum_bt_kern::jit_avx2_u8_copy_sum_bt_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_u8_copy_sum_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_an_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_an_kern_autogen.cpp index 1d49a2ddbec..3173025a123 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_vnni_u8_copy_an_kern::jit_avx2_vnni_u8_copy_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_vnni_u8_copy_an_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_at_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_at_kern_autogen.cpp index d4e82a44368..3faf1d81855 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_vnni_u8_copy_at_kern::jit_avx2_vnni_u8_copy_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_vnni_u8_copy_at_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_bn_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_bn_kern_autogen.cpp index e554d48e1d5..286bcc23537 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_vnni_u8_copy_bn_kern::jit_avx2_vnni_u8_copy_bn_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_vnni_u8_copy_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_bt_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_bt_kern_autogen.cpp index 9ac53e82efd..69ac8854cda 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_vnni_u8_copy_bt_kern::jit_avx2_vnni_u8_copy_bt_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_vnni_u8_copy_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_an_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_an_kern_autogen.cpp index 476694dc1e9..60be2ff7150 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_vnni_u8_copy_sum_an_kern::jit_avx2_vnni_u8_copy_sum_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_vnni_u8_copy_sum_an_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_at_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_at_kern_autogen.cpp index 2e7147d5938..3a207448583 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_vnni_u8_copy_sum_at_kern::jit_avx2_vnni_u8_copy_sum_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_vnni_u8_copy_sum_at_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_bn_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_bn_kern_autogen.cpp index f338c8eb2fe..3ad8bd29bf2 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_vnni_u8_copy_sum_bn_kern::jit_avx2_vnni_u8_copy_sum_bn_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_vnni_u8_copy_sum_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_bt_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_bt_kern_autogen.cpp index ad94fb37065..6e3b2fe4ae7 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx2_vnni_u8_copy_sum_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx2_vnni_u8_copy_sum_bt_kern::jit_avx2_vnni_u8_copy_sum_bt_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx2_vnni_u8_copy_sum_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_gemm_s8u8s32_kern.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_gemm_s8u8s32_kern.cpp index 0776cef4255..62cdec8b801 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_gemm_s8u8s32_kern.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_gemm_s8u8s32_kern.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2018-2024 Intel Corporation +* Copyright 2018-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -486,7 +486,7 @@ void jit_avx512_core_gemm_s8u8s32_kern::generate() { jit_avx512_core_gemm_s8u8s32_kern::jit_avx512_core_gemm_s8u8s32_kern( bool beta_zero, bool enable_offset_c, bool enable_offset_r) - : jit_generator(jit_name(), + : jit_generator_t(jit_name(), mayiuse(avx512_core_vnni) ? avx512_core_vnni : avx512_core) , beta_zero_(beta_zero) , enable_offset_c_(enable_offset_c) diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_gemm_s8u8s32_kern.hpp b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_gemm_s8u8s32_kern.hpp index ddeb78f82cd..d6b6fbf4d52 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_gemm_s8u8s32_kern.hpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_gemm_s8u8s32_kern.hpp @@ -24,7 +24,7 @@ namespace impl { namespace cpu { namespace x64 { -class jit_avx512_core_gemm_s8u8s32_kern : public jit_generator { +class jit_avx512_core_gemm_s8u8s32_kern : public jit_generator_t { public: jit_avx512_core_gemm_s8u8s32_kern( bool beta_zero, bool enable_offset_c, bool enable_offset_r); diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_kernel_gemv_s8x8s32_kern.hpp b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_kernel_gemv_s8x8s32_kern.hpp index d5535c587a1..233d6094743 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_kernel_gemv_s8x8s32_kern.hpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_kernel_gemv_s8x8s32_kern.hpp @@ -33,7 +33,7 @@ namespace x64 { enum class ver_t { undef, s8s8, s8u8, u8s8 }; -class jit_avx512_core_gemv_s8x8s32_kern : public jit_generator { +class jit_avx512_core_gemv_s8x8s32_kern : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_gemv_s8x8s32_kern); enum class vnni_op_t { add, sub }; @@ -79,7 +79,7 @@ class jit_avx512_core_gemv_s8x8s32_kern : public jit_generator { public: jit_avx512_core_gemv_s8x8s32_kern(ver_t ver) - : jit_generator(jit_name(), + : jit_generator_t(jit_name(), mayiuse(avx512_core_vnni) ? avx512_core_vnni : avx512_core) , ver(ver) {} }; diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_an_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_an_kern_autogen.cpp index 2550875bda9..38cbbbafc0c 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2018-2024 Intel Corporation +* Copyright 2018-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_u8_copy_an_kern::jit_avx512_core_u8_copy_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_u8_copy_an_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_at_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_at_kern_autogen.cpp index f085c71d309..e553f45dd0a 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2018-2024 Intel Corporation +* Copyright 2018-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_u8_copy_at_kern::jit_avx512_core_u8_copy_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_u8_copy_at_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_bn_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_bn_kern_autogen.cpp index f7c5ad27ec9..5a8b2d9e148 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2018-2024 Intel Corporation +* Copyright 2018-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_u8_copy_bn_kern::jit_avx512_core_u8_copy_bn_kern(bool s8_case) - : jit_generator(jit_name()), s8_case(s8_case) {} + : jit_generator_t(jit_name()), s8_case(s8_case) {} void jit_avx512_core_u8_copy_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_bt_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_bt_kern_autogen.cpp index 64c29a6cec0..6e3dca0c654 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2018-2024 Intel Corporation +* Copyright 2018-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_u8_copy_bt_kern::jit_avx512_core_u8_copy_bt_kern(bool s8_case) - : jit_generator(jit_name()), s8_case(s8_case) {} + : jit_generator_t(jit_name()), s8_case(s8_case) {} void jit_avx512_core_u8_copy_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_an_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_an_kern_autogen.cpp index 482ea306c16..c680ef309fc 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2018-2024 Intel Corporation +* Copyright 2018-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_u8_copy_sum_an_kern::jit_avx512_core_u8_copy_sum_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_u8_copy_sum_an_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_at_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_at_kern_autogen.cpp index c7f4aa6bba2..53973138f08 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2018-2024 Intel Corporation +* Copyright 2018-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx512_core_u8_copy_sum_at_kern::jit_avx512_core_u8_copy_sum_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx512_core_u8_copy_sum_at_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_bn_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_bn_kern_autogen.cpp index df0690d4386..0e2374a7a28 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2018-2024 Intel Corporation +* Copyright 2018-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ namespace x64 { jit_avx512_core_u8_copy_sum_bn_kern::jit_avx512_core_u8_copy_sum_bn_kern( bool s8_case) - : jit_generator(jit_name()), s8_case(s8_case) {} + : jit_generator_t(jit_name()), s8_case(s8_case) {} void jit_avx512_core_u8_copy_sum_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_bt_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_bt_kern_autogen.cpp index 56f26c0fee6..23455a98584 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx512_core_u8_copy_sum_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2018-2024 Intel Corporation +* Copyright 2018-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ namespace x64 { jit_avx512_core_u8_copy_sum_bt_kern::jit_avx512_core_u8_copy_sum_bt_kern( bool s8_case) - : jit_generator(jit_name()), s8_case(s8_case) {} + : jit_generator_t(jit_name()), s8_case(s8_case) {} void jit_avx512_core_u8_copy_sum_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_b_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_b_gemm_s8u8s32_kern_autogen.cpp index 407e101023e..a23a565d3b6 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_b_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_b_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_kernel_b0_b_gemm_s8u8s32_kern::jit_avx_kernel_b0_b_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_kernel_b0_b_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_c_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_c_gemm_s8u8s32_kern_autogen.cpp index a9e0404359c..ed7ecd86b2a 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_c_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_c_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_kernel_b0_c_gemm_s8u8s32_kern::jit_avx_kernel_b0_c_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_kernel_b0_c_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_gemm_s8u8s32_kern_autogen.cpp index 787703f0d14..3946806f811 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_kernel_b0_gemm_s8u8s32_kern::jit_avx_kernel_b0_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_kernel_b0_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_r_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_r_gemm_s8u8s32_kern_autogen.cpp index 27e97312629..7bb0846d0bf 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_r_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b0_r_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_kernel_b0_r_gemm_s8u8s32_kern::jit_avx_kernel_b0_r_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_kernel_b0_r_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b_gemm_s8u8s32_kern_autogen.cpp index 13a64796615..8ebbc32ebb3 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_b_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_kernel_b_gemm_s8u8s32_kern::jit_avx_kernel_b_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_kernel_b_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_c_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_c_gemm_s8u8s32_kern_autogen.cpp index 09e47717e1f..f1a5528c683 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_c_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_c_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_kernel_c_gemm_s8u8s32_kern::jit_avx_kernel_c_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_kernel_c_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_gemm_s8u8s32_kern_autogen.cpp index 90ff166e17b..45c2e9a46d6 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_kernel_gemm_s8u8s32_kern::jit_avx_kernel_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_kernel_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_r_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_r_gemm_s8u8s32_kern_autogen.cpp index 12e426f9620..899191da533 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_r_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_kernel_r_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_kernel_r_gemm_s8u8s32_kern::jit_avx_kernel_r_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_kernel_r_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_an_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_an_kern_autogen.cpp index b326d519015..95fb9c6eb77 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_u8_copy_an_kern::jit_avx_u8_copy_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_u8_copy_an_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_at_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_at_kern_autogen.cpp index 436a9f22e60..cde3474fe65 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_u8_copy_at_kern::jit_avx_u8_copy_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_u8_copy_at_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_bn_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_bn_kern_autogen.cpp index 59eb1dfffcd..bfe94cd2bfe 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_u8_copy_bn_kern::jit_avx_u8_copy_bn_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_u8_copy_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_bt_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_bt_kern_autogen.cpp index 449e323c69c..1363423e93c 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_u8_copy_bt_kern::jit_avx_u8_copy_bt_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_u8_copy_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_an_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_an_kern_autogen.cpp index f69d61f18bf..4e14fab912b 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_u8_copy_sum_an_kern::jit_avx_u8_copy_sum_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_u8_copy_sum_an_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_at_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_at_kern_autogen.cpp index dd01dfc19ac..5a76e6851fb 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_u8_copy_sum_at_kern::jit_avx_u8_copy_sum_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_u8_copy_sum_at_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_bn_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_bn_kern_autogen.cpp index 187806913f1..15d4226a6e6 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_u8_copy_sum_bn_kern::jit_avx_u8_copy_sum_bn_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_u8_copy_sum_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_bt_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_bt_kern_autogen.cpp index f8df1c48a27..bd473e8f1a6 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_avx_u8_copy_sum_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_avx_u8_copy_sum_bt_kern::jit_avx_u8_copy_sum_bt_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_avx_u8_copy_sum_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_b_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_b_gemm_s8u8s32_kern_autogen.cpp index b1e7500e332..035b13f7e35 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_b_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_b_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ namespace x64 { jit_sse41_kernel_b0_b_gemm_s8u8s32_kern:: jit_sse41_kernel_b0_b_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_kernel_b0_b_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_c_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_c_gemm_s8u8s32_kern_autogen.cpp index 409777cac31..b8d5366536c 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_c_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_c_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ namespace x64 { jit_sse41_kernel_b0_c_gemm_s8u8s32_kern:: jit_sse41_kernel_b0_c_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_kernel_b0_c_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_gemm_s8u8s32_kern_autogen.cpp index 1807b589e25..2fd865cf0aa 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_kernel_b0_gemm_s8u8s32_kern::jit_sse41_kernel_b0_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_kernel_b0_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_r_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_r_gemm_s8u8s32_kern_autogen.cpp index 782a5d29eb3..419a6244052 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_r_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b0_r_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ namespace x64 { jit_sse41_kernel_b0_r_gemm_s8u8s32_kern:: jit_sse41_kernel_b0_r_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_kernel_b0_r_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b_gemm_s8u8s32_kern_autogen.cpp index cbed7557c21..30ed412ca6a 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_b_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_kernel_b_gemm_s8u8s32_kern::jit_sse41_kernel_b_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_kernel_b_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_c_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_c_gemm_s8u8s32_kern_autogen.cpp index 606e1e390fa..b9a2c2fdf8d 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_c_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_c_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_kernel_c_gemm_s8u8s32_kern::jit_sse41_kernel_c_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_kernel_c_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_gemm_s8u8s32_kern_autogen.cpp index 4eb2fe9c89d..cc3fdf5c1a2 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_kernel_gemm_s8u8s32_kern::jit_sse41_kernel_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_kernel_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_r_gemm_s8u8s32_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_r_gemm_s8u8s32_kern_autogen.cpp index 4de9d776821..3c16cb0fc7f 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_r_gemm_s8u8s32_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_kernel_r_gemm_s8u8s32_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_kernel_r_gemm_s8u8s32_kern::jit_sse41_kernel_r_gemm_s8u8s32_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_kernel_r_gemm_s8u8s32_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_an_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_an_kern_autogen.cpp index ceec68a3f26..27ac8048729 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_u8_copy_an_kern::jit_sse41_u8_copy_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_u8_copy_an_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_at_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_at_kern_autogen.cpp index daa4bd6aaa5..1f65076f341 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_u8_copy_at_kern::jit_sse41_u8_copy_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_u8_copy_at_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_bn_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_bn_kern_autogen.cpp index 1da99eefb6a..05f818b72bf 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_u8_copy_bn_kern::jit_sse41_u8_copy_bn_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_u8_copy_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_bt_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_bt_kern_autogen.cpp index 3d5382f3849..5486ae4f69c 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_u8_copy_bt_kern::jit_sse41_u8_copy_bt_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_u8_copy_bt_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_an_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_an_kern_autogen.cpp index 11e0048f627..67c404780ad 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_an_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_an_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_u8_copy_sum_an_kern::jit_sse41_u8_copy_sum_an_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_u8_copy_sum_an_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_at_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_at_kern_autogen.cpp index a9faa7dc314..043de4e79e6 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_at_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_at_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_u8_copy_sum_at_kern::jit_sse41_u8_copy_sum_at_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_u8_copy_sum_at_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_bn_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_bn_kern_autogen.cpp index 61d0601ba8b..47b2c602f4f 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_bn_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_bn_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_u8_copy_sum_bn_kern::jit_sse41_u8_copy_sum_bn_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_u8_copy_sum_bn_kern::generate() { diff --git a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_bt_kern_autogen.cpp b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_bt_kern_autogen.cpp index a46e0325391..a1b8ff2ff99 100644 --- a/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_bt_kern_autogen.cpp +++ b/src/cpu/x64/gemm/s8x8s32/jit_sse41_u8_copy_sum_bt_kern_autogen.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2024 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,7 +24,7 @@ namespace cpu { namespace x64 { jit_sse41_u8_copy_sum_bt_kern::jit_sse41_u8_copy_sum_bt_kern() - : jit_generator(jit_name()) {} + : jit_generator_t(jit_name()) {} void jit_sse41_u8_copy_sum_bt_kern::generate() { diff --git a/src/cpu/x64/gemm_bf16_convolution.cpp b/src/cpu/x64/gemm_bf16_convolution.cpp index 4fd77fddbfd..c36428bd2bf 100644 --- a/src/cpu/x64/gemm_bf16_convolution.cpp +++ b/src/cpu/x64/gemm_bf16_convolution.cpp @@ -71,7 +71,7 @@ void cvt_acc_to_dst(const conv_gemm_conf_t &jcp, size_t g_start, size_t g_end, template gemm_bf16_convolution_fwd_t::pp_ker_t::pp_ker_t(const pd_t *pd) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , jcp_(pd->jcp_) , do_sum_(dst_data_type != data_type::f32 && jcp_.with_sum) , max_data_reg_idx_(31) @@ -317,7 +317,7 @@ void gemm_bf16_convolution_fwd_t::pp_ker_t::operator()( args.post_ops_binary_rhs_arg_vec = post_ops_binary_rhs_arg_vec; args.dst_orig = dst_orig; args.g_oc_offset = g_oc_offset; - jit_generator::operator()(&args); + jit_generator_t::operator()(&args); } // operator () specialized for ncsp format @@ -343,7 +343,7 @@ void gemm_bf16_convolution_fwd_t::pp_ker_t::operator()( args.post_ops_binary_rhs_arg_vec = post_ops_binary_rhs_arg_vec; args.dst_orig = dst_orig; args.g_oc_offset = g_oc_offset; - jit_generator::operator()(&args); + jit_generator_t::operator()(&args); } template diff --git a/src/cpu/x64/gemm_bf16_convolution.hpp b/src/cpu/x64/gemm_bf16_convolution.hpp index 8ad36ebbc6f..6226fd39a9e 100644 --- a/src/cpu/x64/gemm_bf16_convolution.hpp +++ b/src/cpu/x64/gemm_bf16_convolution.hpp @@ -131,7 +131,7 @@ struct gemm_bf16_convolution_fwd_t : public primitive_t { const pd_t *pd() const { return (const pd_t *)primitive_t::pd().get(); } - class pp_ker_t : public jit_generator { + class pp_ker_t : public jit_generator_t { public: DECLARE_CPU_JIT_AUX_FUNCTIONS(gemm_bf16_convolution_fwd_t::pp_kernel); pp_ker_t(const pd_t *pd); diff --git a/src/cpu/x64/injectors/injector_utils.cpp b/src/cpu/x64/injectors/injector_utils.cpp index c79a8b4fc07..2254d73243d 100644 --- a/src/cpu/x64/injectors/injector_utils.cpp +++ b/src/cpu/x64/injectors/injector_utils.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2021 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ static std::size_t calc_vmm_to_preserve_size_bytes( }); } -register_preserve_guard_t::register_preserve_guard_t(jit_generator *host, +register_preserve_guard_t::register_preserve_guard_t(jit_generator_t *host, std::initializer_list reg64_to_preserve, std::initializer_list vmm_to_preserve) : host_(host) @@ -108,7 +108,7 @@ size_t register_preserve_guard_t::stack_space_occupied() const { }; conditional_register_preserve_guard_t::conditional_register_preserve_guard_t( - bool condition_to_be_met, jit_generator *host, + bool condition_to_be_met, jit_generator_t *host, std::initializer_list reg64_to_preserve, std::initializer_list vmm_to_preserve) : register_preserve_guard_t {condition_to_be_met diff --git a/src/cpu/x64/injectors/injector_utils.hpp b/src/cpu/x64/injectors/injector_utils.hpp index 6498f814b67..5e3708c67ea 100644 --- a/src/cpu/x64/injectors/injector_utils.hpp +++ b/src/cpu/x64/injectors/injector_utils.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2020-2022 Intel Corporation +* Copyright 2020-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -53,7 +53,7 @@ inline layout_t get_layout_type(const memory_desc_wrapper &dst_d) { class register_preserve_guard_t { public: - register_preserve_guard_t(jit_generator *host, + register_preserve_guard_t(jit_generator_t *host, std::initializer_list reg64_to_preserve, std::initializer_list vmm_to_preserve = {}); register_preserve_guard_t(register_preserve_guard_t &&other) = default; @@ -64,7 +64,7 @@ class register_preserve_guard_t { size_t stack_space_occupied() const; private: - jit_generator *host_; + jit_generator_t *host_; std::stack reg64_stack_; std::stack vmm_stack_; size_t vmm_to_preserve_size_bytes_; @@ -73,7 +73,7 @@ class register_preserve_guard_t { class conditional_register_preserve_guard_t : public register_preserve_guard_t { public: conditional_register_preserve_guard_t(bool condition_to_be_met, - jit_generator *host, + jit_generator_t *host, std::initializer_list reg64_to_preserve, std::initializer_list vmm_to_preserve = {}); DNNL_DISALLOW_COPY_AND_ASSIGN(conditional_register_preserve_guard_t); diff --git a/src/cpu/x64/injectors/jit_uni_binary_injector.cpp b/src/cpu/x64/injectors/jit_uni_binary_injector.cpp index 5fa3cb8d475..02711129d3e 100644 --- a/src/cpu/x64/injectors/jit_uni_binary_injector.cpp +++ b/src/cpu/x64/injectors/jit_uni_binary_injector.cpp @@ -268,7 +268,7 @@ rhs_arg_static_params_t::rhs_arg_static_params_t( template jit_uni_binary_injector_t::jit_uni_binary_injector_t( - jit_generator *host, const static_params_t &static_params) + jit_generator_t *host, const static_params_t &static_params) : host_(host) , f8_e5m2_emu_(static_params.f8_e5m2_emu_) , f8_e4m3_emu_(static_params.f8_e4m3_emu_) @@ -326,18 +326,18 @@ int jit_uni_binary_injector_t::adjust_temp_vmm_hint( } template -static void push_vmm(jit_generator *host, const Vmm &vmm) { +static void push_vmm(jit_generator_t *host, const Vmm &vmm) { host->sub(host->rsp, vreg_traits_t::vlen); host->uni_vmovups(host->ptr[host->rsp], vmm); } template -static void pop_vmm(jit_generator *host, const Vmm &vmm) { +static void pop_vmm(jit_generator_t *host, const Vmm &vmm) { host->uni_vmovups(vmm, host->ptr[host->rsp]); host->add(host->rsp, vreg_traits_t::vlen); } -static void push_opmask(jit_generator *host, const Xbyak::Opmask &k) { +static void push_opmask(jit_generator_t *host, const Xbyak::Opmask &k) { static constexpr int k_mask_size = 8; host->sub(host->rsp, k_mask_size); if (mayiuse(avx512_core)) @@ -346,7 +346,7 @@ static void push_opmask(jit_generator *host, const Xbyak::Opmask &k) { host->kmovw(host->ptr[host->rsp], k); } -static void pop_opmask(jit_generator *host, const Xbyak::Opmask &k) { +static void pop_opmask(jit_generator_t *host, const Xbyak::Opmask &k) { static constexpr int k_mask_size = 8; if (mayiuse(avx512_core)) host->kmovq(k, host->ptr[host->rsp]); @@ -356,7 +356,7 @@ static void pop_opmask(jit_generator *host, const Xbyak::Opmask &k) { } template -static void restore_stack(jit_generator *host, const Vmm &vmm) { +static void restore_stack(jit_generator_t *host, const Vmm &vmm) { host->add(host->rsp, vreg_traits_t::vlen); } @@ -2619,7 +2619,7 @@ struct helper_broadcast_s8u8_t {}; template struct helper_broadcast_s8u8_t { - static void execute_broadcast_s8u8_no_tail(jit_generator *host, + static void execute_broadcast_s8u8_no_tail(jit_generator_t *host, const int rhs_helper_reg_idx, const data_type_t &data_type, const Vmm &tmp_vmm, const Xbyak::Address &rhs_addr, const std::function &post_process) { @@ -2756,7 +2756,7 @@ void jit_uni_binary_injector_t::execute_broadcast_tail_with_opmask( static constexpr int xmm_size_elem = 4; -static void load_tail_avx(jit_generator *host, std::size_t ymm_idx, +static void load_tail_avx(jit_generator_t *host, std::size_t ymm_idx, std::size_t tail_size, const std::function &init_op, const std::function &ymm_upper_half_op, const std::function &ymm_lower_half_op) { @@ -2785,7 +2785,7 @@ static void load_tail_avx(jit_generator *host, std::size_t ymm_idx, } } -static void load_tail_avx(jit_generator *host, std::size_t ymm_idx, +static void load_tail_avx(jit_generator_t *host, std::size_t ymm_idx, std::size_t tail_size, const std::function &ymm_upper_half_op, const std::function &ymm_lower_half_op) { @@ -2798,7 +2798,7 @@ static Xbyak::uint8 MM_SHUFFLE( return (((z) << 6) | ((y) << 4) | ((x) << 2) | (w)); } -static void execute_broadcast_f32_tail_avx(jit_generator *host, +static void execute_broadcast_f32_tail_avx(jit_generator_t *host, const Xbyak::Ymm &vmm, const Xbyak::Address &rhs_addr, std::size_t tail_size) { @@ -2823,7 +2823,7 @@ static void execute_broadcast_f32_tail_avx(jit_generator *host, host, vmm_idx, tail_size, init_op, upper_half_op, lower_half_op); } -static void execute_broadcast_f32_tail_avx(jit_generator *host, +static void execute_broadcast_f32_tail_avx(jit_generator_t *host, const Xbyak::Xmm &vmm, const Xbyak::Address &rhs_addr, std::size_t tail_size) { @@ -2843,7 +2843,7 @@ struct helper_bcast_tail_t {}; template struct helper_bcast_tail_t { - static void execute_broadcast_tail_statically(jit_generator *host, + static void execute_broadcast_tail_statically(jit_generator_t *host, const size_t tail_size, const data_type_t &data_type, const Vmm &tmp_vmm, const Xbyak::Address &rhs_addr) { host->uni_vxorps(tmp_vmm, tmp_vmm, tmp_vmm); @@ -2866,7 +2866,7 @@ struct helper_bcast_tail_t { template struct helper_bcast_tail_t { - static void execute_broadcast_tail_statically(jit_generator *host, + static void execute_broadcast_tail_statically(jit_generator_t *host, const size_t tail_size, const data_type_t &data_type, const Vmm &tmp_vmm, const Xbyak::Address &rhs_addr, fp8_emulation_e5m2_t *f8_e5m2_emu, @@ -3223,7 +3223,7 @@ struct helper_load_tail_t {}; template struct helper_load_tail_t { - static void load_rhs_tail_statically(jit_generator *host, + static void load_rhs_tail_statically(jit_generator_t *host, const size_t tail_size, const Xbyak::Reg64 &rhs_addr_reg, const data_type_t &data_type, const Vmm &tmp_vmm, const Xbyak::Address &rhs_addr) { @@ -3238,7 +3238,7 @@ struct helper_load_tail_t { template struct helper_load_tail_t { - static void load_rhs_tail_statically(jit_generator *host, + static void load_rhs_tail_statically(jit_generator_t *host, const size_t tail_size, const Xbyak::Reg64 &rhs_addr_reg, const data_type_t &data_type, const Vmm &tmp_vmm, const Xbyak::Address &rhs_addr) { @@ -3449,22 +3449,22 @@ void jit_uni_binary_injector_t::execute_binary(alg_kind_t binary_alg, case alg_kind::binary_div: host_->uni_vdivps(dst, lhs, rhs); break; case alg_kind::binary_sub: host_->uni_vsubps(dst, lhs, rhs); break; case alg_kind::binary_ge: - execute_cmp_binary(dst, lhs, rhs, jit_generator::_cmp_nlt_us); + execute_cmp_binary(dst, lhs, rhs, jit_generator_t::_cmp_nlt_us); break; case alg_kind::binary_gt: - execute_cmp_binary(dst, lhs, rhs, jit_generator::_cmp_nle_us); + execute_cmp_binary(dst, lhs, rhs, jit_generator_t::_cmp_nle_us); break; case alg_kind::binary_le: - execute_cmp_binary(dst, lhs, rhs, jit_generator::_cmp_le_os); + execute_cmp_binary(dst, lhs, rhs, jit_generator_t::_cmp_le_os); break; case alg_kind::binary_lt: - execute_cmp_binary(dst, lhs, rhs, jit_generator::_cmp_lt_os); + execute_cmp_binary(dst, lhs, rhs, jit_generator_t::_cmp_lt_os); break; case alg_kind::binary_eq: - execute_cmp_binary(dst, lhs, rhs, jit_generator::_cmp_eq_oq); + execute_cmp_binary(dst, lhs, rhs, jit_generator_t::_cmp_eq_oq); break; case alg_kind::binary_ne: - execute_cmp_binary(dst, lhs, rhs, jit_generator::_cmp_neq_uq); + execute_cmp_binary(dst, lhs, rhs, jit_generator_t::_cmp_neq_uq); break; default: assert(!"unsupported algorithm"); } @@ -3476,7 +3476,7 @@ struct helper_binary_t {}; template struct helper_binary_t { template - static void execute_binary(jit_generator *host, F execute_cmp_binary, + static void execute_binary(jit_generator_t *host, F execute_cmp_binary, alg_kind_t binary_alg, const Vmm &dst, const Vmm &lhs, const T &rhs) { switch (binary_alg) { @@ -3487,22 +3487,22 @@ struct helper_binary_t { case alg_kind::binary_div: host->uni_vdivps(dst, lhs, rhs); break; case alg_kind::binary_sub: host->uni_vsubps(dst, lhs, rhs); break; case alg_kind::binary_ge: - execute_cmp_binary(dst, lhs, rhs, jit_generator::_cmp_nlt_us); + execute_cmp_binary(dst, lhs, rhs, jit_generator_t::_cmp_nlt_us); break; case alg_kind::binary_gt: - execute_cmp_binary(dst, lhs, rhs, jit_generator::_cmp_nle_us); + execute_cmp_binary(dst, lhs, rhs, jit_generator_t::_cmp_nle_us); break; case alg_kind::binary_le: - execute_cmp_binary(dst, lhs, rhs, jit_generator::_cmp_le_os); + execute_cmp_binary(dst, lhs, rhs, jit_generator_t::_cmp_le_os); break; case alg_kind::binary_lt: - execute_cmp_binary(dst, lhs, rhs, jit_generator::_cmp_lt_os); + execute_cmp_binary(dst, lhs, rhs, jit_generator_t::_cmp_lt_os); break; case alg_kind::binary_eq: - execute_cmp_binary(dst, lhs, rhs, jit_generator::_cmp_eq_oq); + execute_cmp_binary(dst, lhs, rhs, jit_generator_t::_cmp_eq_oq); break; case alg_kind::binary_ne: - execute_cmp_binary(dst, lhs, rhs, jit_generator::_cmp_neq_uq); + execute_cmp_binary(dst, lhs, rhs, jit_generator_t::_cmp_neq_uq); break; default: assert(!"unsupported algorithm"); } @@ -3550,7 +3550,7 @@ void jit_uni_binary_injector_t::execute_prelu( Xbyak::Opmask aux_kmask = get_aux_kmask(); host_->vxorps(tmp_vmm, tmp_vmm, tmp_vmm); host_->vcmpps(aux_kmask | maybe_tail_kmask, dst_vmm, tmp_vmm, - jit_generator::_cmp_le_os); + jit_generator_t::_cmp_le_os); host_->vmulps(dst_vmm | aux_kmask, dst_vmm, rhs); } else if (is_superset(isa, avx)) { // Three operand version diff --git a/src/cpu/x64/injectors/jit_uni_binary_injector.hpp b/src/cpu/x64/injectors/jit_uni_binary_injector.hpp index 1896e5fefaa..d284481dddc 100644 --- a/src/cpu/x64/injectors/jit_uni_binary_injector.hpp +++ b/src/cpu/x64/injectors/jit_uni_binary_injector.hpp @@ -161,7 +161,7 @@ struct rhs_arg_static_params_t { * * @param param1 - register storing abi param1. At the moment of calling * compute_vector_range method can be different than the default one defined - * inside jit_generator. + * inside jit_generator_t. * @param bcast_set_t supported_strategy_set - set allowing disabling particular * bcast strategies * @param rhs_arg_static_params - params related to all binary post-ops right-hand side @@ -259,7 +259,7 @@ template ::Vmm> class jit_uni_binary_injector_t { public: jit_uni_binary_injector_t( - jit_generator *host, const static_params_t &static_params); + jit_generator_t *host, const static_params_t &static_params); /* * Generates code of binary post_op injected to host primitive. Applied to @@ -597,7 +597,7 @@ class jit_uni_binary_injector_t { */ Xbyak::Opmask get_aux_kmask() const; - jit_generator *host_; + jit_generator_t *host_; fp8_emulation_e5m2_t *f8_e5m2_emu_ {nullptr}; fp8_emulation_e4m3_t *f8_e4m3_emu_ {nullptr}; const rhs_arg_static_params_t rhs_arg_static_params_; diff --git a/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp b/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp index d44602eac4b..63b18f4ec31 100644 --- a/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp +++ b/src/cpu/x64/injectors/jit_uni_eltwise_injector.hpp @@ -91,9 +91,9 @@ struct jit_uni_eltwise_injector_t { // - algorithm derivative. // use_dst - defines whether source or destination point is passed to alg // code. Depends on algorithm. See `_use_dst_for_bwd` algs definition. - jit_uni_eltwise_injector_t(jit_generator *host, alg_kind_t alg, float alpha, - float beta, float scale, data_type_t dt = data_type::f32, - bool save_state = true, + jit_uni_eltwise_injector_t(jit_generator_t *host, alg_kind_t alg, + float alpha, float beta, float scale, + data_type_t dt = data_type::f32, bool save_state = true, Xbyak::Reg64 p_table = Xbyak::Reg64(Xbyak::Operand::RAX), Xbyak::Opmask k_mask = Xbyak::Opmask(1), bool is_fwd = true, bool use_dst = false, bool preserve_vmm = true, @@ -117,7 +117,7 @@ struct jit_uni_eltwise_injector_t { register_table_entries(); } - jit_uni_eltwise_injector_t(jit_generator *host, + jit_uni_eltwise_injector_t(jit_generator_t *host, const post_ops_t::entry_t::eltwise_t &eltwise, data_type_t dt = data_type::f32, bool save_state = true, Xbyak::Reg64 p_table = Xbyak::Reg64(Xbyak::Operand::RAX), @@ -152,7 +152,7 @@ struct jit_uni_eltwise_injector_t { const float scale_; const data_type_t dt_; - jit_generator *const h; + jit_generator_t *const h; const bool save_state_; const Xbyak::Reg64 p_table_; @@ -165,16 +165,16 @@ struct jit_uni_eltwise_injector_t { Xbyak::Label l_table_; - // if only the injector was inherited from jit_generator... + // if only the injector was inherited from jit_generator_t... enum { - _cmp_eq_oq = jit_generator::_cmp_eq_oq, - _cmp_neq_uq = jit_generator::_cmp_neq_uq, - _cmp_lt_os = jit_generator::_cmp_lt_os, - _cmp_le_os = jit_generator::_cmp_le_os, - _cmp_ge_os = jit_generator::_cmp_nlt_us, - _cmp_gt_os = jit_generator::_cmp_nle_us, - _op_floor = jit_generator::_op_floor, - _op_mxcsr = jit_generator::_op_mxcsr + _cmp_eq_oq = jit_generator_t::_cmp_eq_oq, + _cmp_neq_uq = jit_generator_t::_cmp_neq_uq, + _cmp_lt_os = jit_generator_t::_cmp_lt_os, + _cmp_le_os = jit_generator_t::_cmp_le_os, + _cmp_ge_os = jit_generator_t::_cmp_nlt_us, + _cmp_gt_os = jit_generator_t::_cmp_nle_us, + _op_floor = jit_generator_t::_op_floor, + _op_mxcsr = jit_generator_t::_op_mxcsr }; const bool is_avx512_ = is_superset(isa, avx512_core); diff --git a/src/cpu/x64/injectors/jit_uni_postops_injector.cpp b/src/cpu/x64/injectors/jit_uni_postops_injector.cpp index 02ad28a44fb..d160969edde 100644 --- a/src/cpu/x64/injectors/jit_uni_postops_injector.cpp +++ b/src/cpu/x64/injectors/jit_uni_postops_injector.cpp @@ -53,7 +53,7 @@ size_t aux_vec_count(const post_ops_t &post_ops, cpu_isa_t isa, bool is_fwd) { template jit_uni_postops_injector_t::jit_uni_postops_injector_t( - jit_generator *host, const post_ops_t &post_ops, + jit_generator_t *host, const post_ops_t &post_ops, const binary_injector::static_params_t &binary_static_params, const eltwise_injector::static_params_t &eltwise_static_params, const lambda_jit_injectors_t &lambda_jit_injectors) @@ -100,14 +100,14 @@ jit_uni_postops_injector_t::jit_uni_postops_injector_t( template jit_uni_postops_injector_t::jit_uni_postops_injector_t( - jit_generator *host, const post_ops_t &post_ops, + jit_generator_t *host, const post_ops_t &post_ops, const binary_injector::static_params_t &binary_static_params) : jit_uni_postops_injector_t(host, post_ops, binary_static_params, eltwise_injector::static_params_t(), lambda_jit_injectors_t()) {} template jit_uni_postops_injector_t::jit_uni_postops_injector_t( - jit_generator *host, const post_ops_t &post_ops, + jit_generator_t *host, const post_ops_t &post_ops, const binary_injector::static_params_t &binary_static_params, const lambda_jit_injectors_t &lambda_jit_injectors) : jit_uni_postops_injector_t(host, post_ops, binary_static_params, @@ -115,7 +115,7 @@ jit_uni_postops_injector_t::jit_uni_postops_injector_t( template jit_uni_postops_injector_t::jit_uni_postops_injector_t( - jit_generator *host, const post_ops_t &post_ops, + jit_generator_t *host, const post_ops_t &post_ops, const binary_injector::static_params_t &binary_static_params, const eltwise_injector::static_params_t &eltwise_static_params) : jit_uni_postops_injector_t(host, post_ops, binary_static_params, @@ -125,7 +125,7 @@ jit_uni_postops_injector_t::jit_uni_postops_injector_t( // Vmm that don't make any sense like sse41 + Zmm. template <> jit_uni_postops_injector_base_t * -jit_uni_postops_injector_base_t::create(jit_generator *host, +jit_uni_postops_injector_base_t::create(jit_generator_t *host, cpu_isa_t isa, const post_ops_t &post_ops, const binary_injector::static_params_t &binary_static_params, const eltwise_injector::static_params_t &eltwise_static_params) { @@ -161,7 +161,7 @@ jit_uni_postops_injector_base_t::create(jit_generator *host, template <> jit_uni_postops_injector_base_t * -jit_uni_postops_injector_base_t::create(jit_generator *host, +jit_uni_postops_injector_base_t::create(jit_generator_t *host, cpu_isa_t isa, const post_ops_t &post_ops, const binary_injector::static_params_t &binary_static_params, const eltwise_injector::static_params_t &eltwise_static_params) { @@ -201,7 +201,7 @@ jit_uni_postops_injector_base_t::create(jit_generator *host, template <> jit_uni_postops_injector_base_t * -jit_uni_postops_injector_base_t::create(jit_generator *host, +jit_uni_postops_injector_base_t::create(jit_generator_t *host, cpu_isa_t isa, const post_ops_t &post_ops, const binary_injector::static_params_t &binary_static_params, const eltwise_injector::static_params_t &eltwise_static_params) { @@ -243,8 +243,8 @@ jit_uni_postops_injector_base_t::create(jit_generator *host, template jit_uni_postops_injector_base_t * -jit_uni_postops_injector_base_t::create(jit_generator *host, cpu_isa_t isa, - const post_ops_t &post_ops, +jit_uni_postops_injector_base_t::create(jit_generator_t *host, + cpu_isa_t isa, const post_ops_t &post_ops, const binary_injector::static_params_t &binary_static_params) { const eltwise_injector::static_params_t eltwise_static_params; return create( diff --git a/src/cpu/x64/injectors/jit_uni_postops_injector.hpp b/src/cpu/x64/injectors/jit_uni_postops_injector.hpp index 3c96e9922e2..399350b3c71 100644 --- a/src/cpu/x64/injectors/jit_uni_postops_injector.hpp +++ b/src/cpu/x64/injectors/jit_uni_postops_injector.hpp @@ -65,12 +65,12 @@ class jit_uni_postops_injector_base_t { // cases it's aligned with the former kernel ISA if such enum value is // instantiated for injectors. If not, uses the next available isa enum // value in compliance with same vector length. - static jit_uni_postops_injector_base_t *create(jit_generator *host, + static jit_uni_postops_injector_base_t *create(jit_generator_t *host, cpu_isa_t isa, const post_ops_t &post_ops, const binary_injector::static_params_t &binary_static_params, const eltwise_injector::static_params_t &eltwise_static_params); - static jit_uni_postops_injector_base_t *create(jit_generator *host, + static jit_uni_postops_injector_base_t *create(jit_generator_t *host, cpu_isa_t isa, const post_ops_t &post_ops, const binary_injector::static_params_t &binary_static_params); @@ -126,15 +126,19 @@ class jit_uni_postops_injector_t : public jit_uni_postops_injector_base_t { * @param lambda_jit_injectors - allows user specify custom injector * function for given post-op type */ - jit_uni_postops_injector_t(jit_generator *host, const post_ops_t &post_ops, + jit_uni_postops_injector_t(jit_generator_t *host, + const post_ops_t &post_ops, const binary_injector::static_params_t &binary_static_params); - jit_uni_postops_injector_t(jit_generator *host, const post_ops_t &post_ops, + jit_uni_postops_injector_t(jit_generator_t *host, + const post_ops_t &post_ops, const binary_injector::static_params_t &binary_static_params, const lambda_jit_injectors_t &lambda_jit_injectors); - jit_uni_postops_injector_t(jit_generator *host, const post_ops_t &post_ops, + jit_uni_postops_injector_t(jit_generator_t *host, + const post_ops_t &post_ops, const binary_injector::static_params_t &binary_static_params, const eltwise_injector::static_params_t &eltwise_static_params); - jit_uni_postops_injector_t(jit_generator *host, const post_ops_t &post_ops, + jit_uni_postops_injector_t(jit_generator_t *host, + const post_ops_t &post_ops, const binary_injector::static_params_t &binary_static_params, const eltwise_injector::static_params_t &eltwise_static_params, const lambda_jit_injectors_t &lambda_jit_injectors); @@ -171,7 +175,7 @@ class jit_uni_postops_injector_t : public jit_uni_postops_injector_base_t { private: post_ops_t post_ops_; - jit_generator *host_; + jit_generator_t *host_; // Key is a numerical order of a post-op in attributes. std::map> alg_to_eltwise_injector_; diff --git a/src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.cpp b/src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.cpp index 4aa0049c454..c56c4d71f3b 100644 --- a/src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.cpp +++ b/src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.cpp @@ -48,7 +48,7 @@ using namespace Xbyak; jit_avx2_1x1_conv_kernel_f32::jit_avx2_1x1_conv_kernel_f32( const jit_1x1_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name(), avx2), jcp(ajcp), attr_(attr) { + : jit_generator_t(jit_name(), avx2), jcp(ajcp), attr_(attr) { if (jcp.with_eltwise || jcp.with_binary) { using namespace binary_injector; static constexpr bool preserve_gpr = true; diff --git a/src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.hpp b/src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.hpp index a2e7e3c48ff..64ab3a6ac23 100644 --- a/src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.hpp +++ b/src/cpu/x64/jit_avx2_1x1_conv_kernel_f32.hpp @@ -30,7 +30,7 @@ namespace impl { namespace cpu { namespace x64 { -struct jit_avx2_1x1_conv_kernel_f32 : public jit_generator { +struct jit_avx2_1x1_conv_kernel_f32 : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_1x1_conv_kernel_f32) jit_avx2_1x1_conv_kernel_f32(const jit_1x1_conv_conf_t &ajcp, diff --git a/src/cpu/x64/jit_avx2_1x1_convolution.cpp b/src/cpu/x64/jit_avx2_1x1_convolution.cpp index 9aa8a335698..4a0a3182370 100644 --- a/src/cpu/x64/jit_avx2_1x1_convolution.cpp +++ b/src/cpu/x64/jit_avx2_1x1_convolution.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2016-2023 Intel Corporation +* Copyright 2016-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -129,7 +129,7 @@ void jit_avx2_1x1_convolution_fwd_t::execute_forward_thr(const int ithr, const int nb_buffer = jcp.nb_load_blocking; auto jcp_dw = pd()->jcp_dw_; std::vector addrs; - jit_generator *dw_jit_ker = nullptr; + jit_generator_t *dw_jit_ker = nullptr; const bool is_src_layout_nxc = utils::one_of( jcp.src_tag, format_tag::nwc, format_tag::nhwc, format_tag::ndhwc); diff --git a/src/cpu/x64/jit_avx2_conv_kernel_f32.cpp b/src/cpu/x64/jit_avx2_conv_kernel_f32.cpp index 86de656b737..eaa916117e3 100644 --- a/src/cpu/x64/jit_avx2_conv_kernel_f32.cpp +++ b/src/cpu/x64/jit_avx2_conv_kernel_f32.cpp @@ -53,7 +53,7 @@ bool tag_is_flat(format_tag_t tag, format_tag_t ncx, format_tag_t nxc, int ic) { jit_avx2_conv_fwd_kernel_f32::jit_avx2_conv_fwd_kernel_f32( const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name(), avx2), jcp(ajcp), attr_(attr) { + : jit_generator_t(jit_name(), avx2), jcp(ajcp), attr_(attr) { if (jcp.with_eltwise || jcp.with_binary) { using namespace binary_injector; static constexpr bool preserve_gpr = true; diff --git a/src/cpu/x64/jit_avx2_conv_kernel_f32.hpp b/src/cpu/x64/jit_avx2_conv_kernel_f32.hpp index 72fab8e4758..e4e5b77ac74 100644 --- a/src/cpu/x64/jit_avx2_conv_kernel_f32.hpp +++ b/src/cpu/x64/jit_avx2_conv_kernel_f32.hpp @@ -30,7 +30,7 @@ namespace impl { namespace cpu { namespace x64 { -struct jit_avx2_conv_fwd_kernel_f32 : public jit_generator { +struct jit_avx2_conv_fwd_kernel_f32 : public jit_generator_t { jit_avx2_conv_fwd_kernel_f32(const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md); @@ -145,11 +145,11 @@ struct jit_avx2_conv_fwd_kernel_f32 : public jit_generator { void generate() override; }; -struct jit_avx2_conv_bwd_data_kernel_f32 : public jit_generator { +struct jit_avx2_conv_bwd_data_kernel_f32 : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_conv_bwd_data_kernel_f32) jit_avx2_conv_bwd_data_kernel_f32(const jit_conv_conf_t &ajcp) - : jit_generator(jit_name()), jcp(ajcp) {} + : jit_generator_t(jit_name()), jcp(ajcp) {} static status_t init_conf(jit_conv_conf_t &jcp, const convolution_desc_t &cd, const memory_desc_wrapper &diff_src_d, @@ -256,11 +256,11 @@ struct jit_avx2_conv_bwd_data_kernel_f32 : public jit_generator { } }; -struct jit_avx2_conv_bwd_weights_kernel_f32 : public jit_generator { +struct jit_avx2_conv_bwd_weights_kernel_f32 : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx2_conv_bwd_weights_kernel_f32) jit_avx2_conv_bwd_weights_kernel_f32(const jit_conv_conf_t &ajcp) - : jit_generator(jit_name()), jcp(ajcp) {} + : jit_generator_t(jit_name()), jcp(ajcp) {} static status_t init_conf(jit_conv_conf_t &jcp, const convolution_desc_t &cd, const memory_desc_wrapper &src_d, diff --git a/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.cpp b/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.cpp index ef3dbfddea4..d14c6068a4a 100644 --- a/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.cpp @@ -50,7 +50,7 @@ using namespace Xbyak; jit_avx512_common_1x1_conv_kernel::jit_avx512_common_1x1_conv_kernel( const jit_1x1_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name()), jcp(ajcp), attr_(attr) { + : jit_generator_t(jit_name()), jcp(ajcp), attr_(attr) { if (jcp.with_eltwise || jcp.with_binary) { using namespace binary_injector; static constexpr bool preserve_gpr = true; diff --git a/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.hpp b/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.hpp index 64dd8792ca5..894efbbfbee 100644 --- a/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_common_1x1_conv_kernel.hpp @@ -29,7 +29,7 @@ namespace impl { namespace cpu { namespace x64 { -struct jit_avx512_common_1x1_conv_kernel : public jit_generator { +struct jit_avx512_common_1x1_conv_kernel : public jit_generator_t { jit_avx512_common_1x1_conv_kernel(const jit_1x1_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md); diff --git a/src/cpu/x64/jit_avx512_common_conv_kernel.cpp b/src/cpu/x64/jit_avx512_common_conv_kernel.cpp index 71004d8f505..fae3fb399d5 100644 --- a/src/cpu/x64/jit_avx512_common_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_common_conv_kernel.cpp @@ -96,7 +96,7 @@ template _jit_avx512_common_conv_fwd_kernel::_jit_avx512_common_conv_fwd_kernel( const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name()), jcp(ajcp), attr_(attr) { + : jit_generator_t(jit_name()), jcp(ajcp), attr_(attr) { if (jcp.with_eltwise || jcp.with_binary) { using namespace binary_injector; static constexpr bool preserve_gpr = true; diff --git a/src/cpu/x64/jit_avx512_common_conv_kernel.hpp b/src/cpu/x64/jit_avx512_common_conv_kernel.hpp index 3a4e0f6d7ab..765ff8819d5 100644 --- a/src/cpu/x64/jit_avx512_common_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_common_conv_kernel.hpp @@ -31,7 +31,7 @@ namespace cpu { namespace x64 { template -struct _jit_avx512_common_conv_fwd_kernel : public jit_generator { +struct _jit_avx512_common_conv_fwd_kernel : public jit_generator_t { _jit_avx512_common_conv_fwd_kernel(const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md); @@ -221,14 +221,14 @@ struct jit_avx512_common_conv_fwd_kernel { private: DNNL_DISALLOW_COPY_AND_ASSIGN(jit_avx512_common_conv_fwd_kernel); - std::unique_ptr kernel_; + std::unique_ptr kernel_; }; template -struct _jit_avx512_common_conv_bwd_data_kernel_f32 : public jit_generator { +struct _jit_avx512_common_conv_bwd_data_kernel_f32 : public jit_generator_t { _jit_avx512_common_conv_bwd_data_kernel_f32(const jit_conv_conf_t &ajcp) - : jit_generator(jit_name()), jcp(ajcp) {} + : jit_generator_t(jit_name()), jcp(ajcp) {} DECLARE_CPU_JIT_AUX_FUNCTIONS(_jit_avx512_common_conv_bwd_data_kernel_f32) jit_conv_conf_t jcp; @@ -389,13 +389,13 @@ struct jit_avx512_common_conv_bwd_data_kernel_f32 { private: DNNL_DISALLOW_COPY_AND_ASSIGN(jit_avx512_common_conv_bwd_data_kernel_f32); - std::unique_ptr kernel_; + std::unique_ptr kernel_; }; -struct jit_avx512_common_conv_bwd_weights_kernel_f32 : public jit_generator { +struct jit_avx512_common_conv_bwd_weights_kernel_f32 : public jit_generator_t { jit_avx512_common_conv_bwd_weights_kernel_f32(const jit_conv_conf_t &ajcp) - : jit_generator(jit_name()), jcp(ajcp) {} + : jit_generator_t(jit_name()), jcp(ajcp) {} void generate() override { if (jcp.harness != harness_nxc) diff --git a/src/cpu/x64/jit_avx512_core_amx_1x1_conv_kernel.cpp b/src/cpu/x64/jit_avx512_core_amx_1x1_conv_kernel.cpp index 57b43005bb1..68d70310bd1 100644 --- a/src/cpu/x64/jit_avx512_core_amx_1x1_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_core_amx_1x1_conv_kernel.cpp @@ -41,7 +41,7 @@ using namespace Xbyak; jit_avx512_core_amx_1x1_fwd_kernel_t::jit_avx512_core_amx_1x1_fwd_kernel_t( const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name(), avx512_core_amx), jcp(ajcp), attr_(attr) { + : jit_generator_t(jit_name(), avx512_core_amx), jcp(ajcp), attr_(attr) { if (jcp.with_eltwise || jcp.with_binary || jcp.with_sum) { using namespace binary_injector; const auto &rhs_addr_reg = bin_injector_helper_reg_1; diff --git a/src/cpu/x64/jit_avx512_core_amx_1x1_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_amx_1x1_conv_kernel.hpp index 8bc8cdb3cf8..207268cc302 100644 --- a/src/cpu/x64/jit_avx512_core_amx_1x1_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_amx_1x1_conv_kernel.hpp @@ -29,7 +29,7 @@ namespace impl { namespace cpu { namespace x64 { -struct jit_avx512_core_amx_1x1_fwd_kernel_t : public jit_generator { +struct jit_avx512_core_amx_1x1_fwd_kernel_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_amx_1x1_fwd_kernel_t) jit_avx512_core_amx_1x1_fwd_kernel_t(const jit_conv_conf_t &ajcp, diff --git a/src/cpu/x64/jit_avx512_core_amx_conv_kernel.cpp b/src/cpu/x64/jit_avx512_core_amx_conv_kernel.cpp index a7741a0c563..1b31b3eccfd 100644 --- a/src/cpu/x64/jit_avx512_core_amx_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_core_amx_conv_kernel.cpp @@ -1063,7 +1063,7 @@ void jit_avx512_core_amx_copy_to_pbuffer_t::generate() { jit_avx512_core_amx_fwd_kernel_t::jit_avx512_core_amx_fwd_kernel_t( const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name(), avx512_core_amx), jcp(ajcp), attr_(attr) { + : jit_generator_t(jit_name(), avx512_core_amx), jcp(ajcp), attr_(attr) { if (jcp.with_eltwise || jcp.with_binary || jcp.with_sum) { using namespace binary_injector; const auto &rhs_addr_reg = bin_injector_helper_reg_1; @@ -1103,7 +1103,7 @@ status_t jit_avx512_core_amx_fwd_kernel_t::create_kernel() { && IMPLICATION(jcp.is_relo, copy_to_wbuffer_); if (!allocation_ok) return status::out_of_memory; - CHECK(jit_generator::create_kernel()); + CHECK(jit_generator_t::create_kernel()); CHECK(copy_to_pbuffer_->create_kernel()); if (jcp.is_relo) CHECK(copy_to_wbuffer_->create_kernel()); if (jcp.req_zero_point_buffer) { diff --git a/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp index de19a179d1f..8779e9be196 100644 --- a/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_amx_conv_kernel.hpp @@ -36,14 +36,14 @@ namespace x64 { /* This struct computes the compensation for src_zero_point related to * padding */ -struct jit_avx512_core_amx_compute_zp_pbuff_t : public jit_generator { +struct jit_avx512_core_amx_compute_zp_pbuff_t : public jit_generator_t { using reg64_t = const Xbyak::Reg64; DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_amx_compute_zp_pbuff_t) jit_avx512_core_amx_compute_zp_pbuff_t(const jit_conv_conf_t &ajcp) - : jit_generator(jit_name(), avx512_core_amx), jcp(ajcp) {} + : jit_generator_t(jit_name(), avx512_core_amx), jcp(ajcp) {} static const int max_regs_ur = 30; @@ -104,13 +104,13 @@ struct jit_avx512_core_amx_compute_zp_pbuff_t : public jit_generator { } }; -struct jit_avx512_core_amx_copy_to_wbuffer_t : public jit_generator { +struct jit_avx512_core_amx_copy_to_wbuffer_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_amx_copy_to_wbuffer_t) using reg64_t = Xbyak::Reg64; jit_avx512_core_amx_copy_to_wbuffer_t(const jit_conv_conf_t &ajcp) - : jit_generator(jit_name(), avx512_core_amx), jcp(ajcp) {} + : jit_generator_t(jit_name(), avx512_core_amx), jcp(ajcp) {} private: jit_conv_conf_t jcp; @@ -129,13 +129,13 @@ struct jit_avx512_core_amx_copy_to_wbuffer_t : public jit_generator { void generate() override; }; -struct jit_avx512_core_amx_copy_to_pbuffer_t : public jit_generator { +struct jit_avx512_core_amx_copy_to_pbuffer_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_amx_copy_to_pbuffer_t) using reg64_t = Xbyak::Reg64; jit_avx512_core_amx_copy_to_pbuffer_t(const jit_conv_conf_t &ajcp) - : jit_generator(jit_name(), avx512_core_amx), jcp(ajcp) {} + : jit_generator_t(jit_name(), avx512_core_amx), jcp(ajcp) {} private: jit_conv_conf_t jcp; @@ -185,7 +185,7 @@ struct jit_avx512_core_amx_copy_to_pbuffer_t : public jit_generator { void copy_row_reduced_lowering(); }; -struct jit_avx512_core_amx_fwd_kernel_t : public jit_generator { +struct jit_avx512_core_amx_fwd_kernel_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_amx_fwd_kernel_t) jit_avx512_core_amx_fwd_kernel_t(const jit_conv_conf_t &ajcp, @@ -420,13 +420,13 @@ struct jit_avx512_core_amx_fwd_kernel_t : public jit_generator { void generate() override; }; -struct jit_avx512_core_amx_bwd_data_copy_kernel_t : public jit_generator { +struct jit_avx512_core_amx_bwd_data_copy_kernel_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_amx_bwd_data_copy_kernel_t) using reg64_t = Xbyak::Reg64; jit_avx512_core_amx_bwd_data_copy_kernel_t(jit_conv_conf_t &ajcp) - : jit_generator(jit_name(), avx512_core_amx), jcp(ajcp) {} + : jit_generator_t(jit_name(), avx512_core_amx), jcp(ajcp) {} private: jit_conv_conf_t jcp; @@ -466,12 +466,12 @@ struct jit_avx512_core_amx_bwd_data_copy_kernel_t : public jit_generator { void kd_loop(bool is_masked); }; -struct jit_avx512_core_amx_bwd_data_kernel_t : public jit_generator { +struct jit_avx512_core_amx_bwd_data_kernel_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_amx_bwd_data_kernel_t) jit_avx512_core_amx_bwd_data_kernel_t( const jit_conv_conf_t &ajcp, const primitive_attr_t &attr) - : jit_generator(jit_name(), avx512_core_amx) + : jit_generator_t(jit_name(), avx512_core_amx) , jcp(ajcp) , attr_(attr) , eltwise_injector_(nullptr) @@ -483,7 +483,7 @@ struct jit_avx512_core_amx_bwd_data_kernel_t : public jit_generator { jit_avx512_core_amx_bwd_data_copy_kernel_t>(jcp); } status_t create_kernel() override { - CHECK(jit_generator::create_kernel()); + CHECK(jit_generator_t::create_kernel()); CHECK(bwd_data_copy_kernel_->create_kernel()); return status::success; } @@ -609,10 +609,10 @@ struct jit_avx512_core_amx_bwd_data_kernel_t : public jit_generator { void generate() override; }; -struct jit_avx512_core_amx_bwd_weights_kernel_t : public jit_generator { +struct jit_avx512_core_amx_bwd_weights_kernel_t : public jit_generator_t { jit_avx512_core_amx_bwd_weights_kernel_t(const jit_conv_conf_t &ajcp) - : jit_generator(jit_name(), avx512_core_amx), jcp(ajcp) {} + : jit_generator_t(jit_name(), avx512_core_amx), jcp(ajcp) {} ~jit_avx512_core_amx_bwd_weights_kernel_t() override = default; @@ -748,10 +748,10 @@ struct jit_avx512_core_amx_bwd_weights_kernel_t : public jit_generator { int ddst_save_offset = 0; }; -struct jit_avx512_core_amx_bwd_bias_kernel_t : public jit_generator { +struct jit_avx512_core_amx_bwd_bias_kernel_t : public jit_generator_t { jit_avx512_core_amx_bwd_bias_kernel_t(const jit_conv_conf_t &ajcp) - : jit_generator(jit_name(), avx512_core_amx), jcp(ajcp) {} + : jit_generator_t(jit_name(), avx512_core_amx), jcp(ajcp) {} ~jit_avx512_core_amx_bwd_bias_kernel_t() override = default; diff --git a/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.cpp b/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.cpp index 85653c984ab..7f68a40ad2e 100644 --- a/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.cpp @@ -45,7 +45,7 @@ using namespace Xbyak; jit_avx512_core_bf16_1x1_conv_kernel::jit_avx512_core_bf16_1x1_conv_kernel( const jit_1x1_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name(), avx512_core_bf16), jcp(ajcp), attr_(attr) { + : jit_generator_t(jit_name(), avx512_core_bf16), jcp(ajcp), attr_(attr) { if (jcp.with_eltwise || jcp.with_binary) { using namespace binary_injector; static constexpr bool preserve_gpr = true; diff --git a/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.hpp index 384c474132f..12e65fe7812 100644 --- a/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_bf16_1x1_conv_kernel.hpp @@ -29,7 +29,7 @@ namespace impl { namespace cpu { namespace x64 { -struct jit_avx512_core_bf16_1x1_conv_kernel : public jit_generator { +struct jit_avx512_core_bf16_1x1_conv_kernel : public jit_generator_t { jit_avx512_core_bf16_1x1_conv_kernel(const jit_1x1_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md); diff --git a/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.cpp b/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.cpp index 503da652a16..b406b7086c9 100644 --- a/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.cpp @@ -100,7 +100,7 @@ template _jit_avx512_core_bf16_fwd_kernel::_jit_avx512_core_bf16_fwd_kernel( const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name(), avx512_core_bf16), jcp(ajcp), attr_(attr) { + : jit_generator_t(jit_name(), avx512_core_bf16), jcp(ajcp), attr_(attr) { if (jcp.with_eltwise || jcp.with_binary) { using namespace binary_injector; static constexpr bool preserve_gpr = true; diff --git a/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.hpp index 1acfbb7f4c2..dd194185448 100644 --- a/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_bf16_conv_kernel.hpp @@ -32,7 +32,7 @@ namespace cpu { namespace x64 { template -struct _jit_avx512_core_bf16_fwd_kernel : public jit_generator { +struct _jit_avx512_core_bf16_fwd_kernel : public jit_generator_t { _jit_avx512_core_bf16_fwd_kernel(const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md); @@ -256,14 +256,14 @@ struct jit_avx512_core_bf16_fwd_kernel { private: DNNL_DISALLOW_COPY_AND_ASSIGN(jit_avx512_core_bf16_fwd_kernel); - std::unique_ptr kernel_; + std::unique_ptr kernel_; }; template -struct _jit_avx512_core_bf16_bwd_data_kernel : public jit_generator { +struct _jit_avx512_core_bf16_bwd_data_kernel : public jit_generator_t { _jit_avx512_core_bf16_bwd_data_kernel(const jit_conv_conf_t &ajcp) - : jit_generator(jit_name(), avx512_core_bf16) + : jit_generator_t(jit_name(), avx512_core_bf16) , jcp(ajcp) , bf16_emu_(nullptr) { if (!isa_has_bf16(jcp.isa)) @@ -469,14 +469,15 @@ struct jit_avx512_core_bf16_bwd_data_kernel { private: DNNL_DISALLOW_COPY_AND_ASSIGN(jit_avx512_core_bf16_bwd_data_kernel); - std::unique_ptr kernel_; + std::unique_ptr kernel_; }; -struct jit_avx512_core_bf16_conv_bwd_weights_kernel_f32 : public jit_generator { +struct jit_avx512_core_bf16_conv_bwd_weights_kernel_f32 + : public jit_generator_t { jit_avx512_core_bf16_conv_bwd_weights_kernel_f32( const jit_conv_conf_t &ajcp) - : jit_generator(jit_name(), avx512_core_bf16) + : jit_generator_t(jit_name(), avx512_core_bf16) , jcp(ajcp) , bf16_emu_(nullptr) { if (!isa_has_bf16(jcp.isa)) { diff --git a/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.cpp b/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.cpp index c03263eef75..3208bbe4e7f 100644 --- a/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.cpp @@ -35,7 +35,7 @@ using namespace dnnl::impl::utils; jit_avx512_dw_conv_fwd_kernel_bf16::jit_avx512_dw_conv_fwd_kernel_bf16( const jit_conv_conf_t &ajcp, const memory_desc_t &dst_md) - : jit_generator(jit_name()), jcp(ajcp) { + : jit_generator_t(jit_name()), jcp(ajcp) { if (jcp.with_eltwise || jcp.with_binary) { using namespace binary_injector; static constexpr bool preserve_gpr = true; diff --git a/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.hpp index d0361bcd0de..a29988c1ed4 100644 --- a/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_bf16_dw_conv_kernel.hpp @@ -32,7 +32,7 @@ namespace impl { namespace cpu { namespace x64 { -struct jit_avx512_dw_conv_fwd_kernel_bf16 : public jit_generator { +struct jit_avx512_dw_conv_fwd_kernel_bf16 : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_dw_conv_fwd_kernel_bf16) jit_avx512_dw_conv_fwd_kernel_bf16( @@ -127,11 +127,11 @@ struct jit_avx512_dw_conv_fwd_kernel_bf16 : public jit_generator { void generate() override; }; -struct jit_avx512_dw_conv_bwd_data_kernel_bf16 : public jit_generator { +struct jit_avx512_dw_conv_bwd_data_kernel_bf16 : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_dw_conv_bwd_data_kernel_bf16) jit_avx512_dw_conv_bwd_data_kernel_bf16(const jit_conv_conf_t &ajcp) - : jit_generator(jit_name()), jcp(ajcp), bf16_emu_(nullptr) { + : jit_generator_t(jit_name()), jcp(ajcp), bf16_emu_(nullptr) { if (!isa_has_bf16(jcp.isa)) bf16_emu_ = utils::make_unique(this, @@ -205,12 +205,12 @@ struct jit_avx512_dw_conv_bwd_data_kernel_bf16 : public jit_generator { DNNL_DISALLOW_COPY_AND_ASSIGN(jit_avx512_dw_conv_bwd_data_kernel_bf16); }; -struct jit_avx512_dw_conv_bwd_weights_kernel_bf16 : public jit_generator { +struct jit_avx512_dw_conv_bwd_weights_kernel_bf16 : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_dw_conv_bwd_weights_kernel_bf16) jit_avx512_dw_conv_bwd_weights_kernel_bf16(const jit_conv_conf_t &ajcp) - : jit_generator(jit_name()), jcp(ajcp), bf16_emu_(nullptr) { + : jit_generator_t(jit_name()), jcp(ajcp), bf16_emu_(nullptr) { if (!isa_has_bf16(jcp.isa)) bf16_emu_ = utils::make_unique(this, diff --git a/src/cpu/x64/jit_avx512_core_bf16cvt.hpp b/src/cpu/x64/jit_avx512_core_bf16cvt.hpp index ecb7529bdbd..fe1d951d819 100644 --- a/src/cpu/x64/jit_avx512_core_bf16cvt.hpp +++ b/src/cpu/x64/jit_avx512_core_bf16cvt.hpp @@ -53,8 +53,8 @@ struct bf16_emulation_t { using Xmm_t = const Xbyak::Xmm; using reg64_t = const Xbyak::Reg64; - bf16_emulation_t(jit_generator *host, Zmm_t one, Zmm_t even, Zmm_t selector, - reg64_t scratch, Zmm_t tr0, Zmm_t tr1) + bf16_emulation_t(jit_generator_t *host, Zmm_t one, Zmm_t even, + Zmm_t selector, reg64_t scratch, Zmm_t tr0, Zmm_t tr1) : host_(host) , one_(one) , even_(even) @@ -63,8 +63,8 @@ struct bf16_emulation_t { , tr0_(tr0) , tr1_(tr1) {} - bf16_emulation_t(jit_generator *host, Zmm_t one, Zmm_t even, Zmm_t selector, - reg64_t scratch, Zmm_t tr0) + bf16_emulation_t(jit_generator_t *host, Zmm_t one, Zmm_t even, + Zmm_t selector, reg64_t scratch, Zmm_t tr0) : bf16_emulation_t(host, one, even, selector, scratch, tr0, tr0) {} void vdpbf16ps(Zmm_t &acc, Zmm_t wei, Zmm_t inp) { @@ -151,7 +151,7 @@ struct bf16_emulation_t { static cpu_isa_t get_isa() { return avx512_core; } private: - jit_generator *const host_; + jit_generator_t *const host_; Zmm_t one_; Zmm_t even_; Zmm_t selector_; @@ -175,11 +175,11 @@ struct bf16_emulation_t { // performs element-by-element sum of inp and add float arrays and stores // result to bfloat16 out array with downconversion -struct jit_avx512_core_add_cvt_ps_to_bf16_t : public jit_generator { +struct jit_avx512_core_add_cvt_ps_to_bf16_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_add_cvt_ps_to_bf16) jit_avx512_core_add_cvt_ps_to_bf16_t() - : jit_generator(jit_name()), simd_w_(16) { + : jit_generator_t(jit_name()), simd_w_(16) { bf16_emu_ = utils::make_unique( this, one, even, selector, scratch, fp32_tmp, fp32_tmp); @@ -253,7 +253,7 @@ struct jit_avx512_core_add_cvt_ps_to_bf16_t : public jit_generator { } void operator()(bf16_support::jit_call_t *params) const { - jit_generator::operator()(params); + jit_generator_t::operator()(params); msan_unpoison(params->out, params->nelems * sizeof(bfloat16_t)); } @@ -288,14 +288,14 @@ struct jit_avx512_core_add_cvt_ps_to_bf16_t : public jit_generator { // it is required for quick implementation of 1x1 bf16 bwd_w jit kernel // w/o using permw instruction inside // TODO: consider modification/replacement for outer transformation jit kernel -struct jit_avx512_core_bf16_reorder_s16c_to_S16c2s_t : public jit_generator { +struct jit_avx512_core_bf16_reorder_s16c_to_S16c2s_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_bf16_reorder_s16c_to_S16c2s) jit_avx512_core_bf16_reorder_s16c_to_S16c2s_t() - : jit_generator(jit_name()), simd_w_(16), in_stride_(16) {} + : jit_generator_t(jit_name()), simd_w_(16), in_stride_(16) {} jit_avx512_core_bf16_reorder_s16c_to_S16c2s_t(int in_stride) - : jit_generator(jit_name()), simd_w_(16), in_stride_(in_stride) {} + : jit_generator_t(jit_name()), simd_w_(16), in_stride_(in_stride) {} ~jit_avx512_core_bf16_reorder_s16c_to_S16c2s_t() override = default; @@ -379,7 +379,7 @@ struct jit_avx512_core_bf16_reorder_s16c_to_S16c2s_t : public jit_generator { } void operator()(bf16_support::jit_call_t *params) const { - jit_generator::operator()(params); + jit_generator_t::operator()(params); msan_unpoison(params->out, params->nelems * sizeof(bfloat16_t)); } diff --git a/src/cpu/x64/jit_avx512_core_fp16cvt.hpp b/src/cpu/x64/jit_avx512_core_fp16cvt.hpp index e61433f4383..4580c1f2930 100644 --- a/src/cpu/x64/jit_avx512_core_fp16cvt.hpp +++ b/src/cpu/x64/jit_avx512_core_fp16cvt.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2022 Intel Corporation +* Copyright 2022-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -45,18 +45,18 @@ struct jit_call_t { // performs element-by-element sum of inp and add float arrays and stores // result to float16 out array with downconversion -struct jit_avx512_core_fp16_add_cvt_ps_to_f16_t : public jit_generator { +struct jit_avx512_core_fp16_add_cvt_ps_to_f16_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_add_cvt_ps_to_f16) jit_avx512_core_fp16_add_cvt_ps_to_f16_t() - : jit_generator(jit_name()), simd_w_(16) { + : jit_generator_t(jit_name()), simd_w_(16) { create_kernel(); } void generate() override; void operator()(f16_support::jit_call_t *params) const { - jit_generator::operator()(params); + jit_generator_t::operator()(params); msan_unpoison(params->out, params->nelems * sizeof(float16_t)); } diff --git a/src/cpu/x64/jit_avx512_core_fp8cvt.cpp b/src/cpu/x64/jit_avx512_core_fp8cvt.cpp index 6b6c433f3eb..80abf550ce0 100644 --- a/src/cpu/x64/jit_avx512_core_fp8cvt.cpp +++ b/src/cpu/x64/jit_avx512_core_fp8cvt.cpp @@ -485,7 +485,7 @@ void fp8_emulation_e4m3_t::tabulate(const data_type_t dt, } jit_cvt_fp8_t::jit_cvt_fp8_t(f32_convert_mode_t mode) - : jit_generator(jit_name(), avx512_core_fp16), mode_(mode) { + : jit_generator_t(jit_name(), avx512_core_fp16), mode_(mode) { switch (mode) { case f8_e5m2_to_f16: case f8_e5m2_to_f32: diff --git a/src/cpu/x64/jit_avx512_core_fp8cvt.hpp b/src/cpu/x64/jit_avx512_core_fp8cvt.hpp index d8f782ffe4c..5fcc2ccca26 100644 --- a/src/cpu/x64/jit_avx512_core_fp8cvt.hpp +++ b/src/cpu/x64/jit_avx512_core_fp8cvt.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2024 Intel Corporation +* Copyright 2024-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,7 +28,7 @@ namespace cpu { namespace x64 { struct fp8_emulation_base_t { - fp8_emulation_base_t(jit_generator *host, const Xbyak::Xmm &xmm_aux1, + fp8_emulation_base_t(jit_generator_t *host, const Xbyak::Xmm &xmm_aux1, const Xbyak::Xmm &xmm_aux2, const Xbyak::Xmm &xmm_aux3, const Xbyak::Reg64 reg64_aux) : host_(host) @@ -68,7 +68,7 @@ struct fp8_emulation_base_t { const Xbyak::Xmm &xmm_out, const Xbyak::Operand &op_in); protected: - jit_generator *const host_; + jit_generator_t *const host_; Xbyak::Label label_table_to_f8_; Xbyak::Label label_vnni_permute_index_table_; const Xbyak::Xmm xmm_aux1_; @@ -102,7 +102,7 @@ struct fp8_emulation_base_t { }; struct fp8_emulation_e5m2_t : public fp8_emulation_base_t { - fp8_emulation_e5m2_t(jit_generator *host, const Xbyak::Xmm &xmm_aux1, + fp8_emulation_e5m2_t(jit_generator_t *host, const Xbyak::Xmm &xmm_aux1, const Xbyak::Xmm &xmm_aux2, const Xbyak::Xmm &xmm_aux3, const Xbyak::Opmask kmask_aux_, const Xbyak::Reg64 reg64_aux) : fp8_emulation_base_t(host, xmm_aux1, xmm_aux2, xmm_aux3, reg64_aux) @@ -136,7 +136,7 @@ struct fp8_emulation_e5m2_t : public fp8_emulation_base_t { }; struct fp8_emulation_e4m3_t : public fp8_emulation_base_t { - fp8_emulation_e4m3_t(jit_generator *host, const Xbyak::Xmm &xmm_aux1, + fp8_emulation_e4m3_t(jit_generator_t *host, const Xbyak::Xmm &xmm_aux1, const Xbyak::Xmm &xmm_aux2, const Xbyak::Xmm &xmm_aux3, const Xbyak::Xmm &xmm_aux4, const Xbyak::Xmm &xmm_aux5, const Xbyak::Reg64 reg64_aux) @@ -187,7 +187,7 @@ enum f32_convert_mode_t { f32_to_f16, }; -struct jit_cvt_fp8_t : public jit_generator { +struct jit_cvt_fp8_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_cvt_fp8_t) jit_cvt_fp8_t(f32_convert_mode_t mode); diff --git a/src/cpu/x64/jit_avx512_core_resampling.cpp b/src/cpu/x64/jit_avx512_core_resampling.cpp index 0ced081dfa0..91eaa631bcf 100644 --- a/src/cpu/x64/jit_avx512_core_resampling.cpp +++ b/src/cpu/x64/jit_avx512_core_resampling.cpp @@ -801,7 +801,7 @@ struct jit_avx512_core_resampling_kernel_t jit_avx512_core_resampling_kernel_base_t:: jit_avx512_core_resampling_kernel_base_t( const resampling_pd_t *pd, const char *name) - : jit_generator(name), pd_(pd) {} + : jit_generator_t(name), pd_(pd) {} data_type_t jit_avx512_core_resampling_kernel_base_t::src_data_type() const { if (pd_->is_fwd()) diff --git a/src/cpu/x64/jit_avx512_core_resampling.hpp b/src/cpu/x64/jit_avx512_core_resampling.hpp index 35e153c2d39..5e034ae8056 100644 --- a/src/cpu/x64/jit_avx512_core_resampling.hpp +++ b/src/cpu/x64/jit_avx512_core_resampling.hpp @@ -32,7 +32,7 @@ namespace x64 { struct jit_resampling_args_t; -struct jit_avx512_core_resampling_kernel_base_t : public jit_generator { +struct jit_avx512_core_resampling_kernel_base_t : public jit_generator_t { jit_avx512_core_resampling_kernel_base_t( const resampling_pd_t *pd, const char *name); ~jit_avx512_core_resampling_kernel_base_t() override = default; diff --git a/src/cpu/x64/jit_avx512_core_scale_precompute.hpp b/src/cpu/x64/jit_avx512_core_scale_precompute.hpp index 8f74ad4dc4c..66c2a80663d 100644 --- a/src/cpu/x64/jit_avx512_core_scale_precompute.hpp +++ b/src/cpu/x64/jit_avx512_core_scale_precompute.hpp @@ -62,13 +62,13 @@ const float *precompute_scales(const memory_tracking::grantor_t &scratchpad, float scale_adjust_factor = 1.0f, bool req_transpose = false); } // namespace scale_utils -struct jit_avx512_core_scale_precompute_t : public jit_generator { +struct jit_avx512_core_scale_precompute_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_scale_precompute_t) jit_avx512_core_scale_precompute_t( const primitive_attr_t *attr, const float scale_adjust_factor = 1) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , attr_(attr) , with_wei_scales_(!attr_->scales_.has_default_values(DNNL_ARG_WEIGHTS)) , wei_scales_dt_(with_wei_scales_ @@ -82,7 +82,7 @@ struct jit_avx512_core_scale_precompute_t : public jit_generator { void generate() override; void operator()(scale_utils::jit_call_t *params) const { - jit_generator::operator()(params); + jit_generator_t::operator()(params); msan_unpoison(params->scales_, params->nelems_ * sizeof(float)); } diff --git a/src/cpu/x64/jit_avx512_core_x8s8s32x_1x1_conv_kernel.cpp b/src/cpu/x64/jit_avx512_core_x8s8s32x_1x1_conv_kernel.cpp index 69fb6dca881..cd0f2085150 100644 --- a/src/cpu/x64/jit_avx512_core_x8s8s32x_1x1_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_core_x8s8s32x_1x1_conv_kernel.cpp @@ -49,7 +49,7 @@ _jit_avx512_core_x8s8s32x_1x1_conv_kernel:: _jit_avx512_core_x8s8s32x_1x1_conv_kernel( const jit_1x1_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , jcp(ajcp) , attr_(attr) , postops_injector_(nullptr) { diff --git a/src/cpu/x64/jit_avx512_core_x8s8s32x_1x1_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_x8s8s32x_1x1_conv_kernel.hpp index 24b9bd63760..1fe08b8e472 100644 --- a/src/cpu/x64/jit_avx512_core_x8s8s32x_1x1_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_x8s8s32x_1x1_conv_kernel.hpp @@ -32,7 +32,7 @@ namespace cpu { namespace x64 { template -struct _jit_avx512_core_x8s8s32x_1x1_conv_kernel : public jit_generator { +struct _jit_avx512_core_x8s8s32x_1x1_conv_kernel : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(_jit_avx512_core_x8s8s32x_1x1_conv_fwd_ker_t) _jit_avx512_core_x8s8s32x_1x1_conv_kernel(const jit_1x1_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md); @@ -194,7 +194,7 @@ struct jit_avx512_core_x8s8s32x_1x1_conv_kernel { private: DNNL_DISALLOW_COPY_AND_ASSIGN(jit_avx512_core_x8s8s32x_1x1_conv_kernel); - std::unique_ptr kernel_; + std::unique_ptr kernel_; }; } // namespace x64 diff --git a/src/cpu/x64/jit_avx512_core_x8s8s32x_conv_kernel.cpp b/src/cpu/x64/jit_avx512_core_x8s8s32x_conv_kernel.cpp index de880ba11a8..8ca87ba8cfb 100644 --- a/src/cpu/x64/jit_avx512_core_x8s8s32x_conv_kernel.cpp +++ b/src/cpu/x64/jit_avx512_core_x8s8s32x_conv_kernel.cpp @@ -57,7 +57,7 @@ template _jit_avx512_core_x8s8s32x_fwd_kernel::_jit_avx512_core_x8s8s32x_fwd_kernel( const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name(), ajcp.isa) + : jit_generator_t(jit_name(), ajcp.isa) , jcp(ajcp) , attr_(attr) , postops_injector_(nullptr) { diff --git a/src/cpu/x64/jit_avx512_core_x8s8s32x_conv_kernel.hpp b/src/cpu/x64/jit_avx512_core_x8s8s32x_conv_kernel.hpp index 32437a3315a..a77ad0c5e4c 100644 --- a/src/cpu/x64/jit_avx512_core_x8s8s32x_conv_kernel.hpp +++ b/src/cpu/x64/jit_avx512_core_x8s8s32x_conv_kernel.hpp @@ -32,7 +32,7 @@ namespace cpu { namespace x64 { template -struct _jit_avx512_core_x8s8s32x_fwd_kernel : public jit_generator { +struct _jit_avx512_core_x8s8s32x_fwd_kernel : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(_jit_avx512_core_x8s8s32x_conv_fwd_ker_t) enum { STATE_FIRST_DST_LOAD = 0x1U }; @@ -283,7 +283,7 @@ struct jit_avx512_core_x8s8s32x_fwd_kernel { private: DNNL_DISALLOW_COPY_AND_ASSIGN(jit_avx512_core_x8s8s32x_fwd_kernel) - std::unique_ptr kernel_; + std::unique_ptr kernel_; }; } // namespace x64 diff --git a/src/cpu/x64/jit_avx512_core_x8s8s32x_deconvolution.cpp b/src/cpu/x64/jit_avx512_core_x8s8s32x_deconvolution.cpp index e49a02b17ec..f4acaffd478 100644 --- a/src/cpu/x64/jit_avx512_core_x8s8s32x_deconvolution.cpp +++ b/src/cpu/x64/jit_avx512_core_x8s8s32x_deconvolution.cpp @@ -43,7 +43,7 @@ template jit_avx512_core_x8s8s32x_deconv_fwd_kernel:: jit_avx512_core_x8s8s32x_deconv_fwd_kernel(const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , jcp(ajcp) , attr_(attr) , postops_injector_(nullptr) { diff --git a/src/cpu/x64/jit_avx512_core_x8s8s32x_deconvolution.hpp b/src/cpu/x64/jit_avx512_core_x8s8s32x_deconvolution.hpp index 2cc1c458168..2c2ff28aed6 100644 --- a/src/cpu/x64/jit_avx512_core_x8s8s32x_deconvolution.hpp +++ b/src/cpu/x64/jit_avx512_core_x8s8s32x_deconvolution.hpp @@ -72,7 +72,7 @@ struct ur_w_blks_params_t { }; template -struct jit_avx512_core_x8s8s32x_deconv_fwd_kernel : public jit_generator { +struct jit_avx512_core_x8s8s32x_deconv_fwd_kernel : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_x8s8s32x_deconv_fwd_ker_t); jit_avx512_core_x8s8s32x_deconv_fwd_kernel(const jit_conv_conf_t &ajcp, @@ -241,7 +241,7 @@ struct _jit_avx512_core_x8s8s32x_deconv_fwd_kernel { private: DNNL_DISALLOW_COPY_AND_ASSIGN(_jit_avx512_core_x8s8s32x_deconv_fwd_kernel); - std::unique_ptr kernel_; + std::unique_ptr kernel_; }; struct jit_avx512_core_x8s8s32x_deconvolution_fwd_t : public primitive_t { diff --git a/src/cpu/x64/jit_avx512_sparse_decompress_kernel.hpp b/src/cpu/x64/jit_avx512_sparse_decompress_kernel.hpp index 3f4145ea470..c18705b5650 100644 --- a/src/cpu/x64/jit_avx512_sparse_decompress_kernel.hpp +++ b/src/cpu/x64/jit_avx512_sparse_decompress_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2023-2024 Intel Corporation +* Copyright 2023-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,7 @@ namespace impl { namespace cpu { namespace x64 { -struct jit_avx512_sparse_decompress_kernel_t : public jit_generator { +struct jit_avx512_sparse_decompress_kernel_t : public jit_generator_t { struct call_params_t { const void *src_ptr; const void *bitmask_ptr; @@ -37,7 +37,7 @@ struct jit_avx512_sparse_decompress_kernel_t : public jit_generator { jit_avx512_sparse_decompress_kernel_t( const matmul::brgemm_matmul_conf_t &bgmmc) - : jit_generator("brgemm_decompress", avx512_core_amx) { + : jit_generator_t("brgemm_decompress", avx512_core_amx) { switch (bgmmc.wei_tag) { case format_tag::BA16a64b4a: case format_tag::aCB16b64c4b: b_blk_sz_ = 64; break; @@ -64,7 +64,7 @@ struct jit_avx512_sparse_decompress_kernel_t : public jit_generator { status_t create_kernel() override { CHECK(ctor_status_); - return jit_generator::create_kernel(); + return jit_generator_t::create_kernel(); } private: diff --git a/src/cpu/x64/jit_brgemm_conv.hpp b/src/cpu/x64/jit_brgemm_conv.hpp index bb63e9a5979..e2466316a84 100644 --- a/src/cpu/x64/jit_brgemm_conv.hpp +++ b/src/cpu/x64/jit_brgemm_conv.hpp @@ -256,7 +256,7 @@ struct brgemm_convolution_fwd_t : public primitive_t { copy_to_relo_pbuffer_; std::unique_ptr copy_to_relo_wbuffer_; - std::unique_ptr comp_vpad_pbuffer_; + std::unique_ptr comp_vpad_pbuffer_; std::unique_ptr jit_scale_precompute_; diff --git a/src/cpu/x64/jit_brgemm_conv_bwd_copy_kernel.cpp b/src/cpu/x64/jit_brgemm_conv_bwd_copy_kernel.cpp index f26fb2c0961..111c9c13ff5 100644 --- a/src/cpu/x64/jit_brgemm_conv_bwd_copy_kernel.cpp +++ b/src/cpu/x64/jit_brgemm_conv_bwd_copy_kernel.cpp @@ -34,7 +34,7 @@ template jit_avx512_core_brgemm_conv_bwd_copy_kernel_t:: jit_avx512_core_brgemm_conv_bwd_copy_kernel_t( const jit_brgemm_conv_conf_t &ajcp) - : jit_generator(jit_name()), jcp(ajcp) {} + : jit_generator_t(jit_name()), jcp(ajcp) {} // use different vmovdqu32/16/8 due to case when tail mask used template diff --git a/src/cpu/x64/jit_brgemm_conv_bwd_copy_kernel.hpp b/src/cpu/x64/jit_brgemm_conv_bwd_copy_kernel.hpp index 04a7162f7e1..fc7dae75c7a 100644 --- a/src/cpu/x64/jit_brgemm_conv_bwd_copy_kernel.hpp +++ b/src/cpu/x64/jit_brgemm_conv_bwd_copy_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2023 Intel Corporation +* Copyright 2023-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -33,7 +33,7 @@ struct jit_brgemm_conv_bwd_copy_kernel_call_s { }; template -struct jit_avx512_core_brgemm_conv_bwd_copy_kernel_t : public jit_generator { +struct jit_avx512_core_brgemm_conv_bwd_copy_kernel_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_brgemm_conv_bwd_copy_kernel_t) using reg64_t = const Xbyak::Reg64; diff --git a/src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp b/src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp index 0732c3d0ae1..649c8d17786 100644 --- a/src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp +++ b/src/cpu/x64/jit_brgemm_conv_bwd_strided.hpp @@ -242,7 +242,7 @@ struct brgemm_convolution_bwd_strided_t : public primitive_t { std::unique_ptr> copy_to_output_buffer_; - std::unique_ptr comp_vpad_pbuffer_; + std::unique_ptr comp_vpad_pbuffer_; std::unique_ptr jit_scale_precompute_; size_t acc_dsz, bia_dsz, src_dsz, wei_dsz, dst_dsz; diff --git a/src/cpu/x64/jit_brgemm_conv_bwd_trans_kernel.cpp b/src/cpu/x64/jit_brgemm_conv_bwd_trans_kernel.cpp index c222cecf6f7..faed1753f59 100644 --- a/src/cpu/x64/jit_brgemm_conv_bwd_trans_kernel.cpp +++ b/src/cpu/x64/jit_brgemm_conv_bwd_trans_kernel.cpp @@ -34,7 +34,7 @@ template jit_avx512_core_brgemm_conv_bwd_trans_kernel_t:: jit_avx512_core_brgemm_conv_bwd_trans_kernel_t( const jit_brgemm_conv_conf_t &ajcp, const char *name) - : jit_generator(name) + : jit_generator_t(name) , jcp(ajcp) , inp_dsz(jcp.src_dsz) , oc_block_sz(inp_dsz * jcp.oc_block) diff --git a/src/cpu/x64/jit_brgemm_conv_bwd_trans_kernel.hpp b/src/cpu/x64/jit_brgemm_conv_bwd_trans_kernel.hpp index 8d90e9397ac..4ae60de006a 100644 --- a/src/cpu/x64/jit_brgemm_conv_bwd_trans_kernel.hpp +++ b/src/cpu/x64/jit_brgemm_conv_bwd_trans_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2022-2023 Intel Corporation +* Copyright 2022-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,7 +37,7 @@ struct jit_brgemm_conv_bwd_trans_kernel_call_s { }; template -struct jit_avx512_core_brgemm_conv_bwd_trans_kernel_t : public jit_generator { +struct jit_avx512_core_brgemm_conv_bwd_trans_kernel_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS( jit_avx512_core_brgemm_conv_bwd_trans_kernel_t) diff --git a/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.cpp b/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.cpp index 74bdba1e587..e5d13322bed 100644 --- a/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.cpp +++ b/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.cpp @@ -35,7 +35,7 @@ template jit_uni_brgemm_conv_comp_pad_kernel_t:: jit_uni_brgemm_conv_comp_pad_kernel_t( const jit_brgemm_conv_conf_t &ajcp) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , jcp_(ajcp) , inp_dsz_(jcp_.wei_dsz) , out_dsz_(jcp_.acc_dsz) @@ -695,7 +695,7 @@ template jit_uni_brgemm_conv_relo_comp_pad_kernel_t:: jit_uni_brgemm_conv_relo_comp_pad_kernel_t( const jit_brgemm_conv_conf_t &ajcp) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , jcp_(ajcp) , inp_dsz_(jcp_.wei_dsz) , out_dsz_(jcp_.acc_dsz) diff --git a/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.hpp b/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.hpp index 8f6378717d5..160952f57bb 100644 --- a/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.hpp +++ b/src/cpu/x64/jit_brgemm_conv_comp_pad_kernel.hpp @@ -42,7 +42,7 @@ struct jit_brgemm_conv_comp_pad_call_s { // Variables with "ic" and "oc" are named from perspective of fwd // For bwd_d "ic" and "oc" are swapped template -struct jit_uni_brgemm_conv_comp_pad_kernel_t : public jit_generator { +struct jit_uni_brgemm_conv_comp_pad_kernel_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_brgemm_conv_comp_pad_kernel_t) @@ -137,7 +137,7 @@ struct jit_uni_brgemm_conv_comp_pad_kernel_t : public jit_generator { }; template -struct jit_uni_brgemm_conv_relo_comp_pad_kernel_t : public jit_generator { +struct jit_uni_brgemm_conv_relo_comp_pad_kernel_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_brgemm_conv_relo_comp_pad_kernel_t) using reg64_t = const Xbyak::Reg64; diff --git a/src/cpu/x64/jit_brgemm_conv_trans_kernel.cpp b/src/cpu/x64/jit_brgemm_conv_trans_kernel.cpp index 930d8dc15e6..5b104d05a91 100644 --- a/src/cpu/x64/jit_brgemm_conv_trans_kernel.cpp +++ b/src/cpu/x64/jit_brgemm_conv_trans_kernel.cpp @@ -33,7 +33,7 @@ namespace jit_avx512_core_brgemm_conv_trans_kernel { jit_avx512_core_brgemm_conv_trans_kernel_t:: jit_avx512_core_brgemm_conv_trans_kernel_t( const jit_brgemm_conv_conf_t &ajcp, const char *name) - : jit_generator(name) + : jit_generator_t(name) , jcp(ajcp) , inp_dsz(jcp.src_dsz) , ic_block_sz( diff --git a/src/cpu/x64/jit_brgemm_conv_trans_kernel.hpp b/src/cpu/x64/jit_brgemm_conv_trans_kernel.hpp index 3a124af7e17..8b72171643f 100644 --- a/src/cpu/x64/jit_brgemm_conv_trans_kernel.hpp +++ b/src/cpu/x64/jit_brgemm_conv_trans_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2021-2024 Intel Corporation +* Copyright 2021-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -36,7 +36,7 @@ struct jit_brgemm_conv_trans_kernel_call_s { size_t b_pad; }; -struct jit_avx512_core_brgemm_conv_trans_kernel_t : public jit_generator { +struct jit_avx512_core_brgemm_conv_trans_kernel_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_avx512_core_brgemm_conv_trans_kernel_t) using reg64_t = const Xbyak::Reg64; diff --git a/src/cpu/x64/jit_brgemm_post_ops.cpp b/src/cpu/x64/jit_brgemm_post_ops.cpp index d45a23a7603..ad9c28a833a 100644 --- a/src/cpu/x64/jit_brgemm_post_ops.cpp +++ b/src/cpu/x64/jit_brgemm_post_ops.cpp @@ -28,7 +28,7 @@ template dnnl::impl::cpu::x64::jit_brgemm_kernel_diff_bias_t:: jit_brgemm_kernel_diff_bias_t(const jit_brgemm_primitive_conf_t &ajbgp, const brgemm_desc_t &abrg) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , brg_(abrg) , reduce_kind_(matmul_reduce_kind::undef) , ddst_dt_((ajbgp.isa == avx512_core_fp16 && ajbgp.use_buffer_b) @@ -46,7 +46,7 @@ template dnnl::impl::cpu::x64::jit_brgemm_kernel_diff_bias_t:: jit_brgemm_kernel_diff_bias_t(const matmul::brgemm_matmul_conf_t &bgmmc, const brgemm_desc_t &abrg) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , brg_(abrg) , reduce_kind_(bgmmc.reduce_kind) // MatMul `src`. @@ -503,7 +503,7 @@ template dnnl::impl::cpu::x64::jit_brgemm_kernel_post_ops_t< Vmm>::jit_brgemm_kernel_post_ops_t(const brgemm_desc_t &abrg, const primitive_attr_t &aattr) - : jit_generator(jit_name(), abrg.isa_impl) + : jit_generator_t(jit_name(), abrg.isa_impl) , brg_(abrg) , attr_(aattr) , max_vregs_(isa_num_vregs(brg_.isa_impl)) diff --git a/src/cpu/x64/jit_brgemm_post_ops.hpp b/src/cpu/x64/jit_brgemm_post_ops.hpp index 78c2c08f1cf..ee591fed8b6 100644 --- a/src/cpu/x64/jit_brgemm_post_ops.hpp +++ b/src/cpu/x64/jit_brgemm_post_ops.hpp @@ -50,7 +50,7 @@ struct brgemm_kernel_diff_bias_t { }; template -struct jit_brgemm_kernel_diff_bias_t : public jit_generator { +struct jit_brgemm_kernel_diff_bias_t : public jit_generator_t { jit_brgemm_kernel_diff_bias_t(const jit_brgemm_primitive_conf_t &ajbgp, const brgemm_desc_t &abrg); @@ -159,7 +159,7 @@ struct jit_brgemm_kernel_post_ops_base_t { // Shouldn't be called directly on implementation side. template struct jit_brgemm_kernel_post_ops_t : public jit_brgemm_kernel_post_ops_base_t, - public jit_generator { + public jit_generator_t { // TODO: the proper design should replace `brgemm_desc_t` argument and // introduce a dedicated struct with members properly initialized. This will @@ -169,12 +169,12 @@ struct jit_brgemm_kernel_post_ops_t : public jit_brgemm_kernel_post_ops_base_t, const brgemm_desc_t &abrg, const primitive_attr_t &aattr); // These two methods are required for a base class to work since it's not - // derived from the jit_generator. + // derived from the jit_generator_t. status_t generate_kernel() override { - return jit_generator::create_kernel(); + return jit_generator_t::create_kernel(); } void operator()(brgemm_kernel_post_ops_args_t *args) const override { - return jit_generator::operator()(args); + return jit_generator_t::operator()(args); } ~jit_brgemm_kernel_post_ops_t() override = default; diff --git a/src/cpu/x64/jit_brgemm_transpose_utils.cpp b/src/cpu/x64/jit_brgemm_transpose_utils.cpp index a5606a6bb0c..e8ba525bd73 100644 --- a/src/cpu/x64/jit_brgemm_transpose_utils.cpp +++ b/src/cpu/x64/jit_brgemm_transpose_utils.cpp @@ -35,16 +35,18 @@ using namespace Xbyak; #define GET_OFF(x) offsetof(ctx_t, x) struct jit_brgemm_trans_m_k_f32_t : public jit_brgemm_trans_src_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_brgemm_trans_m_k_f32_t) jit_brgemm_trans_m_k_f32_t(const jit_brgemm_primitive_conf_t *conf) : jit_brgemm_trans_src_t(conf) - , jit_generator(jit_name()) + , jit_generator_t(jit_name()) , transpose_size(isa_max_vlen(conf_->isa) / typesize) {} - void operator()(ctx_t *ctx) override { jit_generator::operator()(ctx); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + void operator()(ctx_t *ctx) override { jit_generator_t::operator()(ctx); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } private: using reg64_t = const Xbyak::Reg64; @@ -84,7 +86,7 @@ struct jit_brgemm_trans_m_k_f32_t : public jit_brgemm_trans_src_t, Xmm xmm_zero = xmm13; void kmovw(Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovw(k, regw_tmp); + jit_generator_t::kmovw(k, regw_tmp); }; void transpose_16x16(int nrows, int ncolumns); void transpose_16x16_avx2(int nrows, int ncolumns); @@ -470,13 +472,15 @@ void jit_brgemm_trans_m_k_f32_t::generate() { } struct jit_brgemm_trans_m_k_bf16_t : public jit_brgemm_trans_src_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_brgemm_trans_m_k_bf16_t) jit_brgemm_trans_m_k_bf16_t(const jit_brgemm_primitive_conf_t *conf) - : jit_brgemm_trans_src_t(conf), jit_generator(jit_name()) {} + : jit_brgemm_trans_src_t(conf), jit_generator_t(jit_name()) {} - void operator()(ctx_t *ctx) override { jit_generator::operator()(ctx); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + void operator()(ctx_t *ctx) override { jit_generator_t::operator()(ctx); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } private: using reg64_t = const Xbyak::Reg64; @@ -543,12 +547,12 @@ void jit_brgemm_trans_m_k_bf16_t::transpose( auto kmovw = [this](Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovw(k, regw_tmp); + jit_generator_t::kmovw(k, regw_tmp); }; auto kmovd = [this](Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovd(k, regw_tmp); + jit_generator_t::kmovd(k, regw_tmp); }; auto store = [&](Zmm r, int i) { @@ -722,7 +726,7 @@ void jit_brgemm_trans_m_k_bf16_t::generate() { auto kmovw = [this](Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovw(k, regw_tmp); + jit_generator_t::kmovw(k, regw_tmp); }; kmovw(kFFFF, 0xffff); @@ -735,12 +739,12 @@ void jit_brgemm_trans_m_k_bf16_t::generate() { auto vmovdqa64 = [this](Zmm z, const int64_t *addr) { mov(imm_addr64, reinterpret_cast(addr)); - jit_generator::vmovdqa64(z, ptr[imm_addr64]); + jit_generator_t::vmovdqa64(z, ptr[imm_addr64]); }; auto vmovdqa32 = [this](Zmm z, const int32_t *addr) { mov(imm_addr64, reinterpret_cast(addr)); - jit_generator::vmovdqa32(z, ptr[imm_addr64]); + jit_generator_t::vmovdqa32(z, ptr[imm_addr64]); }; vmovdqa64(vidx1, idx1); @@ -833,14 +837,16 @@ void jit_brgemm_trans_m_k_bf16_t::generate() { } struct jit_brgemm_trans_m_k_f16_t : public jit_brgemm_trans_src_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_brgemm_trans_m_k_f16_t) jit_brgemm_trans_m_k_f16_t(const jit_brgemm_primitive_conf_t *conf) - : jit_brgemm_trans_src_t(conf), jit_generator(jit_name()) {} + : jit_brgemm_trans_src_t(conf), jit_generator_t(jit_name()) {} - void operator()(ctx_t *ctx) override { jit_generator::operator()(ctx); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + void operator()(ctx_t *ctx) override { jit_generator_t::operator()(ctx); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } private: using reg64_t = const Xbyak::Reg64; @@ -895,7 +901,7 @@ void jit_brgemm_trans_m_k_f16_t::transpose_16x16(int nrows, int ncolumns) { auto kmovw = [this](Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovw(k, regw_tmp); + jit_generator_t::kmovw(k, regw_tmp); }; auto load = [&](int i) { @@ -1042,7 +1048,7 @@ void jit_brgemm_trans_m_k_f16_t::generate() { auto kmovw = [this](Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovw(k, regw_tmp); + jit_generator_t::kmovw(k, regw_tmp); }; kmovw(k3333, 0x3333); // 0011001100110011 @@ -1322,16 +1328,18 @@ void jit_brgemm_copy_to_coarse_t::generate() { } struct jit_trans_to_vnni_t : public jit_brgemm_trans_to_vnni_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_trans_to_vnni_t) jit_trans_to_vnni_t(const jit_brgemm_primitive_conf_t *conf, jit_brgemm_trans_to_vnni_t::matrix_to_transform_t matrix_to_transform) : jit_brgemm_trans_to_vnni_t(conf, matrix_to_transform) - , jit_generator(jit_name()) {} + , jit_generator_t(jit_name()) {} - void operator()(ctx_t *ctx) override { jit_generator::operator()(ctx); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + void operator()(ctx_t *ctx) override { jit_generator_t::operator()(ctx); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } private: using reg64_t = const Xbyak::Reg64; @@ -1511,11 +1519,11 @@ void jit_trans_to_vnni_t::generate() { auto kmovw = [this](Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovw(k, regw_tmp); + jit_generator_t::kmovw(k, regw_tmp); }; auto kmovd = [this](Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovd(k, regw_tmp); + jit_generator_t::kmovd(k, regw_tmp); }; kmovw(kFFFF, 0xffff); // 1111111111111111 @@ -1523,7 +1531,7 @@ void jit_trans_to_vnni_t::generate() { auto vmovdqa64 = [this](Zmm z, const int64_t *addr) { mov(imm_addr64, reinterpret_cast(addr)); - jit_generator::vmovdqa64(z, ptr[imm_addr64]); + jit_generator_t::vmovdqa64(z, ptr[imm_addr64]); }; vmovdqa64(vidx1, (const int64_t *)idx1); @@ -1627,19 +1635,21 @@ void jit_trans_to_vnni_t::generate() { } struct jit_copy_f32_t : public jit_brgemm_trans_to_vnni_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_copy_f32_t) jit_copy_f32_t(const jit_brgemm_primitive_conf_t *conf, jit_brgemm_trans_to_vnni_t::matrix_to_transform_t matrix_to_transform) : jit_brgemm_trans_to_vnni_t(conf, matrix_to_transform) - , jit_generator(jit_name()) + , jit_generator_t(jit_name()) , column_step(isa_max_vlen(conf->isa) / typesize_data) , num_regs(isa_num_vregs(conf->isa)) , col_shift(static_cast(column_step) * typesize_data) {} - void operator()(ctx_t *ctx) override { jit_generator::operator()(ctx); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + void operator()(ctx_t *ctx) override { jit_generator_t::operator()(ctx); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } private: using reg64_t = const Xbyak::Reg64; @@ -1726,7 +1736,7 @@ void jit_copy_f32_t::init_masks(int tail_length) { if (tail_length == 0) return; auto kmovd = [this](Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovd(k, regw_tmp); + jit_generator_t::kmovd(k, regw_tmp); }; if (isa_has_masks(conf_->isa)) @@ -1812,13 +1822,13 @@ void jit_copy_f32_t::generate() { } struct jit_copy_f16_t : public jit_brgemm_trans_to_vnni_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_copy_f16_t) jit_copy_f16_t(const jit_brgemm_primitive_conf_t *conf, jit_brgemm_trans_to_vnni_t::matrix_to_transform_t matrix_to_transform) : jit_brgemm_trans_to_vnni_t(conf, matrix_to_transform) - , jit_generator(jit_name()) { + , jit_generator_t(jit_name()) { // matrix_to_transform_ == matrix_B, copy(f16) -> f32 // matrix_to_transform_ == matrix_C, copy(f32) -> f16 + zero_pad @@ -1850,8 +1860,10 @@ struct jit_copy_f16_t : public jit_brgemm_trans_to_vnni_t, col_shift_out = column_step * typesize_out; } - void operator()(ctx_t *ctx) override { jit_generator::operator()(ctx); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + void operator()(ctx_t *ctx) override { jit_generator_t::operator()(ctx); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } private: using reg64_t = const Xbyak::Reg64; @@ -1899,7 +1911,7 @@ void jit_copy_f16_t::copy_block(bool is_row_tail, bool is_col_tail) { auto kmovd = [this](Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovd(k, regw_tmp); + jit_generator_t::kmovd(k, regw_tmp); }; const int nc_tail = ncolumns % column_step; @@ -2111,16 +2123,18 @@ void jit_brgemm_relo_copy_to_wbuffer_t::generate() { } struct jit_brgemm_trans_wei_f32_t : public jit_brgemm_trans_wei_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_brgemm_trans_wei_f32_t) jit_brgemm_trans_wei_f32_t(const jit_brgemm_primitive_conf_t *conf) : jit_brgemm_trans_wei_t(conf) - , jit_generator(jit_name()) + , jit_generator_t(jit_name()) , transpose_size(conf->simd_w) {} - void operator()(ctx_t *ctx) override { jit_generator::operator()(ctx); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + void operator()(ctx_t *ctx) override { jit_generator_t::operator()(ctx); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } private: using reg64_t = const Xbyak::Reg64; @@ -2152,7 +2166,7 @@ struct jit_brgemm_trans_wei_f32_t : public jit_brgemm_trans_wei_t, void kmovw(Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovw(k, regw_tmp); + jit_generator_t::kmovw(k, regw_tmp); } void transpose_16x16(int nrows, int ncolumns); void transpose_8x8(); @@ -2296,8 +2310,8 @@ void jit_brgemm_trans_wei_f32_t::transpose_8x8() { mov(reg_tr_src_tmp, reg_tr_src); Xbyak::Ymm ymm_dummy = Ymm(0); Xbyak::Xmm xmm_dummy = Xmm(0); - jit_generator::transpose(reg_src, reg_tr_src_tmp, src_stride, tr_src_stride, - 8, 8, data_type::f32, + jit_generator_t::transpose(reg_src, reg_tr_src_tmp, src_stride, + tr_src_stride, 8, 8, data_type::f32, /*unused*/ ymm_dummy, ymm_dummy, xmm_dummy); } @@ -2408,14 +2422,16 @@ void jit_brgemm_trans_wei_f32_t::generate() { } struct jit_brgemm_trans_wei_bf16_t : public jit_brgemm_trans_wei_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_brgemm_trans_wei_bf16_t) jit_brgemm_trans_wei_bf16_t(const jit_brgemm_primitive_conf_t *conf) - : jit_brgemm_trans_wei_t(conf), jit_generator(jit_name()) {} + : jit_brgemm_trans_wei_t(conf), jit_generator_t(jit_name()) {} - void operator()(ctx_t *ctx) override { jit_generator::operator()(ctx); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + void operator()(ctx_t *ctx) override { jit_generator_t::operator()(ctx); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } private: using reg64_t = const Xbyak::Reg64; @@ -2464,7 +2480,7 @@ void jit_brgemm_trans_wei_bf16_t::transpose_16x16_vnni( auto kmovw = [this](Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovw(k, regw_tmp); + jit_generator_t::kmovw(k, regw_tmp); }; auto load = [&](int i) { @@ -2558,7 +2574,7 @@ void jit_brgemm_trans_wei_bf16_t::generate() { auto vmovdqa64 = [this](Zmm z, const int64_t *addr) { mov(imm_addr64, reinterpret_cast(addr)); - jit_generator::vmovdqa64(z, ptr[imm_addr64]); + jit_generator_t::vmovdqa64(z, ptr[imm_addr64]); }; vmovdqa64(v_abcdefgh_to_abefcdgh, (const int64_t *)abcdefgh_to_abefcdgh); @@ -2621,14 +2637,16 @@ void jit_brgemm_trans_wei_bf16_t::generate() { } struct jit_brgemm_trans_wei_f16_t : public jit_brgemm_trans_wei_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_brgemm_trans_wei_f16_t) jit_brgemm_trans_wei_f16_t(const jit_brgemm_primitive_conf_t *conf) - : jit_brgemm_trans_wei_t(conf), jit_generator(jit_name()) {} + : jit_brgemm_trans_wei_t(conf), jit_generator_t(jit_name()) {} - void operator()(ctx_t *ctx) override { jit_generator::operator()(ctx); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + void operator()(ctx_t *ctx) override { jit_generator_t::operator()(ctx); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } private: using reg64_t = const Xbyak::Reg64; @@ -2685,7 +2703,7 @@ void jit_brgemm_trans_wei_f16_t::transpose_16x16(int nrows, int ncolumns) { auto kmovw = [this](Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovw(k, regw_tmp); + jit_generator_t::kmovw(k, regw_tmp); }; auto load = [&](int i) { @@ -2829,7 +2847,7 @@ void jit_brgemm_trans_wei_f16_t::generate() { auto kmovw = [this](Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovw(k, regw_tmp); + jit_generator_t::kmovw(k, regw_tmp); }; kmovw(k3333, 0x3333); // 0011001100110011 @@ -2897,16 +2915,18 @@ void jit_brgemm_trans_wei_f16_t::generate() { } struct jit_amx_ip_trans_diff_wei_to_vnni_t : public jit_amx_ip_trans_diff_wei, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_amx_ip_trans_diff_wei_to_vnni) jit_amx_ip_trans_diff_wei_to_vnni_t(const jit_brgemm_primitive_conf_t *jbgp, const int ext_ic_block, const int ext_oc_block) : jit_amx_ip_trans_diff_wei(jbgp, ext_ic_block, ext_oc_block) - , jit_generator(jit_name()) {} + , jit_generator_t(jit_name()) {} - void operator()(ctx_t *ctx) override { jit_generator::operator()(ctx); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + void operator()(ctx_t *ctx) override { jit_generator_t::operator()(ctx); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } private: void generate() override; @@ -2939,7 +2959,7 @@ void jit_amx_ip_trans_diff_wei_to_vnni_t::generate() { : 0xffff; auto kmovw = [this, regw_tmp](Xbyak::Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovw(k, regw_tmp); + jit_generator_t::kmovw(k, regw_tmp); }; auto reorder_oc_block = [&](int icb, int ic_block, bool is_oc_tail) { diff --git a/src/cpu/x64/jit_brgemm_transpose_utils.hpp b/src/cpu/x64/jit_brgemm_transpose_utils.hpp index 64d7e63ece8..c889eef75c6 100644 --- a/src/cpu/x64/jit_brgemm_transpose_utils.hpp +++ b/src/cpu/x64/jit_brgemm_transpose_utils.hpp @@ -44,7 +44,7 @@ struct jit_brgemm_trans_src_t { const jit_brgemm_primitive_conf_t *conf_; }; -struct jit_brgemm_copy_to_coarse_t : public jit_generator { +struct jit_brgemm_copy_to_coarse_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_brgemm_copy_to_coarse_t) struct ctx_t { @@ -55,11 +55,13 @@ struct jit_brgemm_copy_to_coarse_t : public jit_generator { dim_t last_row_blk; }; - void operator()(ctx_t *ctx) { jit_generator::operator()(ctx); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + void operator()(ctx_t *ctx) { jit_generator_t::operator()(ctx); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } jit_brgemm_copy_to_coarse_t(const jit_brgemm_primitive_conf_t *conf) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , conf_(conf) , typesize_(sizeof(float) / data_type_vnni_granularity(conf_->wei_dt)) , is_fwd_dir_(utils::one_of(conf_->prop_kind, @@ -180,7 +182,7 @@ struct jit_brgemm_trans_wei_t { const jit_brgemm_primitive_conf_t *conf_; }; -struct jit_brgemm_relo_copy_to_wbuffer_t : public jit_generator { +struct jit_brgemm_relo_copy_to_wbuffer_t : public jit_generator_t { struct cfg_t { data_type_t wei_dt {data_type_t::dnnl_data_type_undef}; int out_oc_block {0}; @@ -202,7 +204,7 @@ struct jit_brgemm_relo_copy_to_wbuffer_t : public jit_generator { using reg64_t = Xbyak::Reg64; jit_brgemm_relo_copy_to_wbuffer_t(const cfg_t &ajcp) - : jit_generator(jit_name(), avx512_core_amx), wjcp(ajcp) {} + : jit_generator_t(jit_name(), avx512_core_amx), wjcp(ajcp) {} private: cfg_t wjcp; diff --git a/src/cpu/x64/jit_gemm_inner_product_utils.cpp b/src/cpu/x64/jit_gemm_inner_product_utils.cpp index 8b8f30eb22d..4aca03ff3f4 100644 --- a/src/cpu/x64/jit_gemm_inner_product_utils.cpp +++ b/src/cpu/x64/jit_gemm_inner_product_utils.cpp @@ -38,7 +38,7 @@ using namespace Xbyak; using namespace data_type; template -struct jit_pp_kernel_t : public pp_kernel_t, public jit_generator { +struct jit_pp_kernel_t : public pp_kernel_t, public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(inner_product_utils::jit_pp_kernel_t); jit_pp_kernel_t(size_t OC, size_t MB, dim_t dst_mb_stride, @@ -54,7 +54,9 @@ struct jit_pp_kernel_t : public pp_kernel_t, public jit_generator { size_t first_mb_matrix_addr_off, const exec_ctx_t &ctx, const memory_desc_t &dst_md) const override; - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } private: using Vmm = typename utils::conditional3::jit_pp_kernel_t(size_t OC, size_t MB, dim_t dst_mb_stride, const memory_desc_t *dst_md, bool skip_sum) : pp_kernel_t( OC, MB, dst_mb_stride, attr, bias_dt, acc_dt, dst_md, skip_sum) - , jit_generator(jit_name(), isa) { + , jit_generator_t(jit_name(), isa) { assert(IMPLICATION(this->dst_data_type_ == bf16, mayiuse(avx512_core))); if (this->do_scale_) vreg_scale = Vmm(idx_compute_vreg_start_++); @@ -1237,7 +1239,7 @@ void jit_pp_kernel_t::operator()(void *dst, const void *acc, args.post_ops_binary_rhs_arg_vec = post_ops_binary_rhs_arg_vec; args.dst_orig = dst_orig; - jit_generator::operator()(&args); + jit_generator_t::operator()(&args); } pp_kernel_t *jit_pp_kernel_create(size_t OC, size_t MB, dim_t dst_mb_stride, diff --git a/src/cpu/x64/jit_gemm_x8s8s32x_conv_zp_src_pad_comp.cpp b/src/cpu/x64/jit_gemm_x8s8s32x_conv_zp_src_pad_comp.cpp index 022ed0d4c41..001a6e6f7a3 100644 --- a/src/cpu/x64/jit_gemm_x8s8s32x_conv_zp_src_pad_comp.cpp +++ b/src/cpu/x64/jit_gemm_x8s8s32x_conv_zp_src_pad_comp.cpp @@ -26,7 +26,7 @@ namespace x64 { namespace gemm_x8s8s32x_convolution_utils { jit_gemm_x8s8s32x_zp_pad_comp_helper::jit_gemm_x8s8s32x_zp_pad_comp_helper( - jit_generator *host, const conv_gemm_conf_t &jcp, + jit_generator_t *host, const conv_gemm_conf_t &jcp, const Xbyak::Reg64 ®_zp_pad_comp, const Xbyak::Reg64 ®_zp_pad_comp_temp, const Xbyak::Reg8 &should_apply_zp_src_pad, const dim_t ndims) @@ -88,7 +88,7 @@ void jit_gemm_x8s8s32x_zp_pad_comp_helper::zp_src_comp_pad_operation( if (op) { Xbyak::Label end; host_->cmp(should_apply_zp_src_pad_, 0); - host_->je(end, jit_generator::T_NEAR); + host_->je(end, jit_generator_t::T_NEAR); op(reg_zp_pad_comp_); host_->L(end); } @@ -195,7 +195,7 @@ void jit_gemm_x8s8s32x_zp_pad_comp_helper::load_zp_src_comp_pad_addr_if_needed( const Xbyak::Address &g_oc_offset) { Xbyak::Label calc_zp_src_comp_pad_addr, end; host_->cmp(should_apply_zp_src_pad_, 0); - host_->je(end, jit_generator::T_NEAR); + host_->je(end, jit_generator_t::T_NEAR); host_->L(calc_zp_src_comp_pad_addr); { @@ -251,20 +251,20 @@ void jit_gemm_x8s8s32x_zp_pad_comp_helper::get_zp_pad_com_dim( host_->L(lower_bound); { host_->cmp(dim_under_lower_bound, 0); - host_->je(upper_bound, jit_generator::T_NEAR); + host_->je(upper_bound, jit_generator_t::T_NEAR); host_->mov(reg_zp_pad_comp_tmp_, out_point_dim); host_->mov(result, reg_zp_pad_comp_tmp_); - host_->jmp(end, jit_generator::T_NEAR); + host_->jmp(end, jit_generator_t::T_NEAR); } host_->L(upper_bound); { host_->cmp(dim_over_eq_upper_bound, 0); - host_->je(mid_point, jit_generator::T_NEAR); + host_->je(mid_point, jit_generator_t::T_NEAR); host_->mov(reg_zp_pad_comp_tmp_, begin_pad + mid_pad + end_pad - out_dim_size); host_->add(reg_zp_pad_comp_tmp_, out_point_dim); host_->mov(result, reg_zp_pad_comp_tmp_); - host_->jmp(end, jit_generator::T_NEAR); + host_->jmp(end, jit_generator_t::T_NEAR); } host_->L(mid_point); @@ -299,7 +299,7 @@ void jit_gemm_x8s8s32x_zp_pad_comp_helper::next_point() { } host_->cmp(reg_w, w_size_addr_); - host_->jl(store_w, jit_generator::T_NEAR); + host_->jl(store_w, jit_generator_t::T_NEAR); if (with_zp_pad_com_h_) { diff --git a/src/cpu/x64/jit_gemm_x8s8s32x_conv_zp_src_pad_comp.hpp b/src/cpu/x64/jit_gemm_x8s8s32x_conv_zp_src_pad_comp.hpp index 91dcb7cbff4..4b4489e831b 100644 --- a/src/cpu/x64/jit_gemm_x8s8s32x_conv_zp_src_pad_comp.hpp +++ b/src/cpu/x64/jit_gemm_x8s8s32x_conv_zp_src_pad_comp.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2021 Intel Corporation +* Copyright 2021-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -30,12 +30,12 @@ struct conv_gemm_conf_t; namespace x64 { -class jit_generator; +class jit_generator_t; namespace gemm_x8s8s32x_convolution_utils { struct jit_gemm_x8s8s32x_zp_pad_comp_helper { - jit_gemm_x8s8s32x_zp_pad_comp_helper(jit_generator *host, + jit_gemm_x8s8s32x_zp_pad_comp_helper(jit_generator_t *host, const conv_gemm_conf_t &jcp, const Xbyak::Reg64 ®_zp_pad_comp, const Xbyak::Reg64 ®_zp_pad_comp_temp, const Xbyak::Reg8 &should_apply_zp_src_pad, const dim_t ndims); @@ -84,7 +84,7 @@ struct jit_gemm_x8s8s32x_zp_pad_comp_helper { void should_apply_zp_src_pad(); void next_point(); - jit_generator *const host_; + jit_generator_t *const host_; const conv_gemm_conf_t &jcp_; const Xbyak::Address w_addr_; const Xbyak::Address h_addr_; diff --git a/src/cpu/x64/jit_gemm_x8s8s32x_convolution_utils.cpp b/src/cpu/x64/jit_gemm_x8s8s32x_convolution_utils.cpp index 36be819d8ff..bacdd63400a 100644 --- a/src/cpu/x64/jit_gemm_x8s8s32x_convolution_utils.cpp +++ b/src/cpu/x64/jit_gemm_x8s8s32x_convolution_utils.cpp @@ -31,13 +31,15 @@ namespace x64 { namespace gemm_x8s8s32x_convolution_utils { using namespace dnnl::impl::cpu::gemm_x8s8s32x_convolution_utils; -struct jit_pp_ker_t : pp_ker_t, public jit_generator { +struct jit_pp_ker_t : pp_ker_t, public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS( gemm_x8s8s32x_convolution_utils::jit_pp_ker_t); jit_pp_ker_t(const convolution_pd_t *pd, const conv_gemm_conf_t &jcp); - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } void operator()(void *void_dst, const acc_data_t *acc, const char *bias, const float *scales, float dst_scale, float sum_scale, float signed_scale, int g, size_t start, size_t end, @@ -140,7 +142,7 @@ struct jit_pp_ker_t : pp_ker_t, public jit_generator { jit_pp_ker_t::jit_pp_ker_t( const convolution_pd_t *pd, const conv_gemm_conf_t &jcp) : pp_ker_t(pd, jcp) - , jit_generator(jit_name()) + , jit_generator_t(jit_name()) , number_of_reserved_zmm_regs_(0) , bias_data_type_size_(jcp.bias_data_type != data_type::undef ? types::data_type_size(jcp.bias_data_type) @@ -257,7 +259,7 @@ void jit_pp_ker_t::operator()(void *void_dst, const acc_data_t *acc, = zp_src_pad_com_d.should_apply_pad_comp_d; } - jit_generator::operator()(&args); + jit_generator_t::operator()(&args); } Xbyak::Zmm jit_pp_ker_t::reserve_zmm() { diff --git a/src/cpu/x64/jit_generator.cpp b/src/cpu/x64/jit_generator.cpp index 40fc96ea966..5c350a3df49 100644 --- a/src/cpu/x64/jit_generator.cpp +++ b/src/cpu/x64/jit_generator.cpp @@ -21,7 +21,7 @@ namespace impl { namespace cpu { namespace x64 { -void jit_generator::transpose(const Xbyak::Reg64 ®_src, +void jit_generator_t::transpose(const Xbyak::Reg64 ®_src, const Xbyak::Reg64 ®_dst, dim_t src_stride, dim_t dst_stride, int nrows, int ncolumns, data_type_t dt, Xbyak::Ymm &ymm_tmp, Xbyak::Ymm &ymm_mask, Xbyak::Xmm &xmm_upper_mask) { diff --git a/src/cpu/x64/jit_generator.hpp b/src/cpu/x64/jit_generator.hpp index 2440512c5f7..c68de87b380 100644 --- a/src/cpu/x64/jit_generator.hpp +++ b/src/cpu/x64/jit_generator.hpp @@ -60,7 +60,7 @@ namespace impl { namespace cpu { namespace x64 { -// TODO: move this to jit_generator class? +// TODO: move this to jit_generator_t class? namespace { typedef enum { @@ -144,9 +144,9 @@ constexpr Xbyak::Operand::Code abi_not_param_reg = #endif -class jit_generator : public Xbyak::MmapAllocator, - public Xbyak::CodeGenerator, - public c_compatible { +class jit_generator_t : public Xbyak::MmapAllocator, + public Xbyak::CodeGenerator, + public c_compatible { public: using c_compatible::operator new; using c_compatible::operator new[]; @@ -2700,17 +2700,17 @@ class jit_generator : public Xbyak::MmapAllocator, data_type_t dt, /*rest of vmms used only if there are tails*/ Xbyak::Ymm &ymm_tmp, Xbyak::Ymm &ymm_mask, Xbyak::Xmm &xmm_upper_mask); - DNNL_DISALLOW_COPY_AND_ASSIGN(jit_generator); + DNNL_DISALLOW_COPY_AND_ASSIGN(jit_generator_t); /* All uni_ instructions -- apart from uni_vzeroupper() -- will comply with * the max_cpu_isa argument */ - jit_generator(const char *name, cpu_isa_t max_cpu_isa = get_max_cpu_isa()) + jit_generator_t(const char *name, cpu_isa_t max_cpu_isa = get_max_cpu_isa()) : Xbyak::MmapAllocator(name) , Xbyak::CodeGenerator(MAX_CODE_SIZE, Xbyak::AutoGrow, /*allocator=*/this) , max_cpu_isa_(max_cpu_isa) {} - ~jit_generator() override = default; + ~jit_generator_t() override = default; virtual const char *name() const = 0; virtual const char *source_file() const = 0; diff --git a/src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.cpp b/src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.cpp index a2aa3787efb..f18b185efdc 100644 --- a/src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.cpp +++ b/src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.cpp @@ -42,7 +42,7 @@ using namespace Xbyak; jit_sse41_1x1_conv_kernel_f32::jit_sse41_1x1_conv_kernel_f32( const jit_1x1_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name(), sse41), jcp(ajcp), attr_(attr) { + : jit_generator_t(jit_name(), sse41), jcp(ajcp), attr_(attr) { if (jcp.with_eltwise || jcp.with_binary) { static constexpr bool preserve_gpr = true; static constexpr bool preserve_vmm = false; diff --git a/src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.hpp b/src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.hpp index 284406e3d80..72946fba41f 100644 --- a/src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.hpp +++ b/src/cpu/x64/jit_sse41_1x1_conv_kernel_f32.hpp @@ -29,7 +29,7 @@ namespace impl { namespace cpu { namespace x64 { -struct jit_sse41_1x1_conv_kernel_f32 : public jit_generator { +struct jit_sse41_1x1_conv_kernel_f32 : public jit_generator_t { jit_sse41_1x1_conv_kernel_f32(const jit_1x1_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md); diff --git a/src/cpu/x64/jit_sse41_conv_kernel_f32.cpp b/src/cpu/x64/jit_sse41_conv_kernel_f32.cpp index 98e56a5bfb0..f184838e92f 100644 --- a/src/cpu/x64/jit_sse41_conv_kernel_f32.cpp +++ b/src/cpu/x64/jit_sse41_conv_kernel_f32.cpp @@ -41,7 +41,7 @@ using namespace Xbyak; jit_sse41_conv_fwd_kernel_f32::jit_sse41_conv_fwd_kernel_f32( const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name(), sse41), jcp(ajcp), attr_(attr) { + : jit_generator_t(jit_name(), sse41), jcp(ajcp), attr_(attr) { if (jcp.with_eltwise || jcp.with_binary) { static constexpr bool preserve_gpr = true; static constexpr bool preserve_vmm = false; diff --git a/src/cpu/x64/jit_sse41_conv_kernel_f32.hpp b/src/cpu/x64/jit_sse41_conv_kernel_f32.hpp index 2534f41d899..ab01f16cf21 100644 --- a/src/cpu/x64/jit_sse41_conv_kernel_f32.hpp +++ b/src/cpu/x64/jit_sse41_conv_kernel_f32.hpp @@ -29,7 +29,7 @@ namespace impl { namespace cpu { namespace x64 { -struct jit_sse41_conv_fwd_kernel_f32 : public jit_generator { +struct jit_sse41_conv_fwd_kernel_f32 : public jit_generator_t { jit_sse41_conv_fwd_kernel_f32(const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md); diff --git a/src/cpu/x64/jit_transpose_utils.cpp b/src/cpu/x64/jit_transpose_utils.cpp index 1cb84a2be20..7d856b60f02 100644 --- a/src/cpu/x64/jit_transpose_utils.cpp +++ b/src/cpu/x64/jit_transpose_utils.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2017-2024 Intel Corporation +* Copyright 2017-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -34,20 +34,22 @@ using namespace Xbyak; #define GET_OFF(x) offsetof(ctx_t, x) -struct jit_trans_iw_ic_t : public jit_trans_src_t, public jit_generator { +struct jit_trans_iw_ic_t : public jit_trans_src_t, public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_trans_iw_ic_t) jit_trans_iw_ic_t(const jit_conv_conf_t *conf) : jit_trans_src_t(conf) - , jit_generator(jit_name()) + , jit_generator_t(jit_name()) , typesize(conf->src_dt == data_type::undef ? 2 : types::data_type_size(conf->src_dt)) , is_layout_nxc(utils::one_of(conf_->src_tag, format_tag::ndhwc, format_tag::nhwc, format_tag::nwc)) {} - void operator()(ctx_t *ctx) override { jit_generator::operator()(ctx); } + void operator()(ctx_t *ctx) override { jit_generator_t::operator()(ctx); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } private: int typesize = 0; @@ -82,11 +84,11 @@ struct jit_trans_iw_ic_t : public jit_trans_src_t, public jit_generator { void kmovw(Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovw(k, regw_tmp); + jit_generator_t::kmovw(k, regw_tmp); } void kmovd(Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovd(k, regw_tmp); + jit_generator_t::kmovd(k, regw_tmp); } Zmm src_zmm(int i) { return Zmm(i); } Ymm src_ymm(int i) { @@ -99,12 +101,12 @@ struct jit_trans_iw_ic_t : public jit_trans_src_t, public jit_generator { } void vmovdqa64(Zmm z, const int64_t *addr) { mov(imm_addr64, reinterpret_cast(addr)); - jit_generator::vmovdqa64(z, ptr[imm_addr64]); + jit_generator_t::vmovdqa64(z, ptr[imm_addr64]); } void vmovdqa32(Zmm z, const int32_t *addr) { mov(imm_addr64, reinterpret_cast(addr)); - jit_generator::vmovdqa32(z, ptr[imm_addr64]); + jit_generator_t::vmovdqa32(z, ptr[imm_addr64]); } void transpose(int nrows, int l_pad, int r_pad, bool nontemporal_stores); @@ -135,7 +137,7 @@ void jit_trans_iw_ic_t::transpose_2b( auto kmovd = [this](Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovd(k, regw_tmp); + jit_generator_t::kmovd(k, regw_tmp); }; int l_pad_tail {0}, l_pad_rows {0}; int r_pad_tail {0}, r_pad_rows {0}; @@ -680,11 +682,11 @@ void jit_trans_iw_ic_t::generate() { postamble(); } -struct jit_trans_ow_oc_t : public jit_trans_dst_t, public jit_generator { +struct jit_trans_ow_oc_t : public jit_trans_dst_t, public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_trans_ow_oc_t) jit_trans_ow_oc_t(const jit_conv_conf_t *conf) : jit_trans_dst_t(conf) - , jit_generator(jit_name()) + , jit_generator_t(jit_name()) , typesize(conf->dst_dt == data_type::undef ? 2 : types::data_type_size(conf->dst_dt)) @@ -694,9 +696,11 @@ struct jit_trans_ow_oc_t : public jit_trans_dst_t, public jit_generator { ? 2 : data_type_vnni_granularity(conf->dst_dt)) {} - void operator()(ctx_t *ctx) override { jit_generator::operator()(ctx); } + void operator()(ctx_t *ctx) override { jit_generator_t::operator()(ctx); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } private: int typesize = 0; @@ -725,15 +729,15 @@ struct jit_trans_ow_oc_t : public jit_trans_dst_t, public jit_generator { void vmovdqa64(Zmm z, const int64_t *addr) { mov(imm_addr64, reinterpret_cast(addr)); - jit_generator::vmovdqa64(z, ptr[imm_addr64]); + jit_generator_t::vmovdqa64(z, ptr[imm_addr64]); } void kmovw(Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovw(k, regw_tmp); + jit_generator_t::kmovw(k, regw_tmp); } void kmovd(Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovd(k, regw_tmp); + jit_generator_t::kmovd(k, regw_tmp); } Zmm src_zmm(int i) { return Zmm(i); } Ymm src_ymm(int i) { @@ -1211,17 +1215,17 @@ void jit_transpose4x16_src::generate() { auto kmovw = [this](Opmask k, unsigned w) { mov(regw_tmp, w); - jit_generator::kmovw(k, regw_tmp); + jit_generator_t::kmovw(k, regw_tmp); }; auto vmovdqa64 = [this](Zmm z, const int64_t *addr) { mov(imm_addr64, reinterpret_cast(addr)); - jit_generator::vmovdqa64(z, ptr[imm_addr64]); + jit_generator_t::vmovdqa64(z, ptr[imm_addr64]); }; auto vmovdqa32 = [this](Zmm z, const int32_t *addr) { mov(imm_addr64, reinterpret_cast(addr)); - jit_generator::vmovdqa32(z, ptr[imm_addr64]); + jit_generator_t::vmovdqa32(z, ptr[imm_addr64]); }; kmovw(kF0, 0xf0); // 11110000 diff --git a/src/cpu/x64/jit_transpose_utils.hpp b/src/cpu/x64/jit_transpose_utils.hpp index 720493a95c9..146c3cc6247 100644 --- a/src/cpu/x64/jit_transpose_utils.hpp +++ b/src/cpu/x64/jit_transpose_utils.hpp @@ -75,12 +75,12 @@ struct jit_transpose4x16_src_t { bool tr_src_pf1; }; -struct jit_transpose4x16_src : public jit_generator { +struct jit_transpose4x16_src : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_transpose4x16_src) jit_transpose4x16_src(const jit_1x1_conv_conf_t *aparams, jit_transpose4x16_src_t *tparams_) - : jit_generator(jit_name()), params(aparams), tparams(tparams_) {} + : jit_generator_t(jit_name()), params(aparams), tparams(tparams_) {} const jit_1x1_conv_conf_t *params; const jit_transpose4x16_src_t *tparams; @@ -117,13 +117,13 @@ struct jit_transpose4x16_src : public jit_generator { void generate() override; }; -struct jit_diff_wei_trans_to_vnni_t : public jit_generator { +struct jit_diff_wei_trans_to_vnni_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_diff_wei_trans_to_vnni_t) jit_diff_wei_trans_to_vnni_t(const data_type_t dt, const int &kd, const int &kh, const int &kw, const int &ic_block, const int &oc_block, const int nb_ic) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , out_dt_(dt) , kd_(kd) , kh_(kh) @@ -134,7 +134,9 @@ struct jit_diff_wei_trans_to_vnni_t : public jit_generator { ~jit_diff_wei_trans_to_vnni_t() override = default; - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } const data_type_t out_dt_; const int kd_, kh_, kw_; diff --git a/src/cpu/x64/jit_uni_1x1_conv_utils.hpp b/src/cpu/x64/jit_uni_1x1_conv_utils.hpp index 4609a98adeb..1084df95af6 100644 --- a/src/cpu/x64/jit_uni_1x1_conv_utils.hpp +++ b/src/cpu/x64/jit_uni_1x1_conv_utils.hpp @@ -125,7 +125,7 @@ inline void rtus_prepare_space_info(conv_pd_t *self, } template -struct rtus_driver_t : public jit_generator { +struct rtus_driver_t : public jit_generator_t { struct call_params_t { const void *ws; /* reduced image (w/ strides = 1) */ @@ -169,7 +169,7 @@ struct rtus_driver_t : public jit_generator { rtus_driver_t(int iw, int stride_w, int src_step_h, int src_step_icb, int ws_step_icb, bool src_to_ws, size_t typesize, int ic, bool is_nspc = false) - : jit_generator(jit_name(), isa) + : jit_generator_t(jit_name(), isa) , iw_(iw) , stride_w_(stride_w) , src_step_h_(src_step_h) diff --git a/src/cpu/x64/jit_uni_batch_normalization.cpp b/src/cpu/x64/jit_uni_batch_normalization.cpp index 693b34efacc..31d480f5828 100644 --- a/src/cpu/x64/jit_uni_batch_normalization.cpp +++ b/src/cpu/x64/jit_uni_batch_normalization.cpp @@ -215,7 +215,7 @@ struct jit_bnorm_conf_t { }; template -struct jit_bnorm_t : public jit_generator { +struct jit_bnorm_t : public jit_generator_t { struct call_params_t { // keep all sizes at 8 bytes -- jit code expects this size_t N_ithr, N_nthr; @@ -2074,7 +2074,7 @@ struct jit_bnorm_t : public jit_generator { } jit_bnorm_t(const batch_normalization_pd_t *pd, const jit_bnorm_conf_t *jbp) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , pd_(pd) , jbp_(jbp) , is_bf16_(pd_->src_md()->data_type == data_type::bf16) @@ -2121,7 +2121,7 @@ struct jit_bnorm_t : public jit_generator { postamble(); } - void operator()(const call_params_t *p) { jit_generator::operator()(p); } + void operator()(const call_params_t *p) { jit_generator_t::operator()(p); } ~jit_bnorm_t() override { delete bf16_emu_; } }; diff --git a/src/cpu/x64/jit_uni_batch_normalization_s8.cpp b/src/cpu/x64/jit_uni_batch_normalization_s8.cpp index 1b21c77165e..e0a609ee118 100644 --- a/src/cpu/x64/jit_uni_batch_normalization_s8.cpp +++ b/src/cpu/x64/jit_uni_batch_normalization_s8.cpp @@ -45,7 +45,7 @@ struct jit_uni_bnorm_s8_call_params_t { }; template -struct jit_bnorm_base_t : public jit_generator { +struct jit_bnorm_base_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_bnorm_s8_t) @@ -208,7 +208,7 @@ struct jit_bnorm_base_t : public jit_generator { } jit_bnorm_base_t(const batch_normalization_pd_t *pd) - : jit_generator(jit_name()), pd_(pd) {} + : jit_generator_t(jit_name()), pd_(pd) {} }; template diff --git a/src/cpu/x64/jit_uni_binary_kernel.cpp b/src/cpu/x64/jit_uni_binary_kernel.cpp index 16fbdd97ca8..02d934d3bc0 100644 --- a/src/cpu/x64/jit_uni_binary_kernel.cpp +++ b/src/cpu/x64/jit_uni_binary_kernel.cpp @@ -41,7 +41,7 @@ static bool is_ne_xf16_supported(cpu_isa_t isa, const data_type_t dtype) { binary_kernel_t::binary_kernel_t(const size_t vlen, const binary_pd_t *pd, const jit_binary_conf_t conf, const char *name, bool tail_kernel) - : jit_generator(name, conf.isa) + : jit_generator_t(name, conf.isa) , vlen_(vlen) , simd_w_(vlen / sizeof(float)) , pd_(pd) diff --git a/src/cpu/x64/jit_uni_binary_kernel.hpp b/src/cpu/x64/jit_uni_binary_kernel.hpp index 85021a168b4..b15d5358a86 100644 --- a/src/cpu/x64/jit_uni_binary_kernel.hpp +++ b/src/cpu/x64/jit_uni_binary_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2021-2024 Intel Corporation +* Copyright 2021-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -38,7 +38,7 @@ namespace x64 { using namespace Xbyak; -struct binary_kernel_t : public jit_generator { +struct binary_kernel_t : public jit_generator_t { using op_t = binary_op_t; using bcast_t = binary_bcast_t; @@ -47,7 +47,7 @@ struct binary_kernel_t : public jit_generator { bool tail_kernel = false); ~binary_kernel_t() override = default; - void operator()(jit_binary_call_s *p) { jit_generator::operator()(p); } + void operator()(jit_binary_call_s *p) { jit_generator_t::operator()(p); } size_t simd_w() const noexcept { return simd_w_; } size_t vlen() const noexcept { return vlen_; } diff --git a/src/cpu/x64/jit_uni_convert_xf16.cpp b/src/cpu/x64/jit_uni_convert_xf16.cpp index bb2311e9788..1ed365aeaaa 100644 --- a/src/cpu/x64/jit_uni_convert_xf16.cpp +++ b/src/cpu/x64/jit_uni_convert_xf16.cpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2022-2023 Intel Corporation +* Copyright 2022-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -331,7 +331,7 @@ void jit_uni_cvt_xf16_to_ps_t::convert_xf16( template struct helper_avx2_cvt_xf16_t { - static void convert_xf16(jit_generator *host, + static void convert_xf16(jit_generator_t *host, const impl::data_type_t input_dt, const Xbyak::Address in_addr, const int even_src, const int odd_src, const int tmp_1, const int tmp_2) { diff --git a/src/cpu/x64/jit_uni_convert_xf16.hpp b/src/cpu/x64/jit_uni_convert_xf16.hpp index 17ca3c6aa61..6e46e25a069 100644 --- a/src/cpu/x64/jit_uni_convert_xf16.hpp +++ b/src/cpu/x64/jit_uni_convert_xf16.hpp @@ -51,12 +51,12 @@ struct jit_cvt_xf16_to_ps_params_t { } // namespace cvt_xf16_support template -struct jit_uni_cvt_ps_to_xf16_t : public jit_generator { +struct jit_uni_cvt_ps_to_xf16_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_cvt_ps_to_xf16_t) jit_uni_cvt_ps_to_xf16_t(impl::data_type_t dt, size_t nelems = 0) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , output_dt_(dt) , nelems_(nelems) , is_dynamic_size_(nelems_ == 0) @@ -152,18 +152,18 @@ struct jit_cvt_ps_to_xf16_t { } private: - std::unique_ptr kernel_; + std::unique_ptr kernel_; const size_t nelems_; }; template -struct jit_uni_cvt_xf16_to_ps_t : public jit_generator { +struct jit_uni_cvt_xf16_to_ps_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_cvt_xf16_to_ps_t) jit_uni_cvt_xf16_to_ps_t( impl::data_type_t dt, bool with_add, size_t row_stride) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , input_dt_(dt) , with_add_(with_add) , row_stride_(row_stride) { @@ -267,7 +267,7 @@ struct jit_cvt_xf16_to_ps_t { } private: - std::unique_ptr kernel_; + std::unique_ptr kernel_; }; } // namespace x64 diff --git a/src/cpu/x64/jit_uni_deconv_zp_pad_str_kernel.cpp b/src/cpu/x64/jit_uni_deconv_zp_pad_str_kernel.cpp index dc8571ac5e6..e1f1b06ee99 100644 --- a/src/cpu/x64/jit_uni_deconv_zp_pad_str_kernel.cpp +++ b/src/cpu/x64/jit_uni_deconv_zp_pad_str_kernel.cpp @@ -30,7 +30,7 @@ namespace zp { jit_uni_deconv_zp_pad_str_kernel_base_t:: jit_uni_deconv_zp_pad_str_kernel_base_t(const jit_conv_conf_t &jcp) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , jcp_(jcp) , tail_size_(jcp.is_depthwise ? jcp.ngroups % jcp.ch_block : jcp.oc_without_padding % jcp.oc_block) {} @@ -155,7 +155,7 @@ void jit_uni_deconv_zp_pad_str_kernel_t::compute_step( template > struct helper_store_t { - static void store(jit_generator *gen, const Vmm &vmm, + static void store(jit_generator_t *gen, const Vmm &vmm, const Xbyak::Reg64 ®_dst, const size_t size, const Xbyak::Opmask &opmask) { gen->store_bytes(vmm, reg_dst, 0, size); @@ -165,7 +165,7 @@ struct helper_store_t { using isa_at_least_avx512_core = std::false_type; template struct helper_store_t { - static void store(jit_generator *gen, const Vmm &vmm, + static void store(jit_generator_t *gen, const Vmm &vmm, const Xbyak::Reg64 ®_dst, const size_t size, const Xbyak::Opmask &opmask) { using namespace Xbyak::util; diff --git a/src/cpu/x64/jit_uni_deconv_zp_pad_str_kernel.hpp b/src/cpu/x64/jit_uni_deconv_zp_pad_str_kernel.hpp index 9b5707804d8..462bf8c32a4 100644 --- a/src/cpu/x64/jit_uni_deconv_zp_pad_str_kernel.hpp +++ b/src/cpu/x64/jit_uni_deconv_zp_pad_str_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2021 Intel Corporation +* Copyright 2021-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -43,12 +43,12 @@ struct jit_uni_deconv_zp_pad_str_call_params_t { * * output_format - dhwc */ -class jit_uni_deconv_zp_pad_str_kernel_base_t : public jit_generator { +class jit_uni_deconv_zp_pad_str_kernel_base_t : public jit_generator_t { public: jit_uni_deconv_zp_pad_str_kernel_base_t(const jit_conv_conf_t &jcp); void operator()(const jit_uni_deconv_zp_pad_str_call_params_t *params) { - jit_generator::operator()(params); + jit_generator_t::operator()(params); } DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_deconv_zp_pad_str_kernel_base_t); diff --git a/src/cpu/x64/jit_uni_dw_conv_kernel_f32.cpp b/src/cpu/x64/jit_uni_dw_conv_kernel_f32.cpp index 682bca89025..5dfd3c64162 100644 --- a/src/cpu/x64/jit_uni_dw_conv_kernel_f32.cpp +++ b/src/cpu/x64/jit_uni_dw_conv_kernel_f32.cpp @@ -38,7 +38,7 @@ using namespace Xbyak; template jit_uni_dw_conv_fwd_kernel_f32::jit_uni_dw_conv_fwd_kernel_f32( const jit_conv_conf_t &ajcp, const memory_desc_t &dst_md) - : jit_generator(jit_name(), isa), jcp(ajcp) { + : jit_generator_t(jit_name(), isa), jcp(ajcp) { if (jcp.with_eltwise || jcp.with_binary) { using namespace binary_injector; static constexpr bool preserve_gpr = true; diff --git a/src/cpu/x64/jit_uni_dw_conv_kernel_f32.hpp b/src/cpu/x64/jit_uni_dw_conv_kernel_f32.hpp index 32609788e47..ff08915eccd 100644 --- a/src/cpu/x64/jit_uni_dw_conv_kernel_f32.hpp +++ b/src/cpu/x64/jit_uni_dw_conv_kernel_f32.hpp @@ -32,7 +32,7 @@ namespace cpu { namespace x64 { template -struct jit_uni_dw_conv_fwd_kernel_f32 : public jit_generator { +struct jit_uni_dw_conv_fwd_kernel_f32 : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_dw_conv_fwd_kernel_f32) jit_uni_dw_conv_fwd_kernel_f32( @@ -126,11 +126,11 @@ struct jit_uni_dw_conv_fwd_kernel_f32 : public jit_generator { }; template -struct jit_uni_dw_conv_bwd_data_kernel_f32 : public jit_generator { +struct jit_uni_dw_conv_bwd_data_kernel_f32 : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_dw_conv_bwd_data_kernel_f32) jit_uni_dw_conv_bwd_data_kernel_f32(const jit_conv_conf_t &ajcp) - : jit_generator(jit_name()), jcp(ajcp) {} + : jit_generator_t(jit_name()), jcp(ajcp) {} jit_conv_conf_t jcp; private: @@ -190,12 +190,12 @@ struct jit_uni_dw_conv_bwd_data_kernel_f32 : public jit_generator { }; template -struct jit_uni_dw_conv_bwd_weights_kernel_f32 : public jit_generator { +struct jit_uni_dw_conv_bwd_weights_kernel_f32 : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_dw_conv_bwd_weights_kernel_f32) jit_uni_dw_conv_bwd_weights_kernel_f32(const jit_conv_conf_t &ajcp) - : jit_generator(jit_name()), jcp(ajcp) {} + : jit_generator_t(jit_name()), jcp(ajcp) {} jit_conv_conf_t jcp; diff --git a/src/cpu/x64/jit_uni_dw_conv_kernel_utils.hpp b/src/cpu/x64/jit_uni_dw_conv_kernel_utils.hpp index 7f5902e055b..57b6b67ea84 100644 --- a/src/cpu/x64/jit_uni_dw_conv_kernel_utils.hpp +++ b/src/cpu/x64/jit_uni_dw_conv_kernel_utils.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2023 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -56,7 +56,7 @@ struct jit_uni_dw_conv_fwd_kernel { static void init_scratchpad(memory_tracking::registrar_t &scratchpad, const jit_conv_conf_t &jcp); - jit_generator *ker() const { return ker_.get(); } + jit_generator_t *ker() const { return ker_.get(); } void operator()(const jit_conv_call_s *p) const { (*ker_)(p); } private: diff --git a/src/cpu/x64/jit_uni_eltwise.cpp b/src/cpu/x64/jit_uni_eltwise.cpp index 56e6056c0b0..e50a3552328 100644 --- a/src/cpu/x64/jit_uni_eltwise.cpp +++ b/src/cpu/x64/jit_uni_eltwise.cpp @@ -43,12 +43,12 @@ struct jit_args_t { size_t work_amount; }; -struct jit_uni_eltwise_kernel : public jit_generator { +struct jit_uni_eltwise_kernel : public jit_generator_t { jit_uni_eltwise_kernel( const eltwise_pd_t *pd, const char *name, cpu_isa_t isa) - : jit_generator(name, isa), pd_(pd) {} + : jit_generator_t(name, isa), pd_(pd) {} - void operator()(jit_args_t *p) { jit_generator::operator()(p); } + void operator()(jit_args_t *p) { jit_generator_t::operator()(p); } protected: const eltwise_pd_t *pd_; diff --git a/src/cpu/x64/jit_uni_eltwise_int.cpp b/src/cpu/x64/jit_uni_eltwise_int.cpp index b5eb5a6d4b6..7b9f422937a 100644 --- a/src/cpu/x64/jit_uni_eltwise_int.cpp +++ b/src/cpu/x64/jit_uni_eltwise_int.cpp @@ -37,12 +37,12 @@ struct jit_args_int8_t { size_t work_amount; }; -struct jit_uni_eltwise_int_kernel : public jit_generator { +struct jit_uni_eltwise_int_kernel : public jit_generator_t { jit_uni_eltwise_int_kernel( const eltwise_pd_t *pd, const cpu_isa_t isa, const char *name) - : jit_generator(name, isa), pd_(pd) {} + : jit_generator_t(name, isa), pd_(pd) {} - void operator()(jit_args_int8_t *p) { jit_generator::operator()(p); } + void operator()(jit_args_int8_t *p) { jit_generator_t::operator()(p); } protected: data_type_t data_type() const { return pd_->src_md()->data_type; } diff --git a/src/cpu/x64/jit_uni_group_normalization.cpp b/src/cpu/x64/jit_uni_group_normalization.cpp index 37929887fc9..4fd6acdc5e2 100644 --- a/src/cpu/x64/jit_uni_group_normalization.cpp +++ b/src/cpu/x64/jit_uni_group_normalization.cpp @@ -61,12 +61,12 @@ const bcast_set_t &get_supported_bcast_strategies() { template struct kernel_t : public jit_uni_group_normalization_fwd_t::kernel_base_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_group_normalization_fwd_t::kernel_t); kernel_t(const group_normalization_pd_t *pd) : jit_uni_group_normalization_fwd_t::kernel_base_t(pd) - , jit_generator(jit_name(), isa) + , jit_generator_t(jit_name(), isa) , src_d_(pd->src_md()) , dst_d_(pd->dst_md()) , C_(pd->C()) @@ -112,7 +112,9 @@ struct kernel_t : public jit_uni_group_normalization_fwd_t::kernel_base_t, axis_simd_tail_, use_scale_, use_shift_); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } void generate() override { const size_t c_src_size = C_ * types::data_type_size(src_d_.data_type()); @@ -216,7 +218,7 @@ struct kernel_t : public jit_uni_group_normalization_fwd_t::kernel_base_t, args.eps = eps_; args.post_ops_binary_rhs_arg_vec = post_ops_binary_rhs_arg_vec; - jit_generator::operator()(&args); + jit_generator_t::operator()(&args); } protected: @@ -385,12 +387,12 @@ template struct kernel_t; template struct kernel_stat_t : public jit_uni_group_normalization_fwd_t::kernel_stat_base_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS( jit_uni_group_normalization_fwd_t::kernel_stat_t); kernel_stat_t(const group_normalization_pd_t *pd, bool compute_var = false) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , src_d_(pd->src_md()) , compute_var_(compute_var) , C_(pd->C()) @@ -433,7 +435,9 @@ struct kernel_stat_t unroll_c_tail_); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } void generate() override { preamble(); @@ -552,7 +556,7 @@ struct kernel_stat_t args.block_size = block_size * C_ * types::data_type_size(src_d_.data_type()); - jit_generator::operator()(&args); + jit_generator_t::operator()(&args); } void operator()(const void *src, const float *mean, float *var, @@ -564,7 +568,7 @@ struct kernel_stat_t args.block_size = block_size * C_ * types::data_type_size(src_d_.data_type()); - jit_generator::operator()(&args); + jit_generator_t::operator()(&args); } protected: diff --git a/src/cpu/x64/jit_uni_i8i8_pooling.cpp b/src/cpu/x64/jit_uni_i8i8_pooling.cpp index 57302b9d517..96c56fd6637 100644 --- a/src/cpu/x64/jit_uni_i8i8_pooling.cpp +++ b/src/cpu/x64/jit_uni_i8i8_pooling.cpp @@ -66,7 +66,7 @@ struct jit_uni_i8i8_pool_call_params_t { }; template -struct jit_uni_i8i8_pooling_fwd_ker_t : public jit_generator { +struct jit_uni_i8i8_pooling_fwd_ker_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_i8i8_pooling_fwd_ker_t) using Vmm = typename cpu_isa_traits_t::Vmm; @@ -232,7 +232,7 @@ struct jit_uni_i8i8_pooling_fwd_ker_t : public jit_generator { jit_uni_i8i8_pooling_fwd_ker_t( const jit_pool_conf_t &jpp_, const memory_desc_t *dst_md) - : jit_generator(jit_name(), isa) + : jit_generator_t(jit_name(), isa) , jpp(jpp_) , postops_injector_(nullptr) { diff --git a/src/cpu/x64/jit_uni_instance_normalization.cpp b/src/cpu/x64/jit_uni_instance_normalization.cpp index 7b6274b0f29..94ac0cea5cd 100644 --- a/src/cpu/x64/jit_uni_instance_normalization.cpp +++ b/src/cpu/x64/jit_uni_instance_normalization.cpp @@ -59,13 +59,13 @@ const bcast_set_t &get_supported_bcast_strategies() { template struct kernel_t : public jit_uni_instance_normalization_fwd_t::kernel_base_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS( jit_uni_instance_normalization_fwd_t::kernel_t); kernel_t(const group_normalization_pd_t *pd) : jit_uni_instance_normalization_fwd_t::kernel_base_t(pd) - , jit_generator(jit_name(), isa) + , jit_generator_t(jit_name(), isa) , src_d_(pd->src_md()) , dst_d_(pd->dst_md()) , C_(pd->C()) @@ -102,7 +102,9 @@ struct kernel_t : public jit_uni_instance_normalization_fwd_t::kernel_base_t, {{dst_d_.data_type(), io_saturation_conf}}); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } void generate() override { const size_t c_src_size = C_ * types::data_type_size(src_d_.data_type()); @@ -206,7 +208,7 @@ struct kernel_t : public jit_uni_instance_normalization_fwd_t::kernel_base_t, args.eps = eps_; args.post_ops_binary_rhs_arg_vec = post_ops_binary_rhs_arg_vec; - jit_generator::operator()(&args); + jit_generator_t::operator()(&args); } protected: @@ -374,12 +376,12 @@ template struct kernel_t; template struct kernel_stat_t : public jit_uni_instance_normalization_fwd_t::kernel_stat_base_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS( jit_uni_instance_normalization_fwd_t::kernel_stat_t); kernel_stat_t(const group_normalization_pd_t *pd, bool compute_var = false) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , src_d_(pd->src_md()) , compute_var_(compute_var) , C_(pd->C()) @@ -404,7 +406,9 @@ struct kernel_stat_t {src_d_.data_type(), f32 /* stats */}, io_conf, io_tail_conf, io_bf16_conf); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } void generate() override { preamble(); @@ -461,7 +465,7 @@ struct kernel_stat_t args.block_size = block_size * C_ * types::data_type_size(src_d_.data_type()); - jit_generator::operator()(&args); + jit_generator_t::operator()(&args); } void operator()(const void *src, const float *mean, float *var, @@ -473,7 +477,7 @@ struct kernel_stat_t args.block_size = block_size * C_ * types::data_type_size(src_d_.data_type()); - jit_generator::operator()(&args); + jit_generator_t::operator()(&args); } protected: diff --git a/src/cpu/x64/jit_uni_layer_normalization.cpp b/src/cpu/x64/jit_uni_layer_normalization.cpp index ab1a28f20fb..dc0f7738d07 100644 --- a/src/cpu/x64/jit_uni_layer_normalization.cpp +++ b/src/cpu/x64/jit_uni_layer_normalization.cpp @@ -78,7 +78,7 @@ static bcast_set_t get_supported_bcast_strategies(int ndims) { template struct jit_stat_and_data_base_kernel_t : stat_and_data_kernel_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_lnorm_stat_and_data_kernel_t); void operator()(const void *src, void *dst, const float *scale, @@ -99,14 +99,16 @@ struct jit_stat_and_data_base_kernel_t : stat_and_data_kernel_t, = block_size * C_ * types::data_type_size(src_d_.data_type()); args.eps = eps_; args.post_ops_binary_rhs_arg_vec = post_ops_binary_rhs_arg_vec; - jit_generator::operator()(&args); + jit_generator_t::operator()(&args); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } jit_stat_and_data_base_kernel_t(const layer_normalization_pd_t *pd) : stat_and_data_kernel_t(pd) - , jit_generator(jit_name(), isa) + , jit_generator_t(jit_name(), isa) , src_d_(pd_->src_md()) , dst_d_(pd_->dst_md()) , simd_w_(vlen / sizeof(float)) @@ -708,7 +710,7 @@ stat_and_data_kernel_t *stat_and_data_kernel_t::create( } template -struct jit_diff_ss_kernel_t : diff_ss_kernel_t, public jit_generator { +struct jit_diff_ss_kernel_t : diff_ss_kernel_t, public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_lnorm_diff_ss_kernel_t); void operator()(const void *src, const void *diff_dst, float *diff_scale, @@ -733,14 +735,16 @@ struct jit_diff_ss_kernel_t : diff_ss_kernel_t, public jit_generator { args.inv_sqrtvar = inv_sqrtvar; args.block_size = block_size * C_ * types::data_type_size(src_d_.data_type()); - jit_generator::operator()(&args); + jit_generator_t::operator()(&args); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } jit_diff_ss_kernel_t(const layer_normalization_pd_t *pd) : diff_ss_kernel_t(pd) - , jit_generator(jit_name()) + , jit_generator_t(jit_name()) , src_d_(pd_->src_md()) , d_dst_d_(pd_->diff_dst_md()) , simd_w_(vlen / sizeof(float)) @@ -909,7 +913,8 @@ diff_ss_kernel_t *diff_ss_kernel_t::create(const layer_normalization_pd_t *pd) { } template -struct jit_diff_data_base_kernel_t : diff_data_kernel_t, public jit_generator { +struct jit_diff_data_base_kernel_t : diff_data_kernel_t, + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_lnorm_diff_data_kernel_t); void operator()(const void *src, const void *diff_dst, void *diff_src, @@ -924,14 +929,16 @@ struct jit_diff_data_base_kernel_t : diff_data_kernel_t, public jit_generator { args.inv_sqrtvar = inv_sqrtvar; args.block_size = block_size * C_ * types::data_type_size(src_d_.data_type()); - jit_generator::operator()(&args); + jit_generator_t::operator()(&args); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } jit_diff_data_base_kernel_t(const layer_normalization_pd_t *pd) : diff_data_kernel_t(pd) - , jit_generator(jit_name()) + , jit_generator_t(jit_name()) , src_d_(pd_->src_md()) , d_dst_d_(pd_->diff_dst_md()) , d_src_d_(pd_->diff_src_md()) diff --git a/src/cpu/x64/jit_uni_pool_kernel.cpp b/src/cpu/x64/jit_uni_pool_kernel.cpp index e762b481cca..aa9f2dc9b56 100644 --- a/src/cpu/x64/jit_uni_pool_kernel.cpp +++ b/src/cpu/x64/jit_uni_pool_kernel.cpp @@ -48,7 +48,7 @@ jit_uni_pool_kernel::~jit_uni_pool_kernel() = default; template jit_uni_pool_kernel::jit_uni_pool_kernel( const jit_pool_conf_t &ajpp, const memory_desc_t *dst_md) - : jit_generator(jit_name(), isa), jpp(ajpp) { + : jit_generator_t(jit_name(), isa), jpp(ajpp) { bool has_f8_e5m2_binary_postops = false; bool has_f8_e4m3_binary_postops = false; diff --git a/src/cpu/x64/jit_uni_pool_kernel.hpp b/src/cpu/x64/jit_uni_pool_kernel.hpp index 56aa1f664af..a080d97b6ec 100644 --- a/src/cpu/x64/jit_uni_pool_kernel.hpp +++ b/src/cpu/x64/jit_uni_pool_kernel.hpp @@ -35,7 +35,7 @@ namespace cpu { namespace x64 { template -struct jit_uni_pool_kernel : public jit_generator { +struct jit_uni_pool_kernel : public jit_generator_t { jit_uni_pool_kernel( const jit_pool_conf_t &ajpp, const memory_desc_t *dst_md); diff --git a/src/cpu/x64/jit_uni_reduction_kernel.hpp b/src/cpu/x64/jit_uni_reduction_kernel.hpp index ac0d320cdb9..0c824b00036 100644 --- a/src/cpu/x64/jit_uni_reduction_kernel.hpp +++ b/src/cpu/x64/jit_uni_reduction_kernel.hpp @@ -33,11 +33,11 @@ namespace impl { namespace cpu { namespace x64 { -struct jit_uni_reduction_kernel_base_t : public jit_generator { +struct jit_uni_reduction_kernel_base_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_reduction) jit_uni_reduction_kernel_base_t(const jit_reduction_conf_t &conf) - : jit_generator(jit_name(), conf.isa) + : jit_generator_t(jit_name(), conf.isa) , conf_(conf) , sum_scales_(conf_.sum_scales) {} ~jit_uni_reduction_kernel_base_t() override = default; diff --git a/src/cpu/x64/jit_uni_reorder.cpp b/src/cpu/x64/jit_uni_reorder.cpp index 0ed3361133c..97d1df52434 100644 --- a/src/cpu/x64/jit_uni_reorder.cpp +++ b/src/cpu/x64/jit_uni_reorder.cpp @@ -90,17 +90,19 @@ bool prb_has_huge_prime_number(const prb_t &prb) { const size_t ker_prb_size_min = 64; /* kernel */ -struct jit_uni_reorder_kernel_f32_t : public kernel_t, public jit_generator { +struct jit_uni_reorder_kernel_f32_t : public kernel_t, public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_reorder_kernel_f32) void operator()(const call_param_t *c) const override { - jit_generator::operator()(c); + jit_generator_t::operator()(c); } void operator()(const tail_call_param_t *c) const override { - jit_generator::operator()(c); + jit_generator_t::operator()(c); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } enum class scale_arg_t { NONE, SRC, DST }; @@ -1523,7 +1525,7 @@ struct jit_uni_reorder_kernel_f32_t : public kernel_t, public jit_generator { jit_uni_reorder_kernel_f32_t(const desc_t &desc) : kernel_t(desc) - , jit_generator(jit_name()) + , jit_generator_t(jit_name()) , isa_(get_max_cpu_isa()) , bf16_emu_(nullptr) , f8_e5m2_emu_(nullptr) @@ -1731,7 +1733,7 @@ struct jit_uni_reorder_kernel_f32_t : public kernel_t, public jit_generator { }; // Seperate class for no unroll/threading burden -struct jit_single_blk_kernel_t : public jit_generator { +struct jit_single_blk_kernel_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_single_blk_kernel) static bool applicable(const prb_t &p) { using namespace data_type; @@ -1777,7 +1779,7 @@ struct jit_single_blk_kernel_t : public jit_generator { } jit_single_blk_kernel_t(const tr::prb_t &prb) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , prb_(prb) , itype_sz_(data_type_size(prb_.itype)) , otype_sz_(data_type_size(prb_.otype)) diff --git a/src/cpu/x64/jit_uni_reorder_direct_copy.cpp b/src/cpu/x64/jit_uni_reorder_direct_copy.cpp index f280e6d1965..06738d2aa56 100644 --- a/src/cpu/x64/jit_uni_reorder_direct_copy.cpp +++ b/src/cpu/x64/jit_uni_reorder_direct_copy.cpp @@ -31,12 +31,12 @@ namespace x64 { template struct direct_copy_kernel_t : public jit_uni_reorder_direct_copy_t::kernel_base_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(direct_copy_kernel_t) direct_copy_kernel_t(const reorder_pd_t *pd, cpu_isa_t isa) : jit_uni_reorder_direct_copy_t::kernel_base_t(pd) - , jit_generator(jit_name(), isa) + , jit_generator_t(jit_name(), isa) , isa_(isa) , src_dt_(pd_->src_md()->data_type) , dst_dt_(pd_->dst_md()->data_type) { @@ -75,10 +75,12 @@ struct direct_copy_kernel_t args.src = src; args.dst = dst; args.work_amount = work_amount; - jit_generator::operator()(&args); + jit_generator_t::operator()(&args); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } Address src_ptr(size_t offt = 0) { return ptr[reg_src + offt]; } diff --git a/src/cpu/x64/jit_uni_resampling_kernel.hpp b/src/cpu/x64/jit_uni_resampling_kernel.hpp index 23dd2134e63..c3114112012 100644 --- a/src/cpu/x64/jit_uni_resampling_kernel.hpp +++ b/src/cpu/x64/jit_uni_resampling_kernel.hpp @@ -34,11 +34,11 @@ namespace impl { namespace cpu { namespace x64 { -struct jit_uni_resampling_kernel_base_t : public jit_generator { +struct jit_uni_resampling_kernel_base_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_uni_resampling) jit_uni_resampling_kernel_base_t(const jit_resampling_conf_t &conf) - : jit_generator(jit_name(), conf.isa) + : jit_generator_t(jit_name(), conf.isa) , conf_(conf) , sum_scales_(conf_.sum_scales) {} diff --git a/src/cpu/x64/jit_uni_softmax.cpp b/src/cpu/x64/jit_uni_softmax.cpp index 231b5c6ae17..5a52600152e 100644 --- a/src/cpu/x64/jit_uni_softmax.cpp +++ b/src/cpu/x64/jit_uni_softmax.cpp @@ -51,7 +51,7 @@ using namespace data_type; template struct jit_softmax_dense_kernel_t : jit_softmax_kernel_base_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_softmax_dense_kernel_t) using Vmm = typename cpu_isa_traits_t::Vmm; @@ -139,10 +139,12 @@ struct jit_softmax_dense_kernel_t : jit_softmax_kernel_base_t, Opmask tail_opmask = Opmask(tail_opmask_idx_); void operator()(const call_params_t *p) const override { - return jit_generator::operator()(p); + return jit_generator_t::operator()(p); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } bool is_data_type_xf16(data_type_t dt) { return utils::one_of(dt, bf16, f16); @@ -949,7 +951,7 @@ struct jit_softmax_dense_kernel_t : jit_softmax_kernel_base_t, jit_softmax_dense_kernel_t(const softmax_pd_t *pd) : jit_softmax_kernel_base_t(pd) - , jit_generator(jit_name(), isa) + , jit_generator_t(jit_name(), isa) , src_d_(pd_->invariant_src_md()) , dst_d_(pd_->dst_md()) , diff_dst_d_(pd_->diff_dst_md()) @@ -997,7 +999,7 @@ struct jit_softmax_dense_kernel_t : jit_softmax_kernel_base_t, template struct jit_softmax_strided_kernel_t : jit_softmax_kernel_base_t, - public jit_generator { + public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_softmax_strided_kernel_t) using Vmm = typename cpu_isa_traits_t::Vmm; @@ -1076,10 +1078,12 @@ struct jit_softmax_strided_kernel_t : jit_softmax_kernel_base_t, Opmask tail_opmask = Opmask(tail_opmask_idx_); void operator()(const call_params_t *p) const override { - return jit_generator::operator()(p); + return jit_generator_t::operator()(p); } - status_t create_kernel() override { return jit_generator::create_kernel(); } + status_t create_kernel() override { + return jit_generator_t::create_kernel(); + } void compute_predefined_variables() { // `axis_simd_full_` is actually `inner_simd_full_`. @@ -1504,7 +1508,7 @@ struct jit_softmax_strided_kernel_t : jit_softmax_kernel_base_t, jit_softmax_strided_kernel_t(const softmax_pd_t *pd) : jit_softmax_kernel_base_t(pd) - , jit_generator(jit_name(), isa) + , jit_generator_t(jit_name(), isa) , src_d_(pd_->invariant_src_md()) , dst_d_(pd_->dst_md()) // Note: must be aligned with pd_t::init()->init_scratchpad(); diff --git a/src/cpu/x64/jit_uni_tbb_batch_normalization.cpp b/src/cpu/x64/jit_uni_tbb_batch_normalization.cpp index b5190f46484..6bcf2ec3d07 100644 --- a/src/cpu/x64/jit_uni_tbb_batch_normalization.cpp +++ b/src/cpu/x64/jit_uni_tbb_batch_normalization.cpp @@ -98,7 +98,7 @@ struct jit_bnorm_process_tail_t { using Vmm = typename cpu_isa_traits_t::Vmm; jit_bnorm_process_tail_t(const batch_normalization_pd_t *pd, - jit_generator *host, Reg64 reg_tmp, Reg64 reg_blk_has_tail, + jit_generator_t *host, Reg64 reg_tmp, Reg64 reg_blk_has_tail, Reg64 reg_C, Vmm vtail_mask, Opmask ktail_mask) : h_(host) , reg_tmp_(reg_tmp) @@ -113,7 +113,7 @@ struct jit_bnorm_process_tail_t { tail_ = pd->C() % (int)(vlen / sizeof(float)); } - jit_generator *const h_; + jit_generator_t *const h_; const Reg64 reg_tmp_; const Reg64 reg_blk_has_tail_; const Reg64 reg_C_; @@ -201,7 +201,7 @@ struct jit_bnorm_process_relu_t { using Vmm = typename cpu_isa_traits_t::Vmm; jit_bnorm_process_relu_t(const batch_normalization_pd_t *pd, - jit_generator *host, Reg64 reg_off_dat, Reg64 reg_tmp, + jit_generator_t *host, Reg64 reg_off_dat, Reg64 reg_tmp, Reg64 reg_ptr_ws, Vmm vzero, Vmm vstore_mask, Opmask kstore_mask, Vmm valpha, Vmm vmask, Reg64 reg_alpha) : h_(host) @@ -225,12 +225,12 @@ struct jit_bnorm_process_relu_t { : 0.f) {} jit_bnorm_process_relu_t(const batch_normalization_pd_t *pd, - jit_generator *host, Reg64 reg_off_dat, Reg64 reg_tmp, + jit_generator_t *host, Reg64 reg_off_dat, Reg64 reg_tmp, Reg64 reg_ptr_ws, Vmm vzero, Vmm vstore_mask, Opmask kstore_mask) : jit_bnorm_process_relu_t(pd, host, reg_off_dat, reg_tmp, reg_ptr_ws, vzero, vstore_mask, kstore_mask, Vmm(), Vmm(), Reg64()) {} - jit_generator *const h_; + jit_generator_t *const h_; const Reg64 reg_off_dat_; const Reg64 reg_tmp_; const Reg64 reg_ptr_ws_; @@ -302,7 +302,7 @@ struct jit_bnorm_process_relu_t { void fwd_process_relu_avx2(Vmm vdst, const int off = 0) { Reg64 reg_store_mask = reg_tmp_; h_->shr(reg_off_dat_, bit_shift_); - h_->vcmpps(vstore_mask_, vzero_, vdst, jit_generator::_cmp_lt_os); + h_->vcmpps(vstore_mask_, vzero_, vdst, jit_generator_t::_cmp_lt_os); h_->vmovmskps(reg_store_mask, vstore_mask_); h_->mov(h_->ptr[reg_ptr_ws_ + reg_off_dat_ + off], reg_store_mask.cvt8()); @@ -312,7 +312,7 @@ struct jit_bnorm_process_relu_t { void fwd_process_relu_avx512_common(Vmm vdst, const int off = 0) { h_->shr(reg_off_dat_, bit_shift_); - h_->vcmpps(kstore_mask_, vzero_, vdst, jit_generator::_cmp_lt_os); + h_->vcmpps(kstore_mask_, vzero_, vdst, jit_generator_t::_cmp_lt_os); h_->kmovw(h_->ptr[reg_ptr_ws_ + reg_off_dat_ + off], kstore_mask_); h_->vblendmps(vdst | kstore_mask_, vzero_, vdst); h_->shl(reg_off_dat_, bit_shift_); @@ -331,7 +331,7 @@ struct jit_bnorm_process_relu_t { const Xmm xmm_aux = Xmm(valpha.getIdx()); h_->vmovq(xmm_aux, reg_alpha); h_->vbroadcastss(valpha, xmm_aux); - h_->vcmpps(kstore_mask_, vzero_, vmm_dst, jit_generator::_cmp_lt_os); + h_->vcmpps(kstore_mask_, vzero_, vmm_dst, jit_generator_t::_cmp_lt_os); h_->vmulps(valpha, vmm_dst, valpha); h_->vblendmps(vmm_dst | kstore_mask_, valpha, vmm_dst); } @@ -341,7 +341,7 @@ struct jit_bnorm_process_relu_t { h_->uni_vpxor(vmask, vmask, vmask); h_->uni_vmovq(xmm_aux, reg_alpha); h_->uni_vbroadcastss(valpha, xmm_aux); - h_->uni_vcmpps(vmask, vmm_dst, vzero_, jit_generator::_cmp_lt_os); + h_->uni_vcmpps(vmask, vmm_dst, vzero_, jit_generator_t::_cmp_lt_os); h_->uni_vmulps(valpha, valpha, vmm_dst); h_->uni_vblendvps( vmm_dst, vmm_dst, valpha, vmask); // swaped aux and dst @@ -372,7 +372,7 @@ struct helper_vmovups_data_t { using Vmm = typename cpu_isa_traits_t::Vmm; helper_vmovups_data_t(const batch_normalization_pd_t *pd, - jit_generator *host, Zmm zmm_reserved_1, Zmm zmm_reserved_2, + jit_generator_t *host, Zmm zmm_reserved_1, Zmm zmm_reserved_2, Zmm zmm_reserved_3, Zmm zmm_reserved_4, Reg64 reg_tmp) : h_(host) , bf16_emu_(nullptr) @@ -386,7 +386,7 @@ struct helper_vmovups_data_t { } } - jit_generator *const h_; + jit_generator_t *const h_; std::unique_ptr bf16_emu_; bool is_bf16_; bool is_f16_; @@ -437,7 +437,7 @@ struct helper_vmovups_data_t { } else if (is_f16_) { auto src_reg = Vmm(src.getIdx()); h_->vcvtps2ph( - dst.getAddress(), src_reg, jit_generator::_op_mxcsr); + dst.getAddress(), src_reg, jit_generator_t::_op_mxcsr); } else { h_->uni_vmovups(dst.getAddress(), Vmm(src.getIdx())); } @@ -462,7 +462,7 @@ struct helper_vmovups_data_t { }; template -struct jit_bnorm_fwd_statistics_t : public jit_generator { +struct jit_bnorm_fwd_statistics_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_bnorm_fwd_statistics_t) using Vmm = typename cpu_isa_traits_t::Vmm; @@ -791,7 +791,7 @@ struct jit_bnorm_fwd_statistics_t : public jit_generator { jit_bnorm_fwd_statistics_t(const batch_normalization_pd_t *pd, const jit_memory_tag_kind_t tag_kind) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , pd_(pd) , tag_kind_(tag_kind) , vlen(get_vlen(tag_kind)) @@ -853,7 +853,7 @@ struct jit_bnorm_fwd_var_t : jit_bnorm_fwd_statistics_t { }; template -struct jit_bnorm_fwd_t : public jit_generator { +struct jit_bnorm_fwd_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_bnorm_fwd_t) using Vmm = typename cpu_isa_traits_t::Vmm; @@ -1177,7 +1177,7 @@ struct jit_bnorm_fwd_t : public jit_generator { jit_bnorm_fwd_t(const batch_normalization_pd_t *pd, const jit_memory_tag_kind_t tag_kind) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , pd_(pd) , tag_kind_(tag_kind) , vlen(get_vlen(tag_kind)) @@ -1228,7 +1228,7 @@ struct jit_bnorm_fwd_t : public jit_generator { }; template -struct jit_bnorm_bwd_t : public jit_generator { +struct jit_bnorm_bwd_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_bnorm_bwd_t) using Vmm = typename cpu_isa_traits_t::Vmm; @@ -1465,7 +1465,7 @@ struct jit_bnorm_bwd_t : public jit_generator { jit_bnorm_bwd_t(const batch_normalization_pd_t *pd, const jit_memory_tag_kind_t tag_kind) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , pd_(pd) , tag_kind_(tag_kind) , vlen(get_vlen(tag_kind)) @@ -1514,7 +1514,7 @@ struct jit_bnorm_bwd_t : public jit_generator { }; template -struct jit_bnorm_bwd_diff_ss_t : public jit_generator { +struct jit_bnorm_bwd_diff_ss_t : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(jit_bnorm_bwd_diff_ss_t) using Vmm = typename cpu_isa_traits_t::Vmm; @@ -1863,7 +1863,7 @@ struct jit_bnorm_bwd_diff_ss_t : public jit_generator { jit_bnorm_bwd_diff_ss_t(const batch_normalization_pd_t *pd, const jit_memory_tag_kind_t tag_kind) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , pd_(pd) , tag_kind_(tag_kind) , vlen(get_vlen(tag_kind)) diff --git a/src/cpu/x64/jit_uni_x8s8s32x_1x1_conv_kernel.cpp b/src/cpu/x64/jit_uni_x8s8s32x_1x1_conv_kernel.cpp index 955d4dec412..47ed3d348fe 100644 --- a/src/cpu/x64/jit_uni_x8s8s32x_1x1_conv_kernel.cpp +++ b/src/cpu/x64/jit_uni_x8s8s32x_1x1_conv_kernel.cpp @@ -48,7 +48,7 @@ template _jit_uni_x8s8s32x_1x1_conv_kernel::_jit_uni_x8s8s32x_1x1_conv_kernel( const jit_1x1_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name(), isa), jcp(ajcp), attr_(attr) { + : jit_generator_t(jit_name(), isa), jcp(ajcp), attr_(attr) { if (jcp.with_eltwise || jcp.with_binary || jcp.with_sum) { using namespace binary_injector; static constexpr bool preserve_gpr = true; diff --git a/src/cpu/x64/jit_uni_x8s8s32x_1x1_conv_kernel.hpp b/src/cpu/x64/jit_uni_x8s8s32x_1x1_conv_kernel.hpp index 204d00d0bc5..df82c3aad88 100644 --- a/src/cpu/x64/jit_uni_x8s8s32x_1x1_conv_kernel.hpp +++ b/src/cpu/x64/jit_uni_x8s8s32x_1x1_conv_kernel.hpp @@ -1,5 +1,5 @@ /******************************************************************************* -* Copyright 2019-2023 Intel Corporation +* Copyright 2019-2025 Intel Corporation * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ namespace cpu { namespace x64 { template -struct _jit_uni_x8s8s32x_1x1_conv_kernel : public jit_generator { +struct _jit_uni_x8s8s32x_1x1_conv_kernel : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(_jit_uni_x8s8s32x_1x1_conv_kernel) _jit_uni_x8s8s32x_1x1_conv_kernel(const jit_1x1_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md); @@ -174,7 +174,7 @@ struct jit_uni_x8s8s32x_1x1_conv_kernel { private: DNNL_DISALLOW_COPY_AND_ASSIGN(jit_uni_x8s8s32x_1x1_conv_kernel); - std::unique_ptr kernel_; + std::unique_ptr kernel_; }; } // namespace x64 diff --git a/src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.cpp b/src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.cpp index 580633060c6..4a93c62292c 100644 --- a/src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.cpp +++ b/src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.cpp @@ -57,7 +57,7 @@ template _jit_uni_x8s8s32x_fwd_kernel::_jit_uni_x8s8s32x_fwd_kernel( const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_t &dst_md) - : jit_generator(jit_name(), isa), jcp(ajcp), attr_(attr) { + : jit_generator_t(jit_name(), isa), jcp(ajcp), attr_(attr) { if (jcp.with_eltwise || jcp.with_binary || jcp.with_sum) { using namespace binary_injector; static constexpr bool preserve_gpr = true; diff --git a/src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.hpp b/src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.hpp index 9a7b9a0b85f..b228ed55461 100644 --- a/src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.hpp +++ b/src/cpu/x64/jit_uni_x8s8s32x_conv_kernel.hpp @@ -31,7 +31,7 @@ namespace cpu { namespace x64 { template -struct _jit_uni_x8s8s32x_fwd_kernel : public jit_generator { +struct _jit_uni_x8s8s32x_fwd_kernel : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(_jit_uni_x8s8s32x_conv_fwd_ker_t_) _jit_uni_x8s8s32x_fwd_kernel(const jit_conv_conf_t &ajcp, @@ -232,7 +232,7 @@ struct jit_uni_x8s8s32x_fwd_kernel { private: DNNL_DISALLOW_COPY_AND_ASSIGN(jit_uni_x8s8s32x_fwd_kernel); - std::unique_ptr kernel_; + std::unique_ptr kernel_; }; } // namespace x64 diff --git a/src/cpu/x64/jit_uni_x8s8s32x_deconvolution.cpp b/src/cpu/x64/jit_uni_x8s8s32x_deconvolution.cpp index 3f7a3340050..de940d607d8 100644 --- a/src/cpu/x64/jit_uni_x8s8s32x_deconvolution.cpp +++ b/src/cpu/x64/jit_uni_x8s8s32x_deconvolution.cpp @@ -417,7 +417,7 @@ template _jit_uni_x8s8s32x_deconv_fwd_kernel::_jit_uni_x8s8s32x_deconv_fwd_kernel(const jit_conv_conf_t &ajcp, const primitive_attr_t &attr, const memory_desc_wrapper &dst_d) - : jit_generator(jit_name(), isa) + : jit_generator_t(jit_name(), isa) , jcp_(ajcp) , postops_injector_(nullptr) , ker_max_regs_(jcp_.has_vnni ? 14 : 12) { diff --git a/src/cpu/x64/jit_uni_x8s8s32x_deconvolution.hpp b/src/cpu/x64/jit_uni_x8s8s32x_deconvolution.hpp index 6e1eb663f3c..6079a9ee426 100644 --- a/src/cpu/x64/jit_uni_x8s8s32x_deconvolution.hpp +++ b/src/cpu/x64/jit_uni_x8s8s32x_deconvolution.hpp @@ -45,7 +45,7 @@ class jit_uni_postops_injector_t; using namespace Xbyak; template -struct _jit_uni_x8s8s32x_deconv_fwd_kernel : public jit_generator { +struct _jit_uni_x8s8s32x_deconv_fwd_kernel : public jit_generator_t { DECLARE_CPU_JIT_AUX_FUNCTIONS(_jit_uni_x8s8s32x_deconv_fwd_kernel); _jit_uni_x8s8s32x_deconv_fwd_kernel(const jit_conv_conf_t &ajcp, @@ -180,7 +180,7 @@ struct jit_uni_x8s8s32x_deconv_fwd_kernel { private: DNNL_DISALLOW_COPY_AND_ASSIGN(jit_uni_x8s8s32x_deconv_fwd_kernel); - std::unique_ptr kernel_; + std::unique_ptr kernel_; }; template diff --git a/src/cpu/x64/jit_uni_xf16_sum.hpp b/src/cpu/x64/jit_uni_xf16_sum.hpp index 7fa54259e9b..1538b5bec49 100644 --- a/src/cpu/x64/jit_uni_xf16_sum.hpp +++ b/src/cpu/x64/jit_uni_xf16_sum.hpp @@ -52,9 +52,9 @@ struct jit_sum_call_t { }; template -struct jit_uni_xf16_sum_kernel_t : public jit_generator { +struct jit_uni_xf16_sum_kernel_t : public jit_generator_t { jit_uni_xf16_sum_kernel_t(jit_sum_conf_t ajsp, unsigned int num_acc_iters) - : jit_generator(jit_name()) + : jit_generator_t(jit_name()) , jsp(ajsp) , reg_src {r8, r9, r10, r11, r12, r13, r14, r15} , num_acc_iters(num_acc_iters) {} @@ -328,7 +328,7 @@ struct jit_xf16_sum_t : public primitive_t { private: const pd_t *pd() const { return (const pd_t *)primitive_t::pd().get(); } - std::unique_ptr kernel_; + std::unique_ptr kernel_; }; } // namespace x64 diff --git a/src/cpu/x64/lrn/jit_avx512_common_lrn_bwd_base.cpp b/src/cpu/x64/lrn/jit_avx512_common_lrn_bwd_base.cpp index dcdefb8d6a6..663fea0a27e 100644 --- a/src/cpu/x64/lrn/jit_avx512_common_lrn_bwd_base.cpp +++ b/src/cpu/x64/lrn/jit_avx512_common_lrn_bwd_base.cpp @@ -70,7 +70,7 @@ void jit_avx512_common_lrn_kernel_bwd_t::load_data( template <> void jit_avx512_common_lrn_kernel_bwd_t::store_data( bool nt, const Address addr, Zmm zr) { - this->vcvtps2ph(addr, zr, jit_generator::_op_mxcsr); + this->vcvtps2ph(addr, zr, jit_generator_t::_op_mxcsr); } template <> @@ -104,7 +104,7 @@ void jit_avx512_common_lrn_kernel_bwd_t::load_tail(int tail_value, Reg64 src, int src_mem_offset, int dst_stack_offset, int tmp_load_to_stack_idx_tail) { // TODO: Investigate if this method can be simplified by using mask or - // jit_generator load utilities. + // jit_generator_t load utilities. static constexpr auto src_acc_size = utils::one_of(d_type, bf16, f16) ? acc_bf_16_size : acc_size; auto tmp_xreg = this->xreg(0, tmp_load_to_stack_idx_tail); @@ -217,7 +217,7 @@ void jit_avx512_common_lrn_kernel_bwd_t::store_tail(int tail_value, template jit_avx512_common_lrn_kernel_bwd_t::jit_avx512_common_lrn_kernel_bwd_t( float alpha, float beta, int local_size, const char *name) - : jit_generator(name, avx512_core_bf16) + : jit_generator_t(name, avx512_core_bf16) , local_size_ {local_size - !(local_size % 2)} , z_prev_ {[this]() { std::vector v(this->local_size_ / 2); diff --git a/src/cpu/x64/lrn/jit_avx512_common_lrn_bwd_base.hpp b/src/cpu/x64/lrn/jit_avx512_common_lrn_bwd_base.hpp index ce56f52286c..c9b0167bfb0 100644 --- a/src/cpu/x64/lrn/jit_avx512_common_lrn_bwd_base.hpp +++ b/src/cpu/x64/lrn/jit_avx512_common_lrn_bwd_base.hpp @@ -39,7 +39,7 @@ using namespace Xbyak; using namespace Xbyak::util; template -class jit_avx512_common_lrn_kernel_bwd_t : public jit_generator { +class jit_avx512_common_lrn_kernel_bwd_t : public jit_generator_t { public: jit_avx512_common_lrn_kernel_bwd_t(float alpha, float beta, int local_size, const char *name = jit_name()); diff --git a/src/cpu/x64/lrn/jit_avx512_common_lrn_fwd_base.cpp b/src/cpu/x64/lrn/jit_avx512_common_lrn_fwd_base.cpp index d326afe78b4..7c7a196638b 100644 --- a/src/cpu/x64/lrn/jit_avx512_common_lrn_fwd_base.cpp +++ b/src/cpu/x64/lrn/jit_avx512_common_lrn_fwd_base.cpp @@ -72,7 +72,7 @@ void jit_avx512_common_lrn_kernel_fwd_t::load_tail(int tail_value, int tmp_load_to_stack_idx_tail) { // TODO: Investigate if this method can be simplified by using mask or - // jit_generator load utilities. + // jit_generator_t load utilities. static constexpr auto src_size = sizeof(data_t); auto tmp_xreg = this->xreg(0, tmp_load_to_stack_idx_tail); @@ -111,7 +111,7 @@ void jit_avx512_common_lrn_kernel_fwd_t::load_tail(int tail_value, template <> void jit_avx512_common_lrn_kernel_fwd_t::store_data( const Address addr, Zmm zr, Ymm yr) { - this->vcvtps2ph(addr, zr, jit_generator::_op_mxcsr); + this->vcvtps2ph(addr, zr, jit_generator_t::_op_mxcsr); } template <> @@ -215,7 +215,7 @@ template jit_avx512_common_lrn_kernel_fwd_t::jit_avx512_common_lrn_kernel_fwd_t( prop_kind_t prop_kind, float alpha, float beta, float k, int local_size, const char *name) - : jit_generator(name, avx512_core_bf16) + : jit_generator_t(name, avx512_core_bf16) , pk_(prop_kind) , alpha_(alpha) , beta_(beta) diff --git a/src/cpu/x64/lrn/jit_avx512_common_lrn_fwd_base.hpp b/src/cpu/x64/lrn/jit_avx512_common_lrn_fwd_base.hpp index 9d74709e234..62c86d1e806 100644 --- a/src/cpu/x64/lrn/jit_avx512_common_lrn_fwd_base.hpp +++ b/src/cpu/x64/lrn/jit_avx512_common_lrn_fwd_base.hpp @@ -39,7 +39,7 @@ using namespace Xbyak; using namespace Xbyak::util; template -class jit_avx512_common_lrn_kernel_fwd_t : public jit_generator { +class jit_avx512_common_lrn_kernel_fwd_t : public jit_generator_t { public: jit_avx512_common_lrn_kernel_fwd_t(prop_kind_t prop_kind, float alpha, float beta, float k, int local_size, const char *name = jit_name()); diff --git a/src/cpu/x64/lrn/jit_uni_lrn_kernel.cpp b/src/cpu/x64/lrn/jit_uni_lrn_kernel.cpp index 1ea0628e09f..dcda13174a2 100644 --- a/src/cpu/x64/lrn/jit_uni_lrn_kernel.cpp +++ b/src/cpu/x64/lrn/jit_uni_lrn_kernel.cpp @@ -50,7 +50,7 @@ template