Merge remote-tracking branch 'origin/master' into cpu/compile_mmap

Author: nshchego

.github/workflows/linux_sanitizers.yml

@@ -204,7 +204,7 @@ jobs:
 
   CXX_Unit_Tests:
     name: C++ unit tests
-    if: always()
+    if: ${{ github.repository_owner == 'openvinotoolkit' }}
     needs: Build
     timeout-minutes: 100
     runs-on: 'aks-linux-16-cores-32gb'

src/frontends/pytorch/src/op/addmm.cpp

@@ -73,17 +73,8 @@ OutputVector translate_conv1d_ext(const NodeContext& context) {
     auto bias = context.get_input(2);
     bias = context.mark_node(std::make_shared<ov::op::v1::ConvertLike>(bias, x));
 
-    auto neg_one = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {-1}));
-    auto zero = context.mark_node(v0::Constant::create(element::i32, Shape{1}, {0}));
-    auto shape_x = context.mark_node(std::make_shared<v3::ShapeOf>(x, element::i32));
-    auto x_last_dim = context.mark_node(std::make_shared<v8::Gather>(shape_x, neg_one, zero));
-    auto x_new_shape = context.mark_node(std::make_shared<v0::Concat>(OutputVector{neg_one, x_last_dim}, 0));
-
-    auto x_new = context.mark_node(std::make_shared<v1::Reshape>(x, x_new_shape, false));
-    auto mm = context.mark_node(std::make_shared<v0::MatMul>(x_new, weight));
-    auto addmm = context.mark_node(std::make_shared<v1::Add>(bias, mm));
-    auto size_out = context.mark_node(std::make_shared<v12::ScatterElementsUpdate>(shape_x, neg_one, neg_one, zero));
-    return {context.mark_node(std::make_shared<v1::Reshape>(addmm, size_out, false))};
+    auto mm = context.mark_node(std::make_shared<v0::MatMul>(x, weight));
+    return {context.mark_node(std::make_shared<v1::Add>(mm, bias))};
 };
 
 }  // namespace op

src/plugins/intel_gpu/src/kernel_selector/cl_kernels/mvn_gpu_bfyx_opt.cl

@@ -29,8 +29,6 @@ KERNEL (mvn_gpu_bfyx_opt)(
     float my_sum = 0;
     float tmp;
 
-    __local float lg_storage[SLM_SIZE];
-
     //each WI reads items_num consecutive items from batch*feature
     for (uint i=0; i<items_num; ++i)
     {
@@ -42,23 +40,7 @@ KERNEL (mvn_gpu_bfyx_opt)(
         my_sum += (float)input[data_set_offset + workers_per_data_set * items_num + in_data_set_idx];
     }
 
-    lg_storage[in_data_set_idx] = my_sum;
-
-    barrier(CLK_LOCAL_MEM_FENCE);
-    for (uint offset = workers_per_data_set / 2; offset > 0; offset /= 2) {
-        if (in_data_set_idx < offset) {
-            lg_storage[in_data_set_idx] += lg_storage[in_data_set_idx + offset];
-	}
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
-
-    if (in_data_set_idx == 0)
-    {
-        lg_storage[0] /= data_set_size;
-    }
-    barrier(CLK_LOCAL_MEM_FENCE);
-
-    my_sum = lg_storage[0];
+    my_sum = work_group_reduce_add(my_sum) / data_set_size;
 
 #if NORMALIZE_VARIANCE == 0
     for (uint i=0; i<items_num; ++i) {
@@ -82,7 +64,6 @@ KERNEL (mvn_gpu_bfyx_opt)(
 #   endif
     }
 #else
-    barrier(CLK_LOCAL_MEM_FENCE);
 
     float my_variance = 0.f;
     //each WI reads items_num consecutive items from batch*feature
@@ -100,30 +81,20 @@ KERNEL (mvn_gpu_bfyx_opt)(
         my_variance = fma(tmp, tmp, my_variance);
     }
 
-    lg_storage[in_data_set_idx] = my_variance;
-
-    barrier(CLK_LOCAL_MEM_FENCE);
-
-    for (uint offset = workers_per_data_set / 2; offset > 0; offset /= 2) {
-        if (in_data_set_idx < offset) {
-            lg_storage[in_data_set_idx] += lg_storage[in_data_set_idx + offset];
-	}
-        barrier(CLK_LOCAL_MEM_FENCE);
-    }
+    my_variance = work_group_reduce_add(my_variance);
 
     if (in_data_set_idx == 0)
     {
-        my_variance = lg_storage[0] / data_set_size;
+        my_variance /= data_set_size;
 
 #   if defined EPS_OUTSIDE_SQRT
-        lg_storage[0] = native_powr(native_sqrt(my_variance) + (float)EPSILON, -1.f);
+        my_variance = native_powr(native_sqrt(my_variance) + (float)EPSILON, -1.f);
 #   elif defined EPS_INSIDE_SQRT
-        lg_storage[0] = native_powr(my_variance + (float)EPSILON, -0.5f);
+        my_variance = native_powr(my_variance + (float)EPSILON, -0.5f);
 #   endif
     }
-    barrier(CLK_LOCAL_MEM_FENCE);
 
-    my_variance = lg_storage[0];
+    my_variance = work_group_broadcast(my_variance, 0);
 
     for (uint i=0; i<items_num; ++i) {
         uint iteration_in_data_set_offset = i * workers_per_data_set;

src/plugins/intel_gpu/src/kernel_selector/cl_kernels/select_gpu_ref.cl

@@ -45,7 +45,11 @@ KERNEL(select)(
     uint output_offset = OUTPUT_GET_INDEX(b, f, y, x);
 #endif
 
-    const OUTPUT_TYPE res = TO_OUTPUT_TYPE(select(INPUT_2, INPUT_1, MASK));
+    #if INPUT1_IS_FP && !OUTPUT_IS_FP
+     const OUTPUT_TYPE res = TO_OUTPUT_TYPE(convert_long(select(INPUT_2, INPUT_1, MASK)));
+    #else
+     const OUTPUT_TYPE res = TO_OUTPUT_TYPE(select(INPUT_2, INPUT_1, MASK));
+    #endif
 
     output[output_offset] = res;
 }

src/plugins/intel_gpu/src/kernel_selector/kernels/mvn/mvn_kernel_bfyx_opt.cpp

@@ -93,14 +93,12 @@ JitConstants MVNKernelBfyxOpt::GetJitConstants(const mvn_params& params, MVNKern
         const std::string lws_0 = "get_local_size(0)";
         jit.AddConstants({
             MakeJitConstant("LWS", lws_0),
-            MakeJitConstant("SLM_SIZE", dispatchData.maxSlmSize),
-            MakeJitConstant("DATA_SETS_COUNT", data_set_count),
             MakeJitConstant("DATA_SET_SIZE", data_set_size),
+            MakeJitConstant("DATA_SETS_COUNT", data_set_count),
         });
     } else {
         jit.AddConstants({
             MakeJitConstant("LWS", dispatchData.lws[0]),
-            MakeJitConstant("SLM_SIZE", dispatchData.lws[0]),
             MakeJitConstant("DATA_SETS_COUNT", dispatchData.dataSetsCount),
             MakeJitConstant("DATA_SET_SIZE", dispatchData.dataSetSize),
         });

src/plugins/intel_gpu/tests/unit/fusions/select_fusion_test.cpp

@@ -63,9 +63,12 @@ class SelectFusingTest : public ::BaseFusingTest<select_test_params> {
 }  // namespace
 
 #define CASE_SELECT_FP32_TO_I8_0  {2, 16, 4, 4}, {2, 16, 4, 4},  data_types::f32, data_types::i8,  format::bfyx, format::bfyx
+#define CASE_SELECT_FP32_TO_U8_0  {2, 16, 4, 4}, {2, 16, 4, 4},  data_types::f32, data_types::u8,  format::bfyx, format::bfyx
 #define CASE_SELECT_FP32_TO_F16_0 {2, 16, 17, 4}, {2, 16, 1, 4}, data_types::f32, data_types::f16, format::bfyx, format::bfyx
 #define CASE_SELECT_FP16_TO_I8_0  {2, 16, 4, 4}, {2, 16, 4, 4},  data_types::f16, data_types::i8,  format::bfyx, format::bfyx
+#define CASE_SELECT_FP16_TO_U8_0  {2, 16, 4, 4}, {2, 16, 4, 4},  data_types::f16, data_types::u8,  format::bfyx, format::bfyx
 #define CASE_SELECT_FP16_TO_I8_1  {2, 16, 4, 4}, {2, 16, 4, 4},  data_types::f16, data_types::i8,  format::bfyx, format::bfzyx
+#define CASE_SELECT_FP16_TO_U8_1  {2, 16, 4, 4}, {2, 16, 4, 4},  data_types::f16, data_types::u8,  format::bfyx, format::bfzyx
 
 class select_reorder_fusion : public SelectFusingTest {};
 TEST_P(select_reorder_fusion, basic) {
@@ -85,8 +88,11 @@ TEST_P(select_reorder_fusion, basic) {
 INSTANTIATE_TEST_SUITE_P(fusings_gpu, select_reorder_fusion, ::testing::ValuesIn(std::vector<select_test_params>{
     select_test_params{ CASE_SELECT_FP32_TO_F16_0, 5, 6},
     select_test_params{ CASE_SELECT_FP32_TO_I8_0, 5, 6},
+    select_test_params{ CASE_SELECT_FP32_TO_U8_0, 5, 6},
     select_test_params{ CASE_SELECT_FP16_TO_I8_0, 5, 6},
+    select_test_params{ CASE_SELECT_FP16_TO_U8_0, 5, 6},
     select_test_params{ CASE_SELECT_FP16_TO_I8_1, 6, 6}, // reorder should not be fused
+    select_test_params{ CASE_SELECT_FP16_TO_U8_1, 6, 6},
 }));
 
 class select_reorder_fusion_dynamic : public SelectFusingTest {};

src/plugins/intel_npu/src/al/include/intel_npu/al/config/npuw.hpp

@@ -51,6 +51,7 @@ DEFINE_OPT(NPUW_DCOFF_SCALE, bool, false, npuw::partitioning::dcoff_with_scale,
 DEFINE_OPT(NPUW_FUNCALL_FOR_ALL, bool, false, npuw::partitioning::funcall_for_all, CompileTime);
 DEFINE_OPT(NPUW_PARALLEL_COMPILE, bool, false, npuw::parallel_compilation, CompileTime);
 DEFINE_OPT(NPUW_WEIGHTS_BANK, std::string, "", npuw::weights_bank, CompileTime);
+DEFINE_OPT(NPUW_WEIGHTS_BANK_ALLOC, std::string, "", npuw::weights_bank_alloc, CompileTime);
 DEFINE_OPT(NPUW_CACHE_DIR, std::string, "", npuw::cache_dir, CompileTime);
 DEFINE_OPT(NPUW_FUNCALL_ASYNC, bool, false, npuw::funcall_async, RunTime);
 DEFINE_OPT(NPUW_ACC_CHECK, bool, false, npuw::accuracy::check, RunTime);

src/plugins/intel_npu/src/al/include/npuw_private_properties.hpp

@@ -45,6 +45,14 @@ static constexpr ov::Property<std::string> submodel_device{"NPUW_SUBMODEL_DEVICE
  */
 static constexpr ov::Property<std::string> weights_bank{"NPUW_WEIGHTS_BANK"};
 
+/**
+ * @brief
+ * Type: std::string.
+ * Specify device name for weights bank which is used to allocate memory.
+ * Default value: "".
+ */
+static constexpr ov::Property<std::string> weights_bank_alloc{"NPUW_WEIGHTS_BANK_ALLOC"};
+
 /**
  * @brief
  * Type: std::string.

src/plugins/intel_npu/src/al/src/config/npuw.cpp

@@ -37,6 +37,7 @@ void intel_npu::registerNPUWOptions(OptionsDesc& desc) {
     desc.add<NPUW_PARALLEL_COMPILE>();
     desc.add<NPUW_FUNCALL_ASYNC>();
     desc.add<NPUW_WEIGHTS_BANK>();
+    desc.add<NPUW_WEIGHTS_BANK_ALLOC>();
     desc.add<NPUW_CACHE_DIR>();
     desc.add<NPUW_ACC_CHECK>();
     desc.add<NPUW_ACC_THRESH>();

src/plugins/intel_npu/src/plugin/npuw/base_sync_infer_request.cpp

@@ -375,6 +375,19 @@ bool ov::npuw::IBaseInferRequest::needs_copy(std::size_t idx) const {
     return true;
 }
 
+bool ov::npuw::IBaseInferRequest::needs_copy(std::size_t idx, std::size_t cidx) const {
+    if (!needs_copy(idx)) {
+        return false;
+    }
+    auto& comp_model_desc = m_npuw_model->m_compiled_submodels[idx];
+    if (comp_model_desc.is_remote[cidx]) {
+        // FIXME: Test if the tensor device and the request device are
+        // the same or compatible!
+        return false;
+    }
+    return true;
+}
+
 std::size_t ov::npuw::IBaseInferRequest::next(std::size_t idx_base) const {
     // Answer the next valid subrequest which is possible to prepare
     // FIXME: this could be a predefined map, not a lookup

src/plugins/intel_npu/src/plugin/npuw/base_sync_infer_request.hpp

@@ -127,6 +127,7 @@ class IBaseInferRequest : public ov::ISyncInferRequest {
     std::size_t m_run_iter = 0u;
 
     bool needs_copy(std::size_t idx) const;
+    bool needs_copy(std::size_t idx, std::size_t cidx) const;
     std::size_t next(std::size_t idx_base) const;
     std::size_t real(std::size_t idx) const;
 

src/plugins/intel_npu/src/plugin/npuw/compiled_model.cpp

@@ -115,7 +115,8 @@ ov::npuw::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model,
 
     // Initialize weights bank
     const std::string weights_bank_opt = m_cfg.get<::intel_npu::NPUW_WEIGHTS_BANK>();
-    m_weights_bank = ov::npuw::weights::bank(weights_bank_opt, plugin->get_core());
+    const std::string wbank_alloc = m_cfg.get<::intel_npu::NPUW_WEIGHTS_BANK_ALLOC>();
+    m_weights_bank = ov::npuw::weights::bank(weights_bank_opt, plugin->get_core(), wbank_alloc);
 
     LOG_VERB("*** Original model ***");
     const auto& orig_parameters = model->get_parameters();
@@ -235,6 +236,8 @@ ov::npuw::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model,
     }  // for(ordered_subgraphs)
     // NOTE(dm): there's a better way to do it, like we do in G-API backends.
 
+    m_update_required = m_cfg.get<::intel_npu::NPUW_FOLD>();
+
     // Store mapping between manually splitted inputs/outputs
     // to connect tensors between compiled submodels
     m_submodels_input_to_prev_output = partitioning.input_to_prev_output;
@@ -302,10 +305,11 @@ ov::npuw::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model,
             m_compiled_submodels[id].host_gather = subgraph._host_gather;
             m_compiled_submodels[id].param_base = fcn_template._param_offset;
             m_compiled_submodels[id].closure = subgraph._closure;
+            m_compiled_submodels[id].lazy_closure = subgraph._lazy_closure;
             m_compiled_submodels[id].scales = subgraph._scales;
             m_compiled_submodels[id].zerops = subgraph._zerops;
-            m_compiled_submodels[id].update_required.resize(subgraph._closure.size(), false);
-            fill_weights_bank(id);
+            m_compiled_submodels[id].forced_to_fcall = subgraph._forced_to_fcall;
+            m_compiled_submodels[id].is_remote.resize(subgraph._lazy_closure.size(), false);
         }  // if(!funcall)
 
         if (!m_compiled_submodels[id].model && !m_compiled_submodels[id].replaced_by) {
@@ -421,6 +425,9 @@ ov::npuw::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model,
         }
     }
 
+    // Finalize memory in closures and weight banks
+    finalize_weights_bank();
+
     // Print stats report when possible
     {
         LOG_INFO("Initial device distribution:");
@@ -434,24 +441,54 @@ ov::npuw::CompiledModel::CompiledModel(const std::shared_ptr<ov::Model>& model,
     reset_io();
 }
 
-void ov::npuw::CompiledModel::fill_weights_bank(const std::size_t idx) {
-    LOG_VERB("Filling weights bank for Subgraph[" << idx << "]...");
-    LOG_BLOCK();
+void ov::npuw::CompiledModel::finalize_weights_bank() {
+    // Register lazy tensors
+    for (std::size_t idx = 0; idx < m_compiled_submodels.size(); ++idx) {
+        auto& comp_model_desc = m_compiled_submodels[idx];
 
-    NPUW_ASSERT(m_compiled_submodels[idx].replaced_by);
+        // Skip optimized out and non-functions
+        if (!comp_model_desc.compiled_model && !comp_model_desc.replaced_by) {
+            return;
+        }
 
-    auto& comp_model_desc = m_compiled_submodels[idx];
+        const auto real_idx = comp_model_desc.replaced_by.value_or(idx);
+        auto& func_desc = m_compiled_submodels[real_idx];
 
-    for (std::size_t cidx = 0u; cidx < comp_model_desc.closure.size(); cidx++) {
-        comp_model_desc.closure[cidx] = m_weights_bank->update(comp_model_desc.closure[cidx]);
-        if (m_cfg.get<::intel_npu::NPUW_FOLD>()) {
-            comp_model_desc.update_required[cidx] = true;
-        } else {
-            comp_model_desc.update_required[cidx] = false;
+        for (std::size_t tidx = 0; tidx < comp_model_desc.lazy_closure.size(); ++tidx) {
+            if (comp_model_desc.closure[tidx]) {
+                continue;  // host-side closure
+            }
+            m_weights_bank->registerLT(comp_model_desc.lazy_closure[tidx], *func_desc.device_it);
         }
     }
 
-    LOG_VERB("DONE");
+    // Evaluate and allocate all LazyTensors inside the bank
+    m_weights_bank->evaluate_and_allocate();
+
+    // Set evaluated and allocated ov::Tensors to closures
+    for (size_t idx = 0; idx < m_compiled_submodels.size(); ++idx) {
+        auto& comp_model_desc = m_compiled_submodels[idx];
+
+        // Skip optimized out and non-functions
+        if (!comp_model_desc.compiled_model && !comp_model_desc.replaced_by) {
+            continue;
+        }
+
+        const auto real_idx = comp_model_desc.replaced_by.value_or(idx);
+        auto& func_desc = m_compiled_submodels[real_idx];
+
+        for (std::size_t tidx = 0; tidx < comp_model_desc.lazy_closure.size(); ++tidx) {
+            if (comp_model_desc.closure[tidx]) {
+                // host-side closure - already set, do nothing
+                comp_model_desc.is_remote[tidx] = false;
+                continue;
+            }
+            const auto& lt = comp_model_desc.lazy_closure[tidx];
+            comp_model_desc.closure[tidx] = m_weights_bank->get(lt, *func_desc.device_it);
+            // FIXME: find a more reliable way to do so
+            comp_model_desc.is_remote[tidx] = m_weights_bank->is_remote(lt);
+        }
+    }
 }
 
 void ov::npuw::CompiledModel::remove_long_output_names(const std::shared_ptr<ov::Model>& model) {
@@ -748,7 +785,6 @@ void ov::npuw::CompiledModel::implement_properties() {
 
     // 1.
     // OV Public
-    // ===============================================
     m_prop_to_opt = {{ov::supported_properties.name(),
                       {ov::PropertyMutability::RO,
                        [&](const ::intel_npu::Config&) -> std::vector<PropertyName>& {
@@ -785,7 +821,6 @@ void ov::npuw::CompiledModel::implement_properties() {
                            return m_loaded_from_cache;
                        }}},
                      // OV Public Hints
-                     // =====================================================
                      {ov::hint::performance_mode.name(),
                       {ov::PropertyMutability::RO,
                        [&](const ::intel_npu::Config&) {
@@ -856,6 +891,7 @@ void ov::npuw::CompiledModel::implement_properties() {
                           BIND(npuw::parallel_compilation, NPUW_PARALLEL_COMPILE),
                           BIND(npuw::funcall_async, NPUW_FUNCALL_ASYNC),
                           BIND(npuw::weights_bank, NPUW_WEIGHTS_BANK),
+                          BIND(npuw::weights_bank_alloc, NPUW_WEIGHTS_BANK_ALLOC),
                           BIND(npuw::cache_dir, NPUW_CACHE_DIR),
                           BIND(npuw::accuracy::check, NPUW_ACC_CHECK),
                           BIND(npuw::accuracy::threshold, NPUW_ACC_THRESH),

src/plugins/intel_npu/src/plugin/npuw/compiled_model.hpp

@@ -72,7 +72,7 @@ class CompiledModel : public ov::ICompiledModel {
 
     void implement_properties();
 
-    void fill_weights_bank(const std::size_t idx);
+    void finalize_weights_bank();
 
     std::shared_ptr<::intel_npu::OptionsDesc> m_options_desc;
     ::intel_npu::Config m_cfg;
@@ -135,10 +135,16 @@ class CompiledModel : public ov::ICompiledModel {
         // FIXME: This is a 1:1 copy of the ov::npuw::Subgraph structure
         // w.r.t. function calls
         std::size_t param_base = 0;
+        // NB: closure and lazy_closure are of the same size - to preserve proper indexing.
+        //     closure is responsible for host-side tensors (DCOFF, Gather, etc) while
+        //     lazy_closure is used for weights sharing and allocating device memory.
         std::vector<ov::Tensor> closure;
+        std::vector<weights::LazyTensor> lazy_closure;
         std::vector<ov::Tensor> scales;
         std::vector<ov::Tensor> zerops;
-        std::vector<bool> update_required;
+        std::vector<bool> is_remote;
+
+        bool forced_to_fcall = false;
 
         // FIXME: Take it out of structure
         ov::SoPtr<ov::ICompiledModel> ref_compiled_model;
@@ -149,6 +155,8 @@ class CompiledModel : public ov::ICompiledModel {
     };
     std::vector<CompiledModelDesc> m_compiled_submodels;
 
+    bool m_update_required;
+
     std::function<bool(const ov::SoPtr<ov::ITensor>&, const ov::SoPtr<ov::ITensor>&)> m_acc_check;
     std::string m_ref_device;