Adding the BENCHMARK_INIT config option in order to allow running the bapp on the init part of the compiled model

Author: razvanapetroaie

samples/cpp/benchmark_app/utils.cpp

@@ -287,11 +287,12 @@ size_t get_batch_size(const benchmark_app::InputsInfo& inputs_info) {
     size_t batch_size = 0;
     for (auto& info : inputs_info) {
         if (ov::layout::has_batch(info.second.layout)) {
-            if (batch_size == 0)
+            if (batch_size == 0) {
                 batch_size = info.second.batch();
-            else if (batch_size != info.second.batch())
-                throw std::logic_error("Can't deterimine batch size: batch is "
-                                       "different for different inputs!");
+            } else if (batch_size != info.second.batch()) {
+                batch_size = 0;
+                break;
+            }
         }
     }
     if (batch_size == 0) {

src/plugins/intel_npu/src/al/include/intel_npu/config/common.hpp

@@ -275,6 +275,16 @@ struct SEPARATE_WEIGHTS final : OptionBase<SEPARATE_WEIGHTS, bool> {
     }
 };
 
+struct BENCHMARK_INIT final : OptionBase<BENCHMARK_INIT, bool> {
+    static std::string_view key() {
+        return ov::intel_npu::benchmark_init.name();
+    }
+
+    static bool defaultValue() {
+        return false;
+    }
+};
+
 }  // namespace intel_npu
 
 namespace ov {

src/plugins/intel_npu/src/al/include/intel_npu/npu_private_properties.hpp

@@ -349,6 +349,8 @@ static constexpr ov::Property<BatchMode> batch_mode{"NPU_BATCH_MODE"};
  */
 static constexpr ov::Property<bool> separate_weights{"NPU_SEPARATE_WEIGHTS"};
 
+static constexpr ov::Property<bool> benchmark_init{"NPU_BENCHMARK_INIT"};
+
 /**
  * @brief [Only for NPU Plugin]
  * Type: integer, default is 1

src/plugins/intel_npu/src/al/src/config/common.cpp

@@ -23,6 +23,7 @@ void intel_npu::registerCommonOptions(OptionsDesc& desc) {
     desc.add<LOADED_FROM_CACHE>();
     desc.add<BATCH_MODE>();
     desc.add<SEPARATE_WEIGHTS>();
+    desc.add<BENCHMARK_INIT>();
 }
 
 //

src/plugins/intel_npu/src/backend/src/zero_infer_request.cpp

@@ -586,8 +586,10 @@ void ZeroInferRequest::infer_async() {
     for (const auto& userTensor : _userInputTensors) {
         const IODescriptor inputDescriptor = _metadata.inputs.at(inputIndex);
 
-        OPENVINO_ASSERT(!inputDescriptor.isInitInputWeights,
-                        "This path should not be used for running inferences for the \"init\" model");
+        if (!_config.get<BENCHMARK_INIT>()) {
+            OPENVINO_ASSERT(!inputDescriptor.isInitInputWeights,
+                            "This path should not be used for running inferences for the \"init\" model");
+        }
 
         if (inputDescriptor.isShapeTensor) {
             OPENVINO_ASSERT(inputDescriptor.relatedDescriptorIndex.has_value(),

src/plugins/intel_npu/src/compiler_adapter/src/driver_compiler_adapter.cpp

@@ -649,6 +649,11 @@ std::string DriverCompilerAdapter::serializeConfig(const Config& config,
                           << VALUE_DELIMITER;
     content = std::regex_replace(content, std::regex(separateWeightsStream.str()), "");
 
+    std::ostringstream benchmarkInitStream;
+    benchmarkInitStream << ov::intel_npu::benchmark_init.name() << KEY_VALUE_SEPARATOR << VALUE_DELIMITER << "\\S+"
+                        << VALUE_DELIMITER;
+    content = std::regex_replace(content, std::regex(benchmarkInitStream.str()), "");
+
     // FINAL step to convert prefixes of remaining params, to ensure backwards compatibility
     // From 5.0.0, driver compiler start to use NPU_ prefix, the old version uses VPU_ prefix
     if (compilerVersion.major < 5) {

src/plugins/intel_npu/src/plugin/src/compiled_model.cpp

@@ -66,7 +66,7 @@ std::shared_ptr<ov::IAsyncInferRequest> CompiledModel::create_infer_request() co
         _device->createInferRequest(shared_from_this(), _config);
     syncInferRequest->initialize_states();
 
-    if (_config.get<SEPARATE_WEIGHTS>()) {
+    if (_config.get<SEPARATE_WEIGHTS>() && _initGraph != nullptr) {
         if (!_config.get<CREATE_EXECUTOR>() || _config.get<DEFER_WEIGHTS_LOAD>()) {
             begin = std::chrono::steady_clock::now();
             _initGraph->initialize(_config);
@@ -88,6 +88,9 @@ std::shared_ptr<ov::IAsyncInferRequest> CompiledModel::create_infer_request() co
         end = std::chrono::steady_clock::now();
         std::cout << "set_weights_inputs() call "
                   << std::chrono::duration_cast<std::chrono::milliseconds>(end - begin).count() << "[ms]" << std::endl;
+    } else if (_config.get<SEPARATE_WEIGHTS>() && _initGraph == nullptr) {
+        _logger.warning("SEPARATE_WEIGHTS config option was set but no compiled model for the init schedule was found. "
+                        "run_init() will not run.");
     }
 
     return std::make_shared<AsyncInferRequest>(syncInferRequest,

src/plugins/intel_npu/src/plugin/src/plugin.cpp

@@ -48,49 +48,85 @@ const char* NPU_PLUGIN_LIB_NAME = "openvino_intel_npu_plugin";
  * @returns The dummy "ov::Model" composed of "parameter" and "result" nodes built using the given descriptors.
  */
 std::shared_ptr<ov::Model> create_dummy_model(const std::vector<IODescriptor>& inputDescriptors,
-                                              const std::vector<IODescriptor>& outputDescriptors) {
+                                              const std::vector<IODescriptor>& outputDescriptors,
+                                              const bool benchmarkInit = false) {
     ov::ParameterVector parameters;
     ov::NodeVector results;
 
     for (const IODescriptor& inputDescriptor : inputDescriptors) {
-        if (inputDescriptor.isStateInput || inputDescriptor.isStateOutput || inputDescriptor.isShapeTensor ||
-            inputDescriptor.isInitInputWeights || inputDescriptor.isMainInputWeights) {
-            continue;
-        }
+        if (!benchmarkInit) {
+            if (inputDescriptor.isStateInput || inputDescriptor.isStateOutput || inputDescriptor.isShapeTensor ||
+                inputDescriptor.isInitInputWeights || inputDescriptor.isMainInputWeights) {
+                continue;
+            }
 
-        std::shared_ptr<ov::op::v0::Parameter> parameter = std::make_shared<ov::op::v0::Parameter>(
-            inputDescriptor.precision,
-            inputDescriptor.shapeFromIRModel.has_value() ? *inputDescriptor.shapeFromIRModel
-                                                         : inputDescriptor.shapeFromCompiler);
+            std::shared_ptr<ov::op::v0::Parameter> parameter = std::make_shared<ov::op::v0::Parameter>(
+                inputDescriptor.precision,
+                inputDescriptor.shapeFromIRModel.has_value() ? *inputDescriptor.shapeFromIRModel
+                                                             : inputDescriptor.shapeFromCompiler);
+            parameter->set_friendly_name(inputDescriptor.nodeFriendlyName);
+            parameter->output(0).get_tensor().set_names(inputDescriptor.outputTensorNames);
+            parameters.push_back(parameter);
+        } else {
+            if (inputDescriptor.isStateInput || inputDescriptor.isStateOutput || inputDescriptor.isShapeTensor ||
+                inputDescriptor.isMainInputWeights) {
+                continue;
+            }
 
-        parameter->set_friendly_name(inputDescriptor.nodeFriendlyName);
-        parameter->output(0).get_tensor().set_names(inputDescriptor.outputTensorNames);
-        parameters.push_back(parameter);
+            std::shared_ptr<ov::op::v0::Parameter> parameter = std::make_shared<ov::op::v0::Parameter>(
+                inputDescriptor.precision,
+                inputDescriptor.shapeFromIRModel.has_value() ? *inputDescriptor.shapeFromIRModel
+                                                             : inputDescriptor.shapeFromCompiler);
+            parameter->set_friendly_name(inputDescriptor.nameFromCompiler);
+            parameter->output(0).get_tensor().set_names(
+                std::unordered_set<std::string>{inputDescriptor.nameFromCompiler});
+            parameters.push_back(parameter);
+        }
     }
 
     // The "result" nodes require a parent node in order to satisfy the API conventions. Additionally, a dummy shape for
     // the "Constant" node was required since the specific constructor does not accept "ov::PartialShape" values (a
     // constant can't have dynamic shape). The dummy tensor was also brought in order to register the correct,
     // potentially dynamic, output shape.
     for (const IODescriptor& outputDescriptor : outputDescriptors) {
-        if (outputDescriptor.isStateInput || outputDescriptor.isStateOutput || outputDescriptor.isShapeTensor ||
-            outputDescriptor.isInitOutputWeights) {
-            continue;
-        }
+        if (!benchmarkInit) {
+            if (outputDescriptor.isStateInput || outputDescriptor.isStateOutput || outputDescriptor.isShapeTensor ||
+                outputDescriptor.isInitOutputWeights) {
+                continue;
+            }
+
+            std::shared_ptr<ov::Node> constantDummy =
+                std::make_shared<ov::op::v0::Constant>(outputDescriptor.precision, CONSTANT_NODE_DUMMY_SHAPE);
 
-        std::shared_ptr<ov::Node> constantDummy =
-            std::make_shared<ov::op::v0::Constant>(outputDescriptor.precision, CONSTANT_NODE_DUMMY_SHAPE);
+            const std::shared_ptr<ov::descriptor::Tensor>& tensorDummy =
+                std::make_shared<ov::descriptor::Tensor>(outputDescriptor.precision,
+                                                         outputDescriptor.shapeFromCompiler,
+                                                         outputDescriptor.outputTensorNames);
 
-        const std::shared_ptr<ov::descriptor::Tensor>& tensorDummy = std::make_shared<ov::descriptor::Tensor>(
-            outputDescriptor.precision,
-            outputDescriptor.shapeFromIRModel.has_value() ? *outputDescriptor.shapeFromIRModel
-                                                          : outputDescriptor.shapeFromCompiler,
-            outputDescriptor.outputTensorNames);
+            std::shared_ptr<ov::Node> result = std::make_shared<ov::op::v0::Result>(constantDummy);
+            result->output(0).set_tensor_ptr(tensorDummy);
+
+            result->set_friendly_name(outputDescriptor.nodeFriendlyName);
+            results.push_back(result);
+        } else {
+            if (outputDescriptor.isStateInput || outputDescriptor.isStateOutput || outputDescriptor.isShapeTensor) {
+                continue;
+            }
 
-        std::shared_ptr<ov::Node> result = std::make_shared<ov::op::v0::Result>(constantDummy);
-        result->output(0).set_tensor_ptr(tensorDummy);
-        result->set_friendly_name(outputDescriptor.nodeFriendlyName);
-        results.push_back(result);
+            std::shared_ptr<ov::Node> constantDummy =
+                std::make_shared<ov::op::v0::Constant>(outputDescriptor.precision, CONSTANT_NODE_DUMMY_SHAPE);
+
+            const std::shared_ptr<ov::descriptor::Tensor>& tensorDummy = std::make_shared<ov::descriptor::Tensor>(
+                outputDescriptor.precision,
+                outputDescriptor.shapeFromCompiler,
+                std::unordered_set<std::string>{outputDescriptor.nameFromCompiler});
+
+            std::shared_ptr<ov::Node> result = std::make_shared<ov::op::v0::Result>(constantDummy);
+            result->output(0).set_tensor_ptr(tensorDummy);
+
+            result->set_friendly_name(outputDescriptor.nameFromCompiler);
+            results.push_back(result);
+        }
     }
 
     return std::make_shared<ov::Model>(results, parameters);
@@ -589,6 +625,12 @@ Plugin::Plugin()
           [](const Config& config) {
               return config.getString<SEPARATE_WEIGHTS>();
           }}},
+        {ov::intel_npu::benchmark_init.name(),
+         {false,
+          ov::PropertyMutability::RW,
+          [](const Config& config) {
+              return config.getString<BENCHMARK_INIT>();
+          }}},
     };
 
     for (auto& property : _properties) {
@@ -860,16 +902,22 @@ std::shared_ptr<ov::ICompiledModel> Plugin::import_model(std::istream& stream, c
             auto graph = compiler->parse(std::move(blob), localConfig);
             graph->update_network_name("net" + std::to_string(_compiledModelLoadCounter++));
 
-            const std::shared_ptr<ov::Model> modelDummy =
-                create_dummy_model(graph->get_metadata().inputs, graph->get_metadata().outputs);
-
-            compiledModel = std::make_shared<CompiledModel>(modelDummy,
-                                                            shared_from_this(),
-                                                            device,
-                                                            graph,
-                                                            localConfig,
-                                                            initGraph,
-                                                            initModel);
+            if (!localConfig.get<BENCHMARK_INIT>()) {
+                const std::shared_ptr<ov::Model> modelDummy =
+                    create_dummy_model(graph->get_metadata().inputs, graph->get_metadata().outputs);
+                compiledModel = std::make_shared<CompiledModel>(modelDummy,
+                                                                shared_from_this(),
+                                                                device,
+                                                                graph,
+                                                                localConfig,
+                                                                initGraph,
+                                                                initModel);
+            } else {
+                const std::shared_ptr<ov::Model> modelDummy =
+                    create_dummy_model(initGraph->get_metadata().inputs, initGraph->get_metadata().outputs, true);
+                compiledModel =
+                    std::make_shared<CompiledModel>(modelDummy, shared_from_this(), device, initGraph, localConfig);
+            }
         }
     } catch (const std::exception& ex) {
         OPENVINO_THROW("Can't import network: ", ex.what());