From 69180e2fc9ee68ae6705b68c934d75f2a6182398 Mon Sep 17 00:00:00 2001 From: Mircea-Aurelian Dan Date: Fri, 6 Sep 2024 14:05:38 +0300 Subject: [PATCH] [intel-npu] Add functional tests to check correct memory deallocation (#25980) ### Details: - *The following new test cases are being added:* ``` 1) allocate / de-allocate a model for N times 2) allocate a model create infer request / destroy infer request for N times de-allocate and destroy the model 3) allocate a model create infer request set_tensort(with new buffer every time) for N times destroy infer request de-allocate and destroy the model 4) allocate a model create infer request local_tensor = infer_request.get_output_tensor() destroy infer request // output tensor should not be deallocated here de-allocate and destroy the model destroy local_tensor // output tensor should be deallocated here ``` ### Tickets: - *E129376* --- .../custom_stream.cpp | 17 +- .../behavior/ov_plugin/core_integration.cpp | 65 ++++ .../tests/functional/common/functions.cpp | 17 ++ .../tests/functional/common/functions.h | 2 + .../overload/ov_plugin/core_integration.hpp | 235 +++++++++++++++ .../behavior/compiled_model/properties.cpp | 62 ++++ .../behavior/ov_plugin/core_integration.cpp | 278 +----------------- .../behavior/ov_plugin/properties_tests.cpp | 21 +- 8 files changed, 403 insertions(+), 294 deletions(-) create mode 100644 src/plugins/intel_npu/tests/functional/behavior/ov_plugin/core_integration.cpp diff --git a/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/custom_stream.cpp b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/custom_stream.cpp index 41ee0988904913..5adc454e81f679 100644 --- a/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/custom_stream.cpp +++ b/src/plugins/intel_npu/tests/functional/behavior/npu_driver_compiler_adapter/custom_stream.cpp @@ -5,6 +5,7 @@ #include #include "base/ov_behavior_test_utils.hpp" +#include "common/functions.h" #include "common/npu_test_env_cfg.hpp" #include "common_test_utils/node_builders/constant.hpp" #include "graph_transformations.hpp" @@ -22,22 +23,6 @@ namespace ov::test::behavior { class DriverCompilerAdapterCustomStreamTestNPU : public ov::test::behavior::OVPluginTestBase, public testing::WithParamInterface { public: - std::shared_ptr createModelWithLargeSize() { - auto data = std::make_shared(ov::element::f16, ov::Shape{4000, 4000}); - auto mul_constant = ov::opset11::Constant::create(ov::element::f16, ov::Shape{1}, {1.5}); - auto mul = std::make_shared(data, mul_constant); - auto add_constant = ov::opset11::Constant::create(ov::element::f16, ov::Shape{1}, {0.5}); - auto add = std::make_shared(mul, add_constant); - // Just a sample model here, large iteration to make the model large - for (int i = 0; i < 1000; i++) { - add = std::make_shared(add, add_constant); - } - auto res = std::make_shared(add); - - /// Create the OpenVINO model - return std::make_shared(ov::ResultVector{std::move(res)}, ov::ParameterVector{std::move(data)}); - } - std::string generateRandomFileName() { std::stringstream ss; auto now = std::chrono::high_resolution_clock::now(); diff --git a/src/plugins/intel_npu/tests/functional/behavior/ov_plugin/core_integration.cpp b/src/plugins/intel_npu/tests/functional/behavior/ov_plugin/core_integration.cpp new file mode 100644 index 00000000000000..daed99b28f09a8 --- /dev/null +++ b/src/plugins/intel_npu/tests/functional/behavior/ov_plugin/core_integration.cpp @@ -0,0 +1,65 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "overload/ov_plugin/core_integration.hpp" + +using namespace ov::test::behavior; + +namespace { + +const char* NPU_PLUGIN_LIB_NAME = "openvino_intel_npu_plugin"; + +std::vector devices = { + std::string(ov::test::utils::DEVICE_NPU), +}; + +std::pair plugins[] = { + std::make_pair(std::string(NPU_PLUGIN_LIB_NAME), std::string(ov::test::utils::DEVICE_NPU)), +}; + +namespace OVClassBasicTestName { +static std::string getTestCaseName(testing::TestParamInfo> obj) { + std::ostringstream result; + result << "OVClassBasicTestName_" << obj.param.first << "_" << obj.param.second; + result << "_targetDevice=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU); + + return result.str(); +} +} // namespace OVClassBasicTestName + +namespace OVClassNetworkTestName { +static std::string getTestCaseName(testing::TestParamInfo obj) { + std::ostringstream result; + result << "OVClassNetworkTestName_" << obj.param; + result << "_targetDevice=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU); + + return result.str(); +} +} // namespace OVClassNetworkTestName + +const std::vector configs = {{}}; + +#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT +INSTANTIATE_TEST_SUITE_P(compatibility_smoke_BehaviorTests_OVClassBasicTestP, + OVClassBasicTestPNPU, + ::testing::ValuesIn(plugins), + OVClassBasicTestName::getTestCaseName); +#endif + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassNetworkTestP, + OVClassNetworkTestPNPU, + ::testing::Combine(::testing::ValuesIn(devices), ::testing::ValuesIn(configs)), + OVClassNetworkTestPNPU::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassLoadNetworkTest, + OVClassLoadNetworkTestNPU, + ::testing::Combine(::testing::ValuesIn(devices), ::testing::ValuesIn(configs)), + OVClassLoadNetworkTestNPU::getTestCaseName); + +INSTANTIATE_TEST_SUITE_P(nightly_BehaviorTests_OVClassGetMetricTest, + OVClassGetMetricAndPrintNoThrow, + ::testing::Values(ov::test::utils::DEVICE_NPU), + OVClassNetworkTestName::getTestCaseName); + +} // namespace diff --git a/src/plugins/intel_npu/tests/functional/common/functions.cpp b/src/plugins/intel_npu/tests/functional/common/functions.cpp index 0b22a8a89beddf..1cb646d7905723 100644 --- a/src/plugins/intel_npu/tests/functional/common/functions.cpp +++ b/src/plugins/intel_npu/tests/functional/common/functions.cpp @@ -5,6 +5,7 @@ #include "functions.h" #include "common/npu_test_env_cfg.hpp" #include "openvino/op/softmax.hpp" +#include "openvino/opsets/opset11.hpp" #include "openvino/runtime/intel_npu/properties.hpp" std::shared_ptr buildSingleLayerSoftMaxNetwork() { @@ -26,6 +27,22 @@ std::shared_ptr buildSingleLayerSoftMaxNetwork() { return ov_model; } +std::shared_ptr createModelWithLargeSize() { + auto data = std::make_shared(ov::element::f16, ov::Shape{4000, 4000}); + auto mul_constant = ov::opset11::Constant::create(ov::element::f16, ov::Shape{1}, {1.5}); + auto mul = std::make_shared(data, mul_constant); + auto add_constant = ov::opset11::Constant::create(ov::element::f16, ov::Shape{1}, {0.5}); + auto add = std::make_shared(mul, add_constant); + // Just a sample model here, large iteration to make the model large + for (int i = 0; i < 1000; i++) { + add = std::make_shared(add, add_constant); + } + auto res = std::make_shared(add); + + /// Create the OpenVINO model + return std::make_shared(ov::ResultVector{std::move(res)}, ov::ParameterVector{std::move(data)}); +} + const std::string PlatformEnvironment::PLATFORM = []() -> std::string { const auto& var = ov::test::utils::NpuTestEnvConfig::getInstance().IE_NPU_TESTS_PLATFORM; if (!var.empty()) { diff --git a/src/plugins/intel_npu/tests/functional/common/functions.h b/src/plugins/intel_npu/tests/functional/common/functions.h index 5f4c91bdf6a5b5..96f881f6dbbd94 100644 --- a/src/plugins/intel_npu/tests/functional/common/functions.h +++ b/src/plugins/intel_npu/tests/functional/common/functions.h @@ -9,6 +9,8 @@ // create dummy network for tests std::shared_ptr buildSingleLayerSoftMaxNetwork(); +std::shared_ptr createModelWithLargeSize(); + // class encapsulated Platform getting from environmental variable class PlatformEnvironment { public: diff --git a/src/plugins/intel_npu/tests/functional/internal/overload/ov_plugin/core_integration.hpp b/src/plugins/intel_npu/tests/functional/internal/overload/ov_plugin/core_integration.hpp index c5e5e91ec19078..858cf6b95cd191 100644 --- a/src/plugins/intel_npu/tests/functional/internal/overload/ov_plugin/core_integration.hpp +++ b/src/plugins/intel_npu/tests/functional/internal/overload/ov_plugin/core_integration.hpp @@ -7,9 +7,12 @@ #include #include "base/ov_behavior_test_utils.hpp" #include "behavior/ov_plugin/properties_tests.hpp" +#include "common/functions.h" #include "common/utils.hpp" #include "common/npu_test_env_cfg.hpp" +#include "common_test_utils/data_utils.hpp" #include "common_test_utils/subgraph_builders/concat_with_params.hpp" +#include "common_test_utils/subgraph_builders/conv_pool_relu.hpp" #include "common_test_utils/subgraph_builders/kso_func.hpp" #include "common_test_utils/subgraph_builders/single_concat_with_constant.hpp" #include "common_test_utils/subgraph_builders/split_conv_concat.hpp" @@ -169,6 +172,238 @@ TEST(compatibility_OVClassBasicPropsTestNPU, smoke_SetConfigDevicePropertiesThro ov::Exception); } +// +// NPU specific metrics +// + +using OVClassGetMetricAndPrintNoThrow = OVClassBaseTestP; +TEST_P(OVClassGetMetricAndPrintNoThrow, DeviceAllocMemSizeLesserThanTotalMemSizeNPU) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::Core ie; + ov::Any p; + + OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_total_mem_size.name())); + uint64_t t = p.as(); + ASSERT_NE(t, 0); + + OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + uint64_t a = p.as(); + + ASSERT_LT(a, t); + + std::cout << "OV NPU device alloc/total memory size: " << a << "/" << t << std::endl; +} + +TEST_P(OVClassGetMetricAndPrintNoThrow, DeviceAllocMemSizeLesserAfterModelIsLoadedNPU) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::Core ie; + ov::Any p; + + OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + uint64_t a1 = p.as(); + + SKIP_IF_CURRENT_TEST_IS_DISABLED() { + auto model = ov::test::utils::make_conv_pool_relu(); + OV_ASSERT_NO_THROW(ie.compile_model(model, target_device, {})); + } + + OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + uint64_t a2 = p.as(); + + std::cout << "OV NPU device {alloc before load network/alloc after load network} memory size: {" << a1 << "/" << a2 + << "}" << std::endl; + + // after the network is loaded onto device, allocated memory value should increase + ASSERT_LE(a1, a2); +} + +TEST_P(OVClassGetMetricAndPrintNoThrow, VpuDeviceAllocMemSizeLesserAfterModelIsLoaded) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::Core ie; + ov::Any p; + + OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + uint64_t a1 = p.as(); + + SKIP_IF_CURRENT_TEST_IS_DISABLED() { + auto model = ov::test::utils::make_conv_pool_relu(); + OV_ASSERT_NO_THROW(ie.compile_model(model, target_device, ov::AnyMap{ov::log::level(ov::log::Level::DEBUG)})); + } + + OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + uint64_t a2 = p.as(); + + std::cout << "OV NPU device {alloc before load network/alloc after load network} memory size: {" << a1 << "/" << a2 + << "}" << std::endl; + + // after the network is loaded onto device, allocated memory value should increase + ASSERT_LE(a1, a2); +} + +TEST_P(OVClassGetMetricAndPrintNoThrow, VpuDeviceAllocMemSizeSameAfterDestroyCompiledModel) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::Core core; + ov::Any deviceAllocMemSizeAny; + + auto model = createModelWithLargeSize(); + + OV_ASSERT_NO_THROW(deviceAllocMemSizeAny = + core.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + uint64_t deviceAllocMemSize = deviceAllocMemSizeAny.as(); + uint64_t deviceAllocMemSizeFinal; + + for (size_t i = 0; i < 1000; ++i) { + ov::CompiledModel compiledModel; + OV_ASSERT_NO_THROW(compiledModel = core.compile_model(model, target_device)); + + compiledModel = {}; + + OV_ASSERT_NO_THROW(deviceAllocMemSizeAny = + core.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + deviceAllocMemSizeFinal = deviceAllocMemSizeAny.as(); + ASSERT_EQ(deviceAllocMemSize, deviceAllocMemSizeFinal) << " at iteration " << i; + } +} + +TEST_P(OVClassGetMetricAndPrintNoThrow, VpuDeviceAllocMemSizeSameAfterDestroyInferRequest) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::Core core; + ov::Any deviceAllocMemSizeAny; + + ov::CompiledModel compiledModel; + auto model = createModelWithLargeSize(); + + OV_ASSERT_NO_THROW(compiledModel = core.compile_model(model, target_device)); + + // After memory consumption updates, need to run first inference before measurements + auto inferRequest = compiledModel.create_infer_request(); + inferRequest.infer(); + inferRequest = {}; + + OV_ASSERT_NO_THROW(deviceAllocMemSizeAny = + core.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + uint64_t deviceAllocMemSize = deviceAllocMemSizeAny.as(); + uint64_t deviceAllocMemSizeFinal; + + for (size_t i = 0; i < 1000; ++i) { + inferRequest = compiledModel.create_infer_request(); + inferRequest.infer(); + + inferRequest = {}; + + OV_ASSERT_NO_THROW(deviceAllocMemSizeAny = + core.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + deviceAllocMemSizeFinal = deviceAllocMemSizeAny.as(); + ASSERT_EQ(deviceAllocMemSize, deviceAllocMemSizeFinal) << " at iteration " << i; + } +} + +TEST_P(OVClassGetMetricAndPrintNoThrow, VpuDeviceAllocMemSizeSameAfterDestroyInferRequestSetTensor) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::Core core; + ov::Any deviceAllocMemSizeAny; + + ov::CompiledModel compiledModel; + auto model = createModelWithLargeSize(); + + OV_ASSERT_NO_THROW(compiledModel = core.compile_model(model, target_device)); + + // After memory consumption updates, need to run first inference before measurements + auto inferRequest = compiledModel.create_infer_request(); + inferRequest.infer(); + inferRequest = {}; + + OV_ASSERT_NO_THROW(deviceAllocMemSizeAny = + core.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + uint64_t deviceAllocMemSize = deviceAllocMemSizeAny.as(); + uint64_t deviceAllocMemSizeFinal; + + for (size_t i = 0; i < 1000; ++i) { + auto inferRequest = compiledModel.create_infer_request(); + auto tensor = ov::Tensor(model->input(0).get_element_type(), model->input(0).get_shape()); + ov::test::utils::fill_data_random(static_cast(tensor.data()), tensor.get_size()); + inferRequest.set_input_tensor(tensor); + inferRequest.infer(); + + inferRequest = {}; + + OV_ASSERT_NO_THROW(deviceAllocMemSizeAny = + core.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + deviceAllocMemSizeFinal = deviceAllocMemSizeAny.as(); + ASSERT_EQ(deviceAllocMemSize, deviceAllocMemSizeFinal) << " at iteration " << i; + } +} + +TEST_P(OVClassGetMetricAndPrintNoThrow, VpuDeviceAllocMemSizeSameAfterDestroyInferRequestGetTensor) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::Core core; + ov::Any deviceAllocMemSizeAny; + + ov::CompiledModel compiledModel; + auto model = createModelWithLargeSize(); + + OV_ASSERT_NO_THROW(deviceAllocMemSizeAny = + core.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + uint64_t deviceAllocMemSize = deviceAllocMemSizeAny.as(); + uint64_t deviceAllocMemSizeFinal; + + for (size_t i = 0; i < 1000; ++i) { + OV_ASSERT_NO_THROW(compiledModel = core.compile_model(model, target_device)); + + auto inferRequest = compiledModel.create_infer_request(); + auto tensor = inferRequest.get_output_tensor(); + inferRequest.infer(); + + inferRequest = {}; + compiledModel = {}; + + OV_ASSERT_NO_THROW(deviceAllocMemSizeAny = + core.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + deviceAllocMemSizeFinal = deviceAllocMemSizeAny.as(); + ASSERT_LT(deviceAllocMemSize, deviceAllocMemSizeFinal) << " at iteration " << i; + + tensor = {}; + + OV_ASSERT_NO_THROW(deviceAllocMemSizeAny = + core.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); + deviceAllocMemSizeFinal = deviceAllocMemSizeAny.as(); + ASSERT_EQ(deviceAllocMemSize, deviceAllocMemSizeFinal) << " at iteration " << i; + } +} + +TEST_P(OVClassGetMetricAndPrintNoThrow, DriverVersionNPU) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::Core ie; + ov::Any p; + + OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::driver_version.name())); + uint32_t t = p.as(); + + std::cout << "NPU driver version is " << t << std::endl; + + OV_ASSERT_PROPERTY_SUPPORTED(ov::intel_npu::driver_version.name()); +} + +using OVClassCompileModel = OVClassBaseTestP; +TEST_P(OVClassCompileModel, CompileModelWithDifferentThreadNumbers) { + SKIP_IF_CURRENT_TEST_IS_DISABLED() + ov::Core ie; + ov::Any p; + + auto model = ov::test::utils::make_conv_pool_relu(); + OV_ASSERT_NO_THROW(ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(1)}})); + + OV_ASSERT_NO_THROW(ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(2)}})); + + OV_ASSERT_NO_THROW(ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(4)}})); + + EXPECT_ANY_THROW(ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(-1)}})); + OV_EXPECT_THROW( + std::ignore = ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(-1)}}), + ::ov::Exception, + testing::HasSubstr("ov::compilation_num_threads must be positive int32 value")); +} + #ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT TEST_P(OVClassBasicTestPNPU, smoke_registerPluginsLibrariesUnicodePath) { diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/properties.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/properties.cpp index b7b5d3c5831779..8abe86a77c55c2 100644 --- a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/properties.cpp +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/compiled_model/properties.cpp @@ -166,6 +166,68 @@ INSTANTIATE_TEST_SUITE_P(smoke_Hetero_BehaviorTests, OVClassCompileModelWithCorr ::testing::ValuesIn(heteroCompiledModelConfigs)), ov::test::utils::appendPlatformTypeTestName); +const std::vector configsWithSecondaryProperties = { + {ov::device::properties(ov::test::utils::DEVICE_NPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::properties(ov::test::utils::DEVICE_NPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties(ov::test::utils::DEVICE_NPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; + +const std::vector multiConfigsWithSecondaryProperties = { + {ov::device::priorities(ov::test::utils::DEVICE_CPU), + ov::device::properties(ov::test::utils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(ov::test::utils::DEVICE_CPU), + ov::device::properties(ov::test::utils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties(ov::test::utils::DEVICE_NPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; + +const std::vector autoConfigsWithSecondaryProperties = { + {ov::device::priorities(ov::test::utils::DEVICE_CPU), + ov::device::properties("AUTO", + ov::enable_profiling(false), + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(ov::test::utils::DEVICE_CPU), + ov::device::properties(ov::test::utils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(ov::test::utils::DEVICE_CPU), + ov::device::properties(ov::test::utils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties(ov::test::utils::DEVICE_NPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}, + {ov::device::priorities(ov::test::utils::DEVICE_CPU), + ov::device::properties("AUTO", + ov::enable_profiling(false), + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)), + ov::device::properties(ov::test::utils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, + {ov::device::priorities(ov::test::utils::DEVICE_CPU), + ov::device::properties("AUTO", + ov::enable_profiling(false), + ov::device::priorities(ov::test::utils::DEVICE_NPU), + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)), + ov::device::properties(ov::test::utils::DEVICE_CPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), + ov::device::properties(ov::test::utils::DEVICE_NPU, + ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; + +INSTANTIATE_TEST_SUITE_P(compatibility_smoke_BehaviorTests_OVClassLoadNetworkWithCorrectSecondaryPropertiesTest, + OVClassCompileModelWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU, "AUTO:NPU", "MULTI:NPU"), + ::testing::ValuesIn(configsWithSecondaryProperties))); + +INSTANTIATE_TEST_SUITE_P(compatibility_smoke_Multi_BehaviorTests_OVClassCompileModelWithCorrectPropertiesTest, + OVClassCompileModelWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values("MULTI"), + ::testing::ValuesIn(multiConfigsWithSecondaryProperties))); + +INSTANTIATE_TEST_SUITE_P(compatibility_smoke_AUTO_BehaviorTests_OVClassCompileModelWithCorrectPropertiesTest, + OVClassCompileModelWithCorrectPropertiesTest, + ::testing::Combine(::testing::Values("AUTO"), + ::testing::ValuesIn(autoConfigsWithSecondaryProperties))); + INSTANTIATE_TEST_SUITE_P(compatibility_smoke_BehaviorTests, OVClassCompiledModelPropertiesIncorrectTests, ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), ::testing::ValuesIn(compiledModelIncorrectConfigs)), diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp index 2803e0fa161e51..79af0f8b875c76 100644 --- a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/core_integration.cpp @@ -2,17 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 // -#include "overload/ov_plugin/core_integration.hpp" - -#include "behavior/compiled_model/properties.hpp" #include "behavior/ov_plugin/core_integration_sw.hpp" -#include "behavior/ov_plugin/properties_tests.hpp" -#include "common/functions.h" #include "common/npu_test_env_cfg.hpp" #include "common/utils.hpp" -#include "common_test_utils/subgraph_builders/conv_pool_relu.hpp" -#include "common_test_utils/test_assertions.hpp" -#include "functional_test_utils/ov_plugin_cache.hpp" +#include "common_test_utils/data_utils.hpp" #include "intel_npu/al/config/common.hpp" #include "openvino/runtime/intel_npu/properties.hpp" @@ -21,275 +14,6 @@ using namespace ov::test::behavior; namespace { -const char* NPU_PLUGIN_LIB_NAME = "openvino_intel_npu_plugin"; - -std::vector devices = { - std::string(ov::test::utils::DEVICE_NPU), -}; - -std::pair plugins[] = { - std::make_pair(std::string(NPU_PLUGIN_LIB_NAME), std::string(ov::test::utils::DEVICE_NPU)), -}; - -namespace OVClassBasicTestName { -static std::string getTestCaseName(testing::TestParamInfo> obj) { - std::ostringstream result; - result << "OVClassBasicTestName_" << obj.param.first << "_" << obj.param.second; - result << "_targetDevice=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU); - - return result.str(); -} -} // namespace OVClassBasicTestName - -namespace OVClassNetworkTestName { -static std::string getTestCaseName(testing::TestParamInfo obj) { - std::ostringstream result; - result << "OVClassNetworkTestName_" << obj.param; - result << "_targetDevice=" << ov::test::utils::getTestsPlatformFromEnvironmentOr(ov::test::utils::DEVICE_NPU); - - return result.str(); -} -} // namespace OVClassNetworkTestName - -// -// IE Class Common tests with -// - -const std::vector configs = {{}}; - -INSTANTIATE_TEST_SUITE_P(compatibility_smoke_BehaviorTests_OVBasicPropertiesTestsP, - OVBasicPropertiesTestsP, - ::testing::ValuesIn(plugins), - OVClassBasicTestName::getTestCaseName); - -#ifdef OPENVINO_ENABLE_UNICODE_PATH_SUPPORT -INSTANTIATE_TEST_SUITE_P(compatibility_smoke_BehaviorTests_OVClassBasicTestP, - OVClassBasicTestPNPU, - ::testing::ValuesIn(plugins), - OVClassBasicTestName::getTestCaseName); -#endif - -INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassNetworkTestP, - OVClassNetworkTestPNPU, - ::testing::Combine(::testing::ValuesIn(devices), ::testing::ValuesIn(configs)), - OVClassNetworkTestPNPU::getTestCaseName); - -// -// IE Class GetMetric -// - -INSTANTIATE_TEST_SUITE_P(compatibility_smoke_BehaviorTests_OVGetMetricPropsTest_nightly, - OVGetMetricPropsTest, - ::testing::ValuesIn(devices), - OVClassNetworkTestName::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P(compatibility_smoke_BehaviorTests_OVGetMetricPropsTest_nightly, - OVGetMetricPropsOptionalTest, - ::testing::ValuesIn(devices), - OVClassNetworkTestName::getTestCaseName); - -INSTANTIATE_TEST_SUITE_P( - smoke_BehaviorTests_OVCheckSetSupportedRWMandatoryMetricsPropsTests, - OVCheckSetSupportedRWMetricsPropsTests, - ::testing::Combine(::testing::Values("MULTI", "AUTO"), - ::testing::ValuesIn(OVCheckSetSupportedRWMetricsPropsTests::getRWOptionalPropertiesValues( - {ov::log::level.name()}))), - ov::test::utils::appendPlatformTypeTestName); - -const std::vector multiConfigs = {{ov::device::priorities(ov::test::utils::DEVICE_NPU)}}; -const std::vector configsDeviceProperties = { - {ov::device::properties(ov::test::utils::DEVICE_NPU, ov::num_streams(4))}}; -const std::vector configsWithSecondaryProperties = { - {ov::device::properties(ov::test::utils::DEVICE_NPU, - ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, - {ov::device::properties(ov::test::utils::DEVICE_NPU, - ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), - ov::device::properties(ov::test::utils::DEVICE_NPU, - ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; - -const std::vector multiConfigsWithSecondaryProperties = { - {ov::device::priorities(ov::test::utils::DEVICE_CPU), - ov::device::properties(ov::test::utils::DEVICE_CPU, - ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, - {ov::device::priorities(ov::test::utils::DEVICE_CPU), - ov::device::properties(ov::test::utils::DEVICE_CPU, - ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), - ov::device::properties(ov::test::utils::DEVICE_NPU, - ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; - -const std::vector autoConfigsWithSecondaryProperties = { - {ov::device::priorities(ov::test::utils::DEVICE_CPU), - ov::device::properties("AUTO", - ov::enable_profiling(false), - ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, - {ov::device::priorities(ov::test::utils::DEVICE_CPU), - ov::device::properties(ov::test::utils::DEVICE_CPU, - ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, - {ov::device::priorities(ov::test::utils::DEVICE_CPU), - ov::device::properties(ov::test::utils::DEVICE_CPU, - ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), - ov::device::properties(ov::test::utils::DEVICE_NPU, - ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}, - {ov::device::priorities(ov::test::utils::DEVICE_CPU), - ov::device::properties("AUTO", - ov::enable_profiling(false), - ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)), - ov::device::properties(ov::test::utils::DEVICE_CPU, - ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT))}, - {ov::device::priorities(ov::test::utils::DEVICE_CPU), - ov::device::properties("AUTO", - ov::enable_profiling(false), - ov::device::priorities(ov::test::utils::DEVICE_NPU), - ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)), - ov::device::properties(ov::test::utils::DEVICE_CPU, - ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT)), - ov::device::properties(ov::test::utils::DEVICE_NPU, - ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY))}}; - -// -// IE Class GetConfig -// - -INSTANTIATE_TEST_SUITE_P(BehaviorTests_OVGetConfigTest_nightly, - OVGetConfigTest, - ::testing::ValuesIn(devices), - OVClassNetworkTestName::getTestCaseName); - -// IE Class Load network - -INSTANTIATE_TEST_SUITE_P(compatibility_smoke_BehaviorTests_OVClassLoadNetworkWithCorrectSecondaryPropertiesTest, - OVClassCompileModelWithCorrectPropertiesTest, - ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU, "AUTO:NPU", "MULTI:NPU"), - ::testing::ValuesIn(configsWithSecondaryProperties))); - -INSTANTIATE_TEST_SUITE_P(compatibility_smoke_Multi_BehaviorTests_OVClassCompileModelWithCorrectPropertiesTest, - OVClassCompileModelWithCorrectPropertiesTest, - ::testing::Combine(::testing::Values("MULTI"), - ::testing::ValuesIn(multiConfigsWithSecondaryProperties))); - -INSTANTIATE_TEST_SUITE_P(compatibility_smoke_AUTO_BehaviorTests_OVClassCompileModelWithCorrectPropertiesTest, - OVClassCompileModelWithCorrectPropertiesTest, - ::testing::Combine(::testing::Values("AUTO"), - ::testing::ValuesIn(autoConfigsWithSecondaryProperties))); - -// IE Class load and check network with ov::device::properties -// OVClassCompileModelAndCheckSecondaryPropertiesTest only works with property num_streams of type int32_t -INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_BehaviorTests_OVClassLoadNetworkAndCheckWithSecondaryPropertiesTest, - OVClassCompileModelAndCheckSecondaryPropertiesTest, - ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU, "AUTO:NPU", "MULTI:NPU"), - ::testing::ValuesIn(configsDeviceProperties))); - -INSTANTIATE_TEST_SUITE_P(smoke_BehaviorTests_OVClassLoadNetworkTest, - OVClassLoadNetworkTestNPU, - ::testing::Combine(::testing::ValuesIn(devices), ::testing::ValuesIn(configs)), - OVClassLoadNetworkTestNPU::getTestCaseName); - -// -// NPU specific metrics -// - -using OVClassGetMetricAndPrintNoThrow = OVClassBaseTestP; -TEST_P(OVClassGetMetricAndPrintNoThrow, DeviceAllocMemSizeLesserThanTotalMemSizeNPU) { - SKIP_IF_CURRENT_TEST_IS_DISABLED() - ov::Core ie; - ov::Any p; - - OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_total_mem_size.name())); - uint64_t t = p.as(); - ASSERT_NE(t, 0); - - OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); - uint64_t a = p.as(); - - ASSERT_LT(a, t); - - std::cout << "OV NPU device alloc/total memory size: " << a << "/" << t << std::endl; -} - -TEST_P(OVClassGetMetricAndPrintNoThrow, DeviceAllocMemSizeLesserAfterModelIsLoadedNPU) { - SKIP_IF_CURRENT_TEST_IS_DISABLED() - ov::Core ie; - ov::Any p; - - OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); - uint64_t a1 = p.as(); - - SKIP_IF_CURRENT_TEST_IS_DISABLED() { - auto model = ov::test::utils::make_conv_pool_relu(); - OV_ASSERT_NO_THROW(ie.compile_model(model, target_device, {})); - } - - OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); - uint64_t a2 = p.as(); - - std::cout << "OV NPU device {alloc before load network/alloc after load network} memory size: {" << a1 << "/" << a2 - << "}" << std::endl; - - // after the network is loaded onto device, allocated memory value should increase - ASSERT_LE(a1, a2); -} - -TEST_P(OVClassGetMetricAndPrintNoThrow, VpuDeviceAllocMemSizeLesserAfterModelIsLoaded) { - SKIP_IF_CURRENT_TEST_IS_DISABLED() - ov::Core ie; - ov::Any p; - - OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); - uint64_t a1 = p.as(); - - SKIP_IF_CURRENT_TEST_IS_DISABLED() { - auto model = ov::test::utils::make_conv_pool_relu(); - OV_ASSERT_NO_THROW(ie.compile_model(model, target_device, ov::AnyMap{ov::log::level(ov::log::Level::DEBUG)})); - } - - OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::device_alloc_mem_size.name())); - uint64_t a2 = p.as(); - - std::cout << "OV NPU device {alloc before load network/alloc after load network} memory size: {" << a1 << "/" << a2 - << "}" << std::endl; - - // after the network is loaded onto device, allocated memory value should increase - ASSERT_LE(a1, a2); -} - -TEST_P(OVClassGetMetricAndPrintNoThrow, DriverVersionNPU) { - SKIP_IF_CURRENT_TEST_IS_DISABLED() - ov::Core ie; - ov::Any p; - - OV_ASSERT_NO_THROW(p = ie.get_property(target_device, ov::intel_npu::driver_version.name())); - uint32_t t = p.as(); - - std::cout << "NPU driver version is " << t << std::endl; - - OV_ASSERT_PROPERTY_SUPPORTED(ov::intel_npu::driver_version.name()); -} - -using OVClassCompileModel = OVClassBaseTestP; -TEST_P(OVClassCompileModel, CompileModelWithDifferentThreadNumbers) { - SKIP_IF_CURRENT_TEST_IS_DISABLED() - ov::Core ie; - ov::Any p; - - auto model = ov::test::utils::make_conv_pool_relu(); - OV_ASSERT_NO_THROW(ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(1)}})); - - OV_ASSERT_NO_THROW(ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(2)}})); - - OV_ASSERT_NO_THROW(ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(4)}})); - - EXPECT_ANY_THROW(ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(-1)}})); - OV_EXPECT_THROW( - std::ignore = ie.compile_model(model, target_device, {{ov::compilation_num_threads.name(), ov::Any(-1)}}), - ::ov::Exception, - testing::HasSubstr("ov::compilation_num_threads must be positive int32 value")); -} - -INSTANTIATE_TEST_SUITE_P(nightly_BehaviorTests_OVClassGetMetricTest, - OVClassGetMetricAndPrintNoThrow, - ::testing::Values(ov::test::utils::DEVICE_NPU), - OVClassNetworkTestName::getTestCaseName); - // Several devices case INSTANTIATE_TEST_SUITE_P(compatibility_nightly_BehaviorTests_OVClassSeveralDevicesTest, OVClassSeveralDevicesTestCompileModel, diff --git a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp index aa514eed14c1fa..c6a1ca705279ec 100644 --- a/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp +++ b/src/plugins/intel_npu/tests/functional/shared_tests_instances/behavior/ov_plugin/properties_tests.cpp @@ -177,6 +177,14 @@ INSTANTIATE_TEST_SUITE_P( ::testing::ValuesIn(getRWMandatoryPropertiesValues(CorrectPluginMutableProperties))), (ov::test::utils::appendPlatformTypeTestName)); +INSTANTIATE_TEST_SUITE_P( + smoke_BehaviorTests_OVCheckSetSupportedRWMandatoryMetricsPropsTests, + OVCheckSetSupportedRWMetricsPropsTests, + ::testing::Combine(::testing::Values("MULTI", "AUTO"), + ::testing::ValuesIn(OVCheckSetSupportedRWMetricsPropsTests::getRWOptionalPropertiesValues( + {ov::log::level.name()}))), + ov::test::utils::appendPlatformTypeTestName); + INSTANTIATE_TEST_SUITE_P( smoke_BehaviorTests_OVCheckGetSupportedROMetricsPropsTests, OVCheckGetSupportedROMetricsPropsTests, @@ -275,7 +283,6 @@ const std::vector configsDevicePropertiesDouble = { {ov::device::properties( ov::AnyMap{{ov::test::utils::DEVICE_NPU, ov::AnyMap{ov::num_streams(ov::streams::AUTO)}}})}}; -// IE Class load and check network with ov::device::properties INSTANTIATE_TEST_SUITE_P( smoke_NPU_BehaviorTests_OVClassCompileModelAndCheckSecondaryPropertiesTest, OVClassCompileModelAndCheckSecondaryPropertiesTest, @@ -288,4 +295,16 @@ INSTANTIATE_TEST_SUITE_P( ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU), ::testing::ValuesIn(configsDevicePropertiesDouble)), (ov::test::utils::appendPlatformTypeTestName)); + +// OVClassCompileModelAndCheckSecondaryPropertiesTest only works with property num_streams of type int32_t +INSTANTIATE_TEST_SUITE_P(DISABLED_smoke_BehaviorTests_OVClassLoadNetworkAndCheckWithSecondaryPropertiesTest, + OVClassCompileModelAndCheckSecondaryPropertiesTest, + ::testing::Combine(::testing::Values(ov::test::utils::DEVICE_NPU, "AUTO:NPU", "MULTI:NPU"), + ::testing::ValuesIn(configsDeviceProperties))); + +INSTANTIATE_TEST_SUITE_P(BehaviorTests_OVGetConfigTest_nightly, + OVGetConfigTest, + ::testing::Values(ov::test::utils::DEVICE_NPU), + ov::test::utils::appendPlatformTypeTestName); + }; // namespace