From 4e150db5d454ae4c41f44024d3762bb0bcba07de Mon Sep 17 00:00:00 2001 From: Luwei Zhou Date: Wed, 2 Jun 2021 15:25:43 +0800 Subject: [PATCH 1/7] Update Swish OP description. Signed-off-by: Luwei Zhou --- docs/ops/activation/Swish_4.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/ops/activation/Swish_4.md b/docs/ops/activation/Swish_4.md index 1a8b7d1b51a4f9..04f17390a16783 100644 --- a/docs/ops/activation/Swish_4.md +++ b/docs/ops/activation/Swish_4.md @@ -9,7 +9,8 @@ **Detailed description** *Swish* operation is introduced in this [article](https://arxiv.org/abs/1710.05941). -It performs element-wise activation function on a given input tensor, based on the following mathematical formula: + +*Swish* is a smooth, non-monotonic function. The non-monotonicity property of *Swish* distinguishes itself from most common activation functions. It performs element-wise activation function on a given input tensor, based on the following mathematical formula: \f[ Swish(x) = x\cdot \sigma(\beta x) = x \left(1 + e^{-(\beta x)}\right)^{-1} From a6f5c74bd6ba56a6d4e4c8eb7d03ff692acc3d4f Mon Sep 17 00:00:00 2001 From: Luwei Zhou Date: Wed, 2 Jun 2021 15:34:37 +0800 Subject: [PATCH 2/7] Use RTTI to declare/define NGraph Swish OP. Add input element type check when constructing Swish OP. Signed-off-by: Luwei Zhou --- ngraph/core/include/ngraph/op/swish.hpp | 3 +-- ngraph/core/src/op/swish.cpp | 8 +++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ngraph/core/include/ngraph/op/swish.hpp b/ngraph/core/include/ngraph/op/swish.hpp index a9fed2785e66a9..144b99846ee993 100644 --- a/ngraph/core/include/ngraph/op/swish.hpp +++ b/ngraph/core/include/ngraph/op/swish.hpp @@ -20,8 +20,7 @@ namespace ngraph class NGRAPH_API Swish : public ngraph::op::Op { public: - static constexpr NodeTypeInfo type_info{"Swish", 4}; - const NodeTypeInfo& get_type_info() const override { return type_info; } + NGRAPH_RTTI_DECLARATION; Swish() = default; /// \brief Constructs an Swish operation. diff --git a/ngraph/core/src/op/swish.cpp b/ngraph/core/src/op/swish.cpp index 5b3b7175950d7a..9f5a4a0c8e3d28 100644 --- a/ngraph/core/src/op/swish.cpp +++ b/ngraph/core/src/op/swish.cpp @@ -14,7 +14,7 @@ using namespace std; using namespace ngraph; -constexpr NodeTypeInfo op::v4::Swish::type_info; +NGRAPH_RTTI_DEFINITION(op::v4::Swish, "Swish", 4); op::v4::Swish::Swish(const Output& arg) : Op({arg}) @@ -43,6 +43,12 @@ void op::v4::Swish::validate_and_infer_types() "Swish must have 1 or 2 inputs, but it has: ", inputs_count); + NODE_VALIDATION_CHECK(this, + get_input_element_type(0).is_real(), + "Swish input tensor must be floating point type(", + get_input_element_type(0), + ")."); + if (inputs_count == 2) { NODE_VALIDATION_CHECK(this, From 5ab3e1406c7103a4ff47978ee4cb0c40cceea0c5 Mon Sep 17 00:00:00 2001 From: Luwei Zhou Date: Wed, 2 Jun 2021 16:12:24 +0800 Subject: [PATCH 3/7] Add Swish into activation serialization test list. Signed-off-by: Luwei Zhou --- .../inference_engine/serialization/single_layer/activation.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/inference-engine/tests/functional/inference_engine/serialization/single_layer/activation.cpp b/inference-engine/tests/functional/inference_engine/serialization/single_layer/activation.cpp index ff6a32786c8a6a..1b4541203ef044 100644 --- a/inference-engine/tests/functional/inference_engine/serialization/single_layer/activation.cpp +++ b/inference-engine/tests/functional/inference_engine/serialization/single_layer/activation.cpp @@ -52,6 +52,7 @@ const std::map>> activationTypes {Ceiling, {}}, {Mish, {}}, {HSwish, {}}, + {Swish, {{0.3f}}}, {SoftPlus, {}}, {HSigmoid, {}}, {RoundHalfToEven, {}}, From 25d0a663cfb37273eff8ad13a8a92caa6ec7570a Mon Sep 17 00:00:00 2001 From: Luwei Zhou Date: Wed, 2 Jun 2021 15:41:06 +0800 Subject: [PATCH 4/7] Add Swish into IE CPU plugin activation single layer test suit. Signed-off-by: Luwei Zhou --- .../shared_tests_instances/single_layer_tests/activation.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/activation.cpp b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/activation.cpp index 51e379a67f128b..4248bdfd8c30da 100644 --- a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/activation.cpp +++ b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/activation.cpp @@ -59,7 +59,8 @@ const std::map>> activationTypes {RoundHalfAwayFromZero, {}}, {Erf, {}}, {GeluErf, {}}, - {GeluTanh, {}} + {GeluTanh, {}}, + {Swish, {{0.4f}}} }; // List of operations that should be tested also with integer precision From 697d7c39d31f8e3dc00b08054c2494edef67516b Mon Sep 17 00:00:00 2001 From: Luwei Zhou Date: Wed, 2 Jun 2021 16:25:00 +0800 Subject: [PATCH 5/7] Add Swish NGraph backend and visitor API tests. Signed-off-by: Luwei Zhou --- ngraph/test/CMakeLists.txt | 2 + ngraph/test/backend/swish.in.cpp | 75 +++++++++++++++++++++++++++++++ ngraph/test/visitors/op/swish.cpp | 26 +++++++++++ 3 files changed, 103 insertions(+) create mode 100644 ngraph/test/backend/swish.in.cpp create mode 100644 ngraph/test/visitors/op/swish.cpp diff --git a/ngraph/test/CMakeLists.txt b/ngraph/test/CMakeLists.txt index 6f4e2bb0db47a2..4d10066af7c081 100644 --- a/ngraph/test/CMakeLists.txt +++ b/ngraph/test/CMakeLists.txt @@ -295,6 +295,7 @@ set(SRC visitors/op/squeeze.cpp visitors/op/sqrt.cpp visitors/op/strided_slice.cpp + visitors/op/swish.cpp visitors/op/tanh.cpp visitors/op/topk.cpp visitors/op/transpose.cpp @@ -471,6 +472,7 @@ set(MULTI_TEST_SRC backend/squared_difference.in.cpp backend/squeeze.in.cpp backend/subtract.in.cpp + backend/swish.in.cpp backend/tan.in.cpp backend/tanh.in.cpp backend/tile.in.cpp diff --git a/ngraph/test/backend/swish.in.cpp b/ngraph/test/backend/swish.in.cpp new file mode 100644 index 00000000000000..549b6678b50d63 --- /dev/null +++ b/ngraph/test/backend/swish.in.cpp @@ -0,0 +1,75 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "gtest/gtest.h" +#include "ngraph/ngraph.hpp" +#include "util/engine/test_engines.hpp" +#include "util/test_case.hpp" +#include "util/test_control.hpp" + +using namespace std; +using namespace ngraph; + +static string s_manifest = "${MANIFEST}"; +using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME}); + +NGRAPH_TEST(${BACKEND_NAME}, swish_2D_with_beta0_6) +{ + Shape in_shape{2, 4}; + element::Type et = element::f32; + auto beta = 0.6f; + + auto args0 = make_shared(et, in_shape); + auto args1 = make_shared(et, Shape{}); + auto swish = make_shared(args0, args1); + auto f = make_shared(swish, ParameterVector{args0, args1}); + + vector> in_vec{vector{0.4, -5.7, -6, 3, -0.9, 23, 5, 3.3} , vector{beta}}; + vector out_vec{in_vec[0]}; + std::transform(out_vec.begin(), out_vec.end(), out_vec.begin(), [&beta](float x) -> float { return (x / (1.0f + std::exp(x * beta * -1.0f)));}); + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs(in_vec); + test_case.add_expected_output(in_shape, out_vec); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, swish_2D_without_beta) +{ + Shape in_shape{2, 3}; + element::Type et = element::f32; + + auto args0 = make_shared(et, in_shape); + auto swish = make_shared(args0); + auto f = make_shared(swish, ParameterVector{args0}); + + vector in_vec{1, 8, -8, 17, -0.5, -1}; + vector out_vec{in_vec}; + std::transform(out_vec.begin(), out_vec.end(), out_vec.begin(), [](float x) -> float { return (x / (1.0f + std::exp(x * -1.0f)));}); + + auto test_case = test::TestCase(f); + test_case.add_input(in_vec); + test_case.add_expected_output(in_shape, out_vec); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, swish_4D_with_beta0_33) +{ + Shape in_shape{2, 2, 1, 2}; + element::Type et = element::f32; + auto beta = 0.33f; + + auto args0 = make_shared(et, in_shape); + auto args1 = make_shared(et, Shape{}); + auto swish = make_shared(args0, args1); + auto f = make_shared(swish, ParameterVector{args0, args1}); + + vector> in_vec{vector{0.1, 0.6, 20, -7, -5.3, 3.5, -9, 11} , vector{beta}}; + vector out_vec{in_vec[0]}; + std::transform(out_vec.begin(), out_vec.end(), out_vec.begin(), [&beta](float x) -> float { return (x / (1.0f + std::exp(x * beta * -1.0f)));}); + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs(in_vec); + test_case.add_expected_output(in_shape, out_vec); + test_case.run(); +} diff --git a/ngraph/test/visitors/op/swish.cpp b/ngraph/test/visitors/op/swish.cpp new file mode 100644 index 00000000000000..a87d90f4a3f5c6 --- /dev/null +++ b/ngraph/test/visitors/op/swish.cpp @@ -0,0 +1,26 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "gtest/gtest.h" + +#include "ngraph/ngraph.hpp" +#include "ngraph/op/util/attr_types.hpp" +#include "ngraph/opsets/opset4.hpp" +#include "util/visitor.hpp" + +using namespace std; +using namespace ngraph; +using ngraph::test::NodeBuilder; + +TEST(attributes, swish_op) +{ + NodeBuilder::get_ops().register_factory(); + const auto A = make_shared(element::f32, Shape{5, 2}); + + const auto swish = make_shared(A); + NodeBuilder builder(swish); + + const auto expected_attr_count = 0; + EXPECT_EQ(builder.get_value_map_size(), expected_attr_count); +} From a96e05c0279d2a305fd3858f24c0fe3f26ac9b61 Mon Sep 17 00:00:00 2001 From: Luwei Zhou Date: Wed, 2 Jun 2021 16:27:13 +0800 Subject: [PATCH 6/7] Add Swish unsupported parameter data type test cases. Signed-off-by: Luwei Zhou --- ngraph/test/type_prop/swish.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ngraph/test/type_prop/swish.cpp b/ngraph/test/type_prop/swish.cpp index bdb45ed1292a36..324d8f46a59cc7 100644 --- a/ngraph/test/type_prop/swish.cpp +++ b/ngraph/test/type_prop/swish.cpp @@ -81,3 +81,17 @@ TEST(type_prop, swish_2_inputs) ASSERT_TRUE(swish_func->get_output_partial_shape(0).same_scheme(data->get_output_shape(0))); ASSERT_TRUE(swish_func->get_output_partial_shape(0).rank().is_static()); } + +TEST(type_prop, swish_incompatible_type_boolean) +{ + auto data = make_shared(element::boolean, Shape{1, 3, 6}); + auto beta = make_shared(element::f32, Shape{}); + ASSERT_THROW(make_shared(data, beta);, ngraph::NodeValidationFailure); +} + +TEST(type_prop, swish_incompatible_types_u32) +{ + auto data = make_shared(element::f32, Shape{1, 3, 6}); + auto beta = make_shared(element::u32, Shape{}); + ASSERT_THROW(make_shared(data, beta);, ngraph::NodeValidationFailure); +} From 15227af9907b57b2ce56f29880395fb6bf3a8619 Mon Sep 17 00:00:00 2001 From: Luwei Zhou Date: Mon, 21 Jun 2021 13:41:38 +0800 Subject: [PATCH 7/7] Update the Swish OP visistor API to use typed test. Signed-off-by: Luwei Zhou --- ngraph/test/visitors/op/swish.cpp | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/ngraph/test/visitors/op/swish.cpp b/ngraph/test/visitors/op/swish.cpp index a87d90f4a3f5c6..0cb39f353cddaf 100644 --- a/ngraph/test/visitors/op/swish.cpp +++ b/ngraph/test/visitors/op/swish.cpp @@ -1,26 +1,11 @@ -// Copyright (C) 2018-2021 Intel Corporation +// Copyright (C) 2021 Intel Corporation // SPDX-License-Identifier: Apache-2.0 // +#include "unary_ops.hpp" -#include "gtest/gtest.h" +using Type = ::testing::Types>; -#include "ngraph/ngraph.hpp" -#include "ngraph/op/util/attr_types.hpp" -#include "ngraph/opsets/opset4.hpp" -#include "util/visitor.hpp" - -using namespace std; -using namespace ngraph; -using ngraph::test::NodeBuilder; - -TEST(attributes, swish_op) -{ - NodeBuilder::get_ops().register_factory(); - const auto A = make_shared(element::f32, Shape{5, 2}); - - const auto swish = make_shared(A); - NodeBuilder builder(swish); - - const auto expected_attr_count = 0; - EXPECT_EQ(builder.get_value_map_size(), expected_attr_count); -} +INSTANTIATE_TYPED_TEST_CASE_P(visitor_without_atrribute, + UnaryOperatorVisitor, + Type, + UnaryOperatorTypeName);