diff --git a/docs/ops/activation/Swish_4.md b/docs/ops/activation/Swish_4.md index 1a8b7d1b51a4f9..04f17390a16783 100644 --- a/docs/ops/activation/Swish_4.md +++ b/docs/ops/activation/Swish_4.md @@ -9,7 +9,8 @@ **Detailed description** *Swish* operation is introduced in this [article](https://arxiv.org/abs/1710.05941). -It performs element-wise activation function on a given input tensor, based on the following mathematical formula: + +*Swish* is a smooth, non-monotonic function. The non-monotonicity property of *Swish* distinguishes itself from most common activation functions. It performs element-wise activation function on a given input tensor, based on the following mathematical formula: \f[ Swish(x) = x\cdot \sigma(\beta x) = x \left(1 + e^{-(\beta x)}\right)^{-1} diff --git a/inference-engine/tests/functional/inference_engine/serialization/single_layer/activation.cpp b/inference-engine/tests/functional/inference_engine/serialization/single_layer/activation.cpp index ff6a32786c8a6a..1b4541203ef044 100644 --- a/inference-engine/tests/functional/inference_engine/serialization/single_layer/activation.cpp +++ b/inference-engine/tests/functional/inference_engine/serialization/single_layer/activation.cpp @@ -52,6 +52,7 @@ const std::map>> activationTypes {Ceiling, {}}, {Mish, {}}, {HSwish, {}}, + {Swish, {{0.3f}}}, {SoftPlus, {}}, {HSigmoid, {}}, {RoundHalfToEven, {}}, diff --git a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/activation.cpp b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/activation.cpp index 51e379a67f128b..4248bdfd8c30da 100644 --- a/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/activation.cpp +++ b/inference-engine/tests/functional/plugin/cpu/shared_tests_instances/single_layer_tests/activation.cpp @@ -59,7 +59,8 @@ const std::map>> activationTypes {RoundHalfAwayFromZero, {}}, {Erf, {}}, {GeluErf, {}}, - {GeluTanh, {}} + {GeluTanh, {}}, + {Swish, {{0.4f}}} }; // List of operations that should be tested also with integer precision diff --git a/ngraph/core/include/ngraph/op/swish.hpp b/ngraph/core/include/ngraph/op/swish.hpp index a9fed2785e66a9..144b99846ee993 100644 --- a/ngraph/core/include/ngraph/op/swish.hpp +++ b/ngraph/core/include/ngraph/op/swish.hpp @@ -20,8 +20,7 @@ namespace ngraph class NGRAPH_API Swish : public ngraph::op::Op { public: - static constexpr NodeTypeInfo type_info{"Swish", 4}; - const NodeTypeInfo& get_type_info() const override { return type_info; } + NGRAPH_RTTI_DECLARATION; Swish() = default; /// \brief Constructs an Swish operation. diff --git a/ngraph/core/src/op/swish.cpp b/ngraph/core/src/op/swish.cpp index 5b3b7175950d7a..9f5a4a0c8e3d28 100644 --- a/ngraph/core/src/op/swish.cpp +++ b/ngraph/core/src/op/swish.cpp @@ -14,7 +14,7 @@ using namespace std; using namespace ngraph; -constexpr NodeTypeInfo op::v4::Swish::type_info; +NGRAPH_RTTI_DEFINITION(op::v4::Swish, "Swish", 4); op::v4::Swish::Swish(const Output& arg) : Op({arg}) @@ -43,6 +43,12 @@ void op::v4::Swish::validate_and_infer_types() "Swish must have 1 or 2 inputs, but it has: ", inputs_count); + NODE_VALIDATION_CHECK(this, + get_input_element_type(0).is_real(), + "Swish input tensor must be floating point type(", + get_input_element_type(0), + ")."); + if (inputs_count == 2) { NODE_VALIDATION_CHECK(this, diff --git a/ngraph/test/CMakeLists.txt b/ngraph/test/CMakeLists.txt index 6f4e2bb0db47a2..4d10066af7c081 100644 --- a/ngraph/test/CMakeLists.txt +++ b/ngraph/test/CMakeLists.txt @@ -295,6 +295,7 @@ set(SRC visitors/op/squeeze.cpp visitors/op/sqrt.cpp visitors/op/strided_slice.cpp + visitors/op/swish.cpp visitors/op/tanh.cpp visitors/op/topk.cpp visitors/op/transpose.cpp @@ -471,6 +472,7 @@ set(MULTI_TEST_SRC backend/squared_difference.in.cpp backend/squeeze.in.cpp backend/subtract.in.cpp + backend/swish.in.cpp backend/tan.in.cpp backend/tanh.in.cpp backend/tile.in.cpp diff --git a/ngraph/test/backend/swish.in.cpp b/ngraph/test/backend/swish.in.cpp new file mode 100644 index 00000000000000..549b6678b50d63 --- /dev/null +++ b/ngraph/test/backend/swish.in.cpp @@ -0,0 +1,75 @@ +// Copyright (C) 2018-2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "gtest/gtest.h" +#include "ngraph/ngraph.hpp" +#include "util/engine/test_engines.hpp" +#include "util/test_case.hpp" +#include "util/test_control.hpp" + +using namespace std; +using namespace ngraph; + +static string s_manifest = "${MANIFEST}"; +using TestEngine = test::ENGINE_CLASS_NAME(${BACKEND_NAME}); + +NGRAPH_TEST(${BACKEND_NAME}, swish_2D_with_beta0_6) +{ + Shape in_shape{2, 4}; + element::Type et = element::f32; + auto beta = 0.6f; + + auto args0 = make_shared(et, in_shape); + auto args1 = make_shared(et, Shape{}); + auto swish = make_shared(args0, args1); + auto f = make_shared(swish, ParameterVector{args0, args1}); + + vector> in_vec{vector{0.4, -5.7, -6, 3, -0.9, 23, 5, 3.3} , vector{beta}}; + vector out_vec{in_vec[0]}; + std::transform(out_vec.begin(), out_vec.end(), out_vec.begin(), [&beta](float x) -> float { return (x / (1.0f + std::exp(x * beta * -1.0f)));}); + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs(in_vec); + test_case.add_expected_output(in_shape, out_vec); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, swish_2D_without_beta) +{ + Shape in_shape{2, 3}; + element::Type et = element::f32; + + auto args0 = make_shared(et, in_shape); + auto swish = make_shared(args0); + auto f = make_shared(swish, ParameterVector{args0}); + + vector in_vec{1, 8, -8, 17, -0.5, -1}; + vector out_vec{in_vec}; + std::transform(out_vec.begin(), out_vec.end(), out_vec.begin(), [](float x) -> float { return (x / (1.0f + std::exp(x * -1.0f)));}); + + auto test_case = test::TestCase(f); + test_case.add_input(in_vec); + test_case.add_expected_output(in_shape, out_vec); + test_case.run(); +} + +NGRAPH_TEST(${BACKEND_NAME}, swish_4D_with_beta0_33) +{ + Shape in_shape{2, 2, 1, 2}; + element::Type et = element::f32; + auto beta = 0.33f; + + auto args0 = make_shared(et, in_shape); + auto args1 = make_shared(et, Shape{}); + auto swish = make_shared(args0, args1); + auto f = make_shared(swish, ParameterVector{args0, args1}); + + vector> in_vec{vector{0.1, 0.6, 20, -7, -5.3, 3.5, -9, 11} , vector{beta}}; + vector out_vec{in_vec[0]}; + std::transform(out_vec.begin(), out_vec.end(), out_vec.begin(), [&beta](float x) -> float { return (x / (1.0f + std::exp(x * beta * -1.0f)));}); + + auto test_case = test::TestCase(f); + test_case.add_multiple_inputs(in_vec); + test_case.add_expected_output(in_shape, out_vec); + test_case.run(); +} diff --git a/ngraph/test/type_prop/swish.cpp b/ngraph/test/type_prop/swish.cpp index bdb45ed1292a36..324d8f46a59cc7 100644 --- a/ngraph/test/type_prop/swish.cpp +++ b/ngraph/test/type_prop/swish.cpp @@ -81,3 +81,17 @@ TEST(type_prop, swish_2_inputs) ASSERT_TRUE(swish_func->get_output_partial_shape(0).same_scheme(data->get_output_shape(0))); ASSERT_TRUE(swish_func->get_output_partial_shape(0).rank().is_static()); } + +TEST(type_prop, swish_incompatible_type_boolean) +{ + auto data = make_shared(element::boolean, Shape{1, 3, 6}); + auto beta = make_shared(element::f32, Shape{}); + ASSERT_THROW(make_shared(data, beta);, ngraph::NodeValidationFailure); +} + +TEST(type_prop, swish_incompatible_types_u32) +{ + auto data = make_shared(element::f32, Shape{1, 3, 6}); + auto beta = make_shared(element::u32, Shape{}); + ASSERT_THROW(make_shared(data, beta);, ngraph::NodeValidationFailure); +} diff --git a/ngraph/test/visitors/op/swish.cpp b/ngraph/test/visitors/op/swish.cpp new file mode 100644 index 00000000000000..0cb39f353cddaf --- /dev/null +++ b/ngraph/test/visitors/op/swish.cpp @@ -0,0 +1,11 @@ +// Copyright (C) 2021 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// +#include "unary_ops.hpp" + +using Type = ::testing::Types>; + +INSTANTIATE_TYPED_TEST_CASE_P(visitor_without_atrribute, + UnaryOperatorVisitor, + Type, + UnaryOperatorTypeName);