[VPU] StaticShapeNMS support

inference-engine/cmake/vpu_dependencies.cmake

@@ -19,7 +19,7 @@ set(VPU_SUPPORTED_FIRMWARES usb-ma2450 usb-ma2x8x pcie-ma248x)
 # Default packages
 #
 
-set(FIRMWARE_PACKAGE_VERSION 1239)
+set(FIRMWARE_PACKAGE_VERSION 1240)
 set(VPU_CLC_MA2X8X_VERSION "movi-cltools-20.02.0")
 
 #

inference-engine/src/vpu/common/include/vpu/ngraph/operations/static_shape_non_maximum_suppression.hpp

@@ -14,7 +14,7 @@ namespace ngraph { namespace vpu { namespace op {
 
 class StaticShapeNonMaxSuppression : public ngraph::op::v4::NonMaxSuppression {
 public:
- static constexpr NodeTypeInfo type_info{"StaticShapeStaticShapeNonMaxSuppression", 0};
+ static constexpr NodeTypeInfo type_info{"StaticShapeNonMaxSuppression", 0};
  const NodeTypeInfo& get_type_info() const override { return type_info; }
  StaticShapeNonMaxSuppression() = default;
 

inference-engine/src/vpu/common/src/ngraph/operations/static_shape_non_maximum_suppression.cpp

@@ -68,11 +68,11 @@ void StaticShapeNonMaxSuppression::validate_and_infer_types() {
 
  const auto out_shape = this->get_output_partial_shape(0);
  NODE_VALIDATION_CHECK(this, out_shape.is_static(),
- "StaticShapeTopK output shape is not fully defined: ", out_shape);
+ "StaticShapeNonMaxSuppression output shape is not fully defined: ", out_shape);
 
  set_output_size(2);
  set_output_type(0, m_output_type, out_shape);
- set_output_type(1, ngraph::element::i64, Shape{2});
+ set_output_type(1, m_output_type, Shape{2});
 }
 
 } // namespace op

inference-engine/src/vpu/graph_transformer/include/vpu/frontend/frontend.hpp

@@ -153,6 +153,7 @@ class FrontEnd final {
  void parseROIAlign(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const;
  void parseOutShapeOfReshape(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const;
  void parseBroadcast(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const;
+ void parseStaticShapeNMS(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const;
 
  //
  // Special layers

inference-engine/src/vpu/graph_transformer/include/vpu/model/stage.hpp

@@ -167,6 +167,7 @@ VPU_DECLARE_ENUM(StageType,
  OutShapeOfReshape = 127,
  Concat = 128,
  Broadcast = 129,
+ StaticShapeNMS = 130,
 )
 
 //

inference-engine/src/vpu/graph_transformer/src/frontend/frontend.cpp

@@ -114,6 +114,7 @@ FrontEnd::FrontEnd(StageBuilder::Ptr stageBuilder)
  {"DynamicShapeResolver", LAYER_PARSER(parseDSR)},
  {"OutShapeOfReshape", LAYER_PARSER(parseOutShapeOfReshape)},
  {"StaticShapeBroadcast", LAYER_PARSER(parseBroadcast)},
+ {"StaticShapeNonMaxSuppression", LAYER_PARSER(parseStaticShapeNMS)},
  }} {}
 
 ModelPtr FrontEnd::buildInitialModel(ie::ICNNNetwork& network) {

inference-engine/src/vpu/graph_transformer/src/stages/static_shape_nms.cpp

@@ -0,0 +1,106 @@
+// Copyright (C) 2018-2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <vpu/frontend/frontend.hpp>
+
+#include <ngraph/op/non_max_suppression.hpp>
+
+#include <memory>
+#include <set>
+
+namespace vpu {
+
+namespace {
+
+class StaticShapeNMS final : public StageNode {
+private:
+ StagePtr cloneImpl() const override {
+ return std::make_shared<StaticShapeNMS>(*this);
+ }
+
+ void propagateDataOrderImpl(StageDataInfo<DimsOrder>& orderInfo) override {
+ }
+
+ void getDataStridesRequirementsImpl(StageDataInfo<StridesRequirement>& stridesInfo) override {
+ }
+
+ void finalizeDataLayoutImpl() override {
+ }
+
+ void getBatchSupportInfoImpl(StageDataInfo<BatchSupport>& batchInfo) override {
+ }
+
+ StageSHAVEsRequirements getSHAVEsRequirementsImpl() const override {
+ return StageSHAVEsRequirements::OnlyOne;
+ }
+
+ void initialCheckImpl() const override {
+ assertInputsOutputsTypes(this,
+ {{DataType::FP16},
+ {DataType::FP16},
+ {DataType::S32},
+ {DataType::FP16},
+ {DataType::FP16}},
+ {{DataType::S32},
+ {DataType::S32}});
+ }
+
+ void finalCheckImpl() const override {
+ initialCheckImpl();
+ }
+
+ void serializeParamsImpl(BlobSerializer& serializer) const override {
+ bool center_point_box = attrs().get<bool>("center_point_box");
+
+ serializer.append(static_cast<int32_t>(center_point_box));
+ }
+
+ void serializeDataImpl(BlobSerializer& serializer) const override {
+ auto input1 = inputEdges()[0]->input();
+ auto input2 = inputEdges()[1]->input();
+ auto input3 = inputEdges()[2]->input();
+ auto input4 = inputEdges()[3]->input();
+ auto input5 = inputEdges()[4]->input();
+ auto outputData = outputEdges()[0]->output();
+ auto outputDims = outputEdges()[1]->output();
+
+ input1->serializeBuffer(serializer);
+ input2->serializeBuffer(serializer);
+ input3->serializeBuffer(serializer);
+ input4->serializeBuffer(serializer);
+ input5->serializeBuffer(serializer);
+ outputData->serializeBuffer(serializer);
+ outputDims->serializeBuffer(serializer);
+ }
+};
+
+} // namespace
+
+void FrontEnd::parseStaticShapeNMS(const Model& model, const ie::CNNLayerPtr& layer, const DataVector& inputs, const DataVector& outputs) const {
+ VPU_THROW_UNLESS(inputs.size() >= 2 && inputs.size() <= 5,
+ "StaticShapeNMS parsing failed, expected number of input is in range [2, 5], but {} provided",
+ inputs.size());
+ VPU_THROW_UNLESS(outputs.size() == 2,
+ "StaticShapeNMS parsing failed, expected number of outputs: 2, but {} provided",
+ outputs.size());
+
+ const auto sortResultDescending = layer->GetParamAsBool("sort_result_descending");
+ const auto boxEncoding = layer->GetParamAsString("box_encoding");
+
+ VPU_THROW_UNLESS(sortResultDescending == false,
+ "StaticShapeNMS: parameter sortResultDescending=true is not supported on VPU");
+ VPU_THROW_UNLESS(boxEncoding == "corner" || boxEncoding == "center",
+ "StaticShapeNMS: boxEncoding currently supports only two values: \"corner\" and \"center\" "
+ "while {} was provided", boxEncoding);
+
+ DataVector tempInputs = inputs;
+ for (auto fake = inputs.size(); fake < 5; fake++) {
+ tempInputs.push_back(model->addFakeData());
+ }
+
+ auto stage = model->addNewStage<StaticShapeNMS>(layer->name, StageType::StaticShapeNMS, layer, tempInputs, outputs);
+ stage->attrs().set<bool>("center_point_box", boxEncoding == "center");
+}
+
+} // namespace vpu

inference-engine/tests/functional/plugin/myriad/single_layer_tests/static_shape_nms.cpp

@@ -0,0 +1,117 @@
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <functional_test_utils/layer_test_utils.hpp>
+#include <functional_test_utils/blob_utils.hpp>
+
+#include <ngraph/opsets/opset3.hpp>
+#include "vpu/ngraph/operations/static_shape_non_maximum_suppression.hpp"
+
+using TensorShape = InferenceEngine::SizeVector;
+
+using StaticShapeNMSParam = std::tuple<
+ int64_t, // Number of batches
+ int64_t, // Number of boxes
+ int64_t, // Number of classes
+ int64_t, // Maximum output boxes per class
+ float, // IOU threshold
+ float>; // Score threshold
+
+using StaticShapeNMSTestParam = std::tuple<
+ StaticShapeNMSParam, // NMS params
+ InferenceEngine::Precision, // Input precision
+ LayerTestsUtils::TargetDevice>; // Device name
+
+namespace LayerTestsDefinitions {
+
+class StaticShapeNMSLayerTest : public testing::WithParamInterface<StaticShapeNMSTestParam>,
+ public LayerTestsUtils::LayerTestsCommon {
+public:
+ static std::string getTestCaseName(const testing::TestParamInfo<StaticShapeNMSTestParam>& obj) {
+ StaticShapeNMSParam NMSParams;
+ InferenceEngine::Precision inputPrecision;
+ std::string targetDevice;
+ std::tie(NMSParams, inputPrecision, targetDevice) = obj.param;
+
+ const auto numBatches = std::get<0>(NMSParams);
+ const auto numBoxes = std::get<1>(NMSParams);
+ const auto numClasses = std::get<2>(NMSParams);
+ const auto maxOutputBoxesPerClass = std::get<3>(NMSParams);
+ const auto iouThreshold = std::get<4>(NMSParams);
+ const auto scoreThreshold = std::get<5>(NMSParams);
+
+ std::ostringstream result;
+ result << "numBatches=" << numBatches << "_";
+ result << "numBoxes=" << numBoxes << "_";
+ result << "numClasses=" << numClasses << "_";
+ result << "maxOutputBoxesPerClass=" << maxOutputBoxesPerClass << "_";
+ result << "iouThreshold=" << iouThreshold << "_";
+ result << "scoreThreshold=" << scoreThreshold << "_";
+ result << "inPrc=" << inputPrecision.name() << "_";
+ result << "targetDevice=" << targetDevice;
+ return result.str();
+ }
+
+protected:
+ void SetUp() override {
+ SetRefMode(LayerTestsUtils::RefMode::INTERPRETER);
+
+ StaticShapeNMSParam NMSParams;
+ std::tie(NMSParams, inPrc, targetDevice) = this->GetParam();
+
+ const auto numBatches = std::get<0>(NMSParams);
+ const auto numBoxes = std::get<1>(NMSParams);
+ const auto numClasses = std::get<2>(NMSParams);
+ const auto maxOutputBoxesPerClass = std::get<3>(NMSParams);
+ const auto iouThreshold = std::get<4>(NMSParams);
+ const auto scoreThreshold = std::get<5>(NMSParams);
+
+ auto ngPrc = FuncTestUtils::PrecisionUtils::convertIE2nGraphPrc(inPrc);
+
+ const auto inputBoxes = std::make_shared<ngraph::opset3::Parameter>(
+ ngPrc, ngraph::Shape({static_cast<size_t>(numBatches), static_cast<size_t>(numBoxes), 4}));
+ const auto inputScores = std::make_shared<ngraph::opset3::Parameter>(
+ ngPrc, ngraph::Shape({static_cast<size_t>(numBatches), static_cast<size_t>(numClasses), static_cast<size_t>(numBoxes)}));
+ const auto maxOutputBoxesPerClassConst = std::make_shared<ngraph::opset3::Constant>(
+ ngraph::element::i64, ngraph::Shape{}, maxOutputBoxesPerClass);
+ const auto iouThresholdConst = std::make_shared<ngraph::opset3::Constant>(
+ ngraph::element::f32, ngraph::Shape{}, iouThreshold);
+ const auto scoreThresholdConst = std::make_shared<ngraph::opset3::Constant>(
+ ngraph::element::f32, ngraph::Shape{}, scoreThreshold);
+
+ const auto staticShapeNMS = std::make_shared<ngraph::vpu::op::StaticShapeNonMaxSuppression>(
+ inputBoxes, inputScores, maxOutputBoxesPerClassConst, iouThresholdConst, scoreThresholdConst,
+ ngraph::opset3::NonMaxSuppression::BoxEncodingType::CORNER, false, ngraph::element::i32);
+
+ ngraph::ResultVector results{std::make_shared<ngraph::opset3::Result>(staticShapeNMS->output(0)),
+ std::make_shared<ngraph::opset3::Result>(staticShapeNMS->output(1))};
+ function = std::make_shared<ngraph::Function>(results, ngraph::ParameterVector{inputBoxes, inputScores});
+ }
+};
+
+TEST_P(StaticShapeNMSLayerTest, accuracy) {
+ Run();
+}
+
+std::vector<StaticShapeNMSParam> NMSParams = {
+ std::make_tuple(1, 10, 5, 10, 0., 0.),
+ std::make_tuple(2, 100, 5, 10, 0., 0.),
+ std::make_tuple(3, 10, 5, 2, 0.5, 0.),
+ std::make_tuple(1, 1000, 1, 2000, 0.5, 0.)
+};
+
+std::vector<InferenceEngine::Precision> NMSPrecisions = {
+ InferenceEngine::Precision::FP32,
+ InferenceEngine::Precision::FP16,
+};
+
+// #-30919
+INSTANTIATE_TEST_CASE_P(DISABLED_accuracy, StaticShapeNMSLayerTest,
+ ::testing::Combine(
+ ::testing::ValuesIn(NMSParams),
+ ::testing::ValuesIn(NMSPrecisions),
+ ::testing::Values(CommonTestUtils::DEVICE_MYRIAD)),
+ StaticShapeNMSLayerTest::getTestCaseName);
+
+} // namespace LayerTestsDefinitions

inference-engine/tests/functional/plugin/myriad/subgraph_tests/nms_nonzero.cpp

@@ -0,0 +1,80 @@
+// Copyright (C) 2020 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include <functional_test_utils/layer_test_utils.hpp>
+#include <ngraph_functions/builders.hpp>
+
+namespace {
+
+using DataType = ngraph::element::Type_t;
+using DataDims = ngraph::Shape;
+
+struct NonMaxSuppressionTestCase {
+ int64_t num_batches, num_boxes, num_classes, max_output_boxes_per_class;
+ float iou_threshold, score_threshold;
+};
+
+using Parameters = std::tuple<
+ DataType,
+ DataType,
+ NonMaxSuppressionTestCase,
+ LayerTestsUtils::TargetDevice
+>;
+
+class NMS_NonZero : public testing::WithParamInterface<Parameters>,
+ public LayerTestsUtils::LayerTestsCommon {
+protected:
+ void SetUp() override {
+ const auto& parameters = GetParam();
+ const auto& float_type = std::get<0>(parameters);
+ const auto& integer_type = std::get<1>(parameters);
+ const auto& nms_setup = std::get<2>(parameters);
+ targetDevice = std::get<3>(parameters);
+
+ const auto boxes = std::make_shared<ngraph::opset3::Parameter>(
+ float_type, ngraph::PartialShape{nms_setup.num_batches, nms_setup.num_boxes, 4});
+ const auto scores = std::make_shared<ngraph::opset3::Parameter>(
+ float_type, ngraph::PartialShape{nms_setup.num_batches, nms_setup.num_classes, nms_setup.num_boxes});
+ const auto max_output_boxes_per_class = std::make_shared<ngraph::opset3::Constant>(
+ integer_type, ngraph::Shape{}, std::vector<int64_t>{nms_setup.max_output_boxes_per_class});
+ const auto iou_threshold = std::make_shared<ngraph::opset3::Constant>(
+ float_type, ngraph::Shape{}, std::vector<float>{nms_setup.iou_threshold});
+ const auto score_threshold = std::make_shared<ngraph::opset3::Constant>(
+ float_type, ngraph::Shape{}, std::vector<float>{nms_setup.score_threshold});
+
+ const auto NMS = std::make_shared<ngraph::opset3::NonMaxSuppression>(
+ boxes, scores, max_output_boxes_per_class, iou_threshold, score_threshold,
+ ngraph::opset3::NonMaxSuppression::BoxEncodingType::CORNER, false, ngraph::element::i32);
+
+ const auto nonZero = std::make_shared<ngraph::opset3::NonZero>(boxes);
+
+ const auto resultNMS = std::make_shared<ngraph::opset3::Result>(NMS);
+ const auto resultNonZero = std::make_shared<ngraph::opset3::Result>(nonZero);
+ function = std::make_shared<ngraph::Function>(ngraph::ResultVector{resultNMS, resultNonZero},
+ ngraph::ParameterVector{boxes, scores}, "opset3::NMS-NonZero");
+ }
+};
+
+TEST_P(NMS_NonZero, CompareWithReference) {
+ Run();
+}
+
+// #-30919
+INSTANTIATE_TEST_CASE_P(DISABLED_DynamicNonMaxSupression, NMS_NonZero,
+ ::testing::Combine(
+ ::testing::Values(
+ ngraph::element::f16,
+ ngraph::element::f32),
+ ::testing::Values(
+ ngraph::element::i32,
+ ngraph::element::u8),
+ ::testing::Values(
+ // num_batches, num_boxes, num_classes, max_output_boxes_per_class, iou_threshold, score_threshold
+ NonMaxSuppressionTestCase{1, 10, 5, 10, 0., 0.},
+ NonMaxSuppressionTestCase{2, 100, 5, 10, 0., 0.},
+ NonMaxSuppressionTestCase{3, 10, 5, 2, 0.5, 0.},
+ NonMaxSuppressionTestCase{1, 1000, 1, 2000, 0.5, 0.}),
+ ::testing::Values(CommonTestUtils::DEVICE_MYRIAD)));
+
+} // namespace