@SunilJB So I tried to follow the example provided by TensorRT, but I seem to have run into a problem. The model I am trying to work with is: models/vision/super_resolution/sub_pixel_cnn_2016 at main · onnx/models · GitHub
The code I have implemented:
samplesCommon::OnnxSampleParams initializeSampleParams() { samplesCommon::OnnxSampleParams params; params.dataDirs.push_back("data/mnist/"); params.dataDirs.push_back("data/samples/mnist/"); /*params.onnxFileName = "mnist.onnx"; params.inputTensorNames.push_back("Input3"); params.outputTensorNames.push_back("Plus214_Output_0");*/ params.onnxFileName = "super_resolution.onnx"; params.inputTensorNames.push_back("input"); params.outputTensorNames.push_back("output"); params.batchSize = 1; params.int8 = false; params.fp16 = false; return params; } void SampleDynamicReshape::build() { auto builder = makeUnique(nvinfer1::createInferBuilder(gLogger.getTRTLogger())); // This function will also set mPredictionInputDims and mPredictionOutputDims, // so it needs to be called before building the preprocessor. buildPredictionEngine(builder); buildPreprocessorEngine(builder); } void SampleDynamicReshape::buildPreprocessorEngine(const SampleUniquePtr<nvinfer1::IBuilder>& builder) { // Create the preprocessor engine using a network that supports full dimensions (createNetworkV2). auto preprocessorNetwork = makeUnique( builder->createNetworkV2(1U << static_cast<uint32_t>(NetworkDefinitionCreationFlag::kEXPLICIT_BATCH))); // Reshape a dynamically shaped input to the size expected by the model, (1, 1, 28, 28). //auto input = preprocessorNetwork->addInput("input", nvinfer1::DataType::kFLOAT, Dims4{ 1, 1, -1, -1 }); auto input = preprocessorNetwork->addInput("input", nvinfer1::DataType::kFLOAT, Dims4{ -1, 1, 1, 1 }); auto resizeLayer = preprocessorNetwork->addResize(*input); resizeLayer->setOutputDimensions(mPredictionInputDims); preprocessorNetwork->markOutput(*resizeLayer->getOutput(0)); // Finally, configure and build the preprocessor engine. auto preprocessorConfig = makeUnique(builder->createBuilderConfig()); // Create an optimization profile so that we can specify a range of input dimensions. auto profile = builder->createOptimizationProfile(); // This profile will be valid for all images whose size falls in the range of [(1, 1, 1, 1), (1, 1, 224, 224)] // but TensorRT will optimize for (1, 1, 28, 28) /*profile->setDimensions(input->getName(), OptProfileSelector::kMIN, Dims4{ 1, 1, 1, 1}); profile->setDimensions(input->getName(), OptProfileSelector::kOPT, Dims4{ 1, 1, 28, 28 }); profile->setDimensions(input->getName(), OptProfileSelector::kMAX, Dims4{ 1, 1, 56, 56 });*/ profile->setDimensions(input->getName(), OptProfileSelector::kMIN, Dims4{ 1, 1, 1, 1 }); profile->setDimensions(input->getName(), OptProfileSelector::kOPT, Dims4{ 1, 1, 224, 224 }); profile->setDimensions(input->getName(), OptProfileSelector::kMAX, Dims4{ 1, 1, 224, 224}); preprocessorConfig->addOptimizationProfile(profile); mPreprocessorEngine = makeUnique(builder->buildEngineWithConfig(*preprocessorNetwork, *preprocessorConfig)); gLogInfo << "Profile dimensions in preprocessor engine:\n"; gLogInfo << " Minimum = " << mPreprocessorEngine->getProfileDimensions(0, 0, OptProfileSelector::kMIN) << '\n'; gLogInfo << " Optimum = " << mPreprocessorEngine->getProfileDimensions(0, 0, OptProfileSelector::kOPT) << '\n'; gLogInfo << " Maximum = " << mPreprocessorEngine->getProfileDimensions(0, 0, OptProfileSelector::kMAX) << std::endl;
}
void SampleDynamicReshape::buildPredictionEngine(const SampleUniquePtr<nvinfer1::IBuilder>& builder) { // Create a network using the parser. const auto explicitBatch = 1U << static_cast<uint32_t>(NetworkDefinitionCreationFlag::kEXPLICIT_BATCH); auto network = makeUnique(builder->createNetworkV2(explicitBatch)); auto parser = nvonnxparser::createParser(*network, gLogger.getTRTLogger()); bool parsingSuccess = parser->parseFromFile( locateFile(mParams.onnxFileName, mParams.dataDirs).c_str(), static_cast<int>(gLogger.getReportableSeverity())); if (!parsingSuccess) { throw std::runtime_error{ "Failed to parse model" }; } /*// Attach a softmax layer to the end of the network. auto softmax = network->addSoftMax(*network->getOutput(0)); // Set softmax axis to 1 since network output has shape [1, 10] in full dims mode softmax->setAxes(1 << 1); network->unmarkOutput(*network->getOutput(0)); network->markOutput(*softmax->getOutput(0));*/ // Get information about the inputs/outputs directly from the model. mPredictionInputDims = network->getInput(0)->getDimensions(); mPredictionOutputDims = network->getOutput(0)->getDimensions(); // Create a builder config auto config = makeUnique(builder->createBuilderConfig()); config->setMaxWorkspaceSize(16_MiB); if (mParams.fp16) { config->setFlag(BuilderFlag::kFP16); } if (mParams.int8) { config->setFlag(BuilderFlag::kINT8); samplesCommon::setAllTensorScales(network.get(), 127.0f, 127.0f); } // Build the prediciton engine. mPredictionEngine = makeUnique(builder->buildEngineWithConfig(*network, *config));
}
So the model seems to load and input/output is read, but I get an error on the following line:
mPredictionEngine = makeUnique(builder->buildEngineWithConfig(*network, *config));
[E] [TRT] Network has dynamic or shape inputs, but no optimization profile has been defined.
[E] [TRT] Network validation failed.
Seems that the right input shape is detected:
