[GitHub] KellenSunderland commented on a change in pull request #14040: Reformat of TensorRT to use subgraph API
KellenSunderland commented on a change in pull request #14040: Reformat of
TensorRT to use subgraph API
URL: https://github.com/apache/incubator-mxnet/pull/14040#discussion_r253288751
##
File path: src/operator/subgraph/tensorrt/tensorrt-inl.h
##
@@ -0,0 +1,217 @@
+#ifndef MXNET_OPERATOR_SUBGRAPH_TENSORRT_TENSORRT_INL_H_
+#define MXNET_OPERATOR_SUBGRAPH_TENSORRT_TENSORRT_INL_H_
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2018 by Contributors
+ * \file tensorrt-inl.h
+ * \brief TensorRT operation registration
+ * \author Marek Kolodziej, Clement Fuji Tsang
+*/
+
+#if MXNET_USE_TENSORRT
+
+
+#include "../common.h"
+#include "../subgraph_property.h"
+#include "nnvm_to_onnx-inl.h"
+#include "./onnx_to_tensorrt.h"
+
+namespace mxnet {
+namespace op {
+
+using int64 = ::google::protobuf::int64;
+
+struct TRTParam {
+ std::unordered_map inputs_to_idx;
+ std::unordered_map outputs_to_idx;
+ std::unordered_map params_map;
+};
+
+struct TRTEngineParam {
+ nvinfer1::IExecutionContext* trt_executor = nullptr;
+ std::vector > binding_vec;
+};
+
+
+class TensorrtSelector : public SubgraphSelector {
+ public:
+ const std::unordered_set unconditionalTRTops = {
+"Convolution",
+"BatchNorm",
+"elemwise_add",
+"elemwise_sub",
+"elemwise_mul",
+"rsqrt",
+"pad",
+"Pad",
+"mean",
+"FullyConnected",
+"Flatten",
+"SoftmaxOutput",
+ };
+
+ const std::unordered_set withWeightsOps = {
+"Convolution",
+"BatchNorm",
+"FullyConnected"
+ };
+
+ bool isTRTCompatible(const nnvm::Node ) {
+const std::string op_name = n.op()->name;
+if (op_name == "Pooling") {
+ return (n.attrs.dict.at("pool_type") == "avg" ||
+ n.attrs.dict.at("pool_type") == "max");
+}
+
+if (unconditionalTRTops.count(op_name)) {
+ return true;
+}
+
+if (op_name == "Activation") {
+ return n.attrs.dict.at("act_type") == "relu" ||
+n.attrs.dict.at("act_type") == "tanh" ||
+n.attrs.dict.at("act_type") == "sigmoid";
+}
+
+return false;
+ }
+
+ bool Select(const nnvm::Node ) override {
+return !n.is_variable() && isTRTCompatible(n);
+ }
+
+ bool SelectInput(const nnvm::Node , const nnvm::Node _node) override {
+if (new_node.is_variable()) {
+ if (withWeightsOps.count(n.op()->name)) {
+return n.inputs[0].node->attrs.name != new_node.attrs.name;
+ } else {
+return false;
+ }
+}
+if (isTRTCompatible(new_node))
+ return true;
+return false;
+ }
+
+ bool SelectOutput(const nnvm::Node , const nnvm::Node _node) override {
+ return isTRTCompatible(new_node);
+ }
+
+ std::vector Filter(const std::vector& candidates)
override {
+bool found_one = false;
+// TensorRT is interesting with at least 2 operations
+for (auto& n : candidates) {
+ if (!n->is_variable()) {
+if (found_one) {
+ return candidates;
+} else {
+ found_one = true;
+}
+ }
+}
+return std::vector();
+ }
+};
+
+class TensorrtProperty : public SubgraphProperty {
+ public:
+ static SubgraphPropertyPtr Create() {
+return std::make_shared();
+ }
+
+ nnvm::NodePtr CreateSubgraphNode(const nnvm::Symbol ,
+ const int subgraph_id = 0) const override {
Review comment:
If possible can we avoid default values in overriding functions.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] KellenSunderland commented on a change in pull request #14040: Reformat of TensorRT to use subgraph API
KellenSunderland commented on a change in pull request #14040: Reformat of
TensorRT to use subgraph API
URL: https://github.com/apache/incubator-mxnet/pull/14040#discussion_r253288427
##
File path: src/operator/subgraph/tensorrt/tensorrt-inl.h
##
@@ -0,0 +1,217 @@
+#ifndef MXNET_OPERATOR_SUBGRAPH_TENSORRT_TENSORRT_INL_H_
+#define MXNET_OPERATOR_SUBGRAPH_TENSORRT_TENSORRT_INL_H_
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2018 by Contributors
+ * \file tensorrt-inl.h
+ * \brief TensorRT operation registration
+ * \author Marek Kolodziej, Clement Fuji Tsang
+*/
+
+#if MXNET_USE_TENSORRT
+
+
+#include "../common.h"
+#include "../subgraph_property.h"
+#include "nnvm_to_onnx-inl.h"
+#include "./onnx_to_tensorrt.h"
+
+namespace mxnet {
+namespace op {
+
+using int64 = ::google::protobuf::int64;
+
+struct TRTParam {
+ std::unordered_map inputs_to_idx;
+ std::unordered_map outputs_to_idx;
+ std::unordered_map params_map;
+};
+
+struct TRTEngineParam {
+ nvinfer1::IExecutionContext* trt_executor = nullptr;
+ std::vector > binding_vec;
+};
+
+
+class TensorrtSelector : public SubgraphSelector {
+ public:
+ const std::unordered_set unconditionalTRTops = {
+"Convolution",
+"BatchNorm",
+"elemwise_add",
+"elemwise_sub",
+"elemwise_mul",
+"rsqrt",
+"pad",
+"Pad",
+"mean",
+"FullyConnected",
+"Flatten",
+"SoftmaxOutput",
+ };
+
+ const std::unordered_set withWeightsOps = {
+"Convolution",
+"BatchNorm",
+"FullyConnected"
+ };
+
+ bool isTRTCompatible(const nnvm::Node ) {
+const std::string op_name = n.op()->name;
+if (op_name == "Pooling") {
+ return (n.attrs.dict.at("pool_type") == "avg" ||
+ n.attrs.dict.at("pool_type") == "max");
+}
+
+if (unconditionalTRTops.count(op_name)) {
+ return true;
+}
+
+if (op_name == "Activation") {
+ return n.attrs.dict.at("act_type") == "relu" ||
+n.attrs.dict.at("act_type") == "tanh" ||
+n.attrs.dict.at("act_type") == "sigmoid";
+}
+
+return false;
+ }
+
+ bool Select(const nnvm::Node ) override {
+return !n.is_variable() && isTRTCompatible(n);
+ }
+
+ bool SelectInput(const nnvm::Node , const nnvm::Node _node) override {
+if (new_node.is_variable()) {
+ if (withWeightsOps.count(n.op()->name)) {
+return n.inputs[0].node->attrs.name != new_node.attrs.name;
+ } else {
+return false;
+ }
+}
+if (isTRTCompatible(new_node))
+ return true;
+return false;
+ }
+
+ bool SelectOutput(const nnvm::Node , const nnvm::Node _node) override {
+ return isTRTCompatible(new_node);
+ }
+
+ std::vector Filter(const std::vector& candidates)
override {
+bool found_one = false;
+// TensorRT is interesting with at least 2 operations
+for (auto& n : candidates) {
+ if (!n->is_variable()) {
+if (found_one) {
+ return candidates;
+} else {
+ found_one = true;
+}
+ }
+}
+return std::vector();
+ }
+};
+
+class TensorrtProperty : public SubgraphProperty {
+ public:
+ static SubgraphPropertyPtr Create() {
+return std::make_shared();
+ }
+
+ nnvm::NodePtr CreateSubgraphNode(const nnvm::Symbol ,
+ const int subgraph_id = 0) const override {
+nnvm::NodePtr n = nnvm::Node::Create();
+nnvm::Symbol new_sym;
+std::unique_copy(sym.outputs.begin(), sym.outputs.end(),
+std::back_inserter(new_sym.outputs), [](
+nnvm::NodeEntry lhs, nnvm::NodeEntry rhs) {
+ return lhs.index == rhs.index && lhs.node.get() == rhs.node.get();
+});
+n->attrs.name = "TensorRT" + std::to_string(subgraph_id);
+n->attrs.op = Op::Get("_TensorRT");
+CHECK(n->attrs.op);
+n->attrs.subgraphs.emplace_back(std::make_shared(new_sym));
+std::ostringstream params_oss;
+for (auto : new_sym.ListInputNames(nnvm::Symbol::kAll)) {
+ params_oss << e << ";";
+}
+auto tensorrt_params_names = params_oss.str();
+tensorrt_params_names.pop_back();
+n->attrs.dict["subgraph_params_names"] = tensorrt_params_names;
+TRTParam param;
+n->attrs.parsed
[GitHub] KellenSunderland commented on a change in pull request #14040: Reformat of TensorRT to use subgraph API
KellenSunderland commented on a change in pull request #14040: Reformat of
TensorRT to use subgraph API
URL: https://github.com/apache/incubator-mxnet/pull/14040#discussion_r253288413
##
File path: src/operator/subgraph/tensorrt/tensorrt-inl.h
##
@@ -0,0 +1,217 @@
+#ifndef MXNET_OPERATOR_SUBGRAPH_TENSORRT_TENSORRT_INL_H_
+#define MXNET_OPERATOR_SUBGRAPH_TENSORRT_TENSORRT_INL_H_
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * Copyright (c) 2018 by Contributors
+ * \file tensorrt-inl.h
+ * \brief TensorRT operation registration
+ * \author Marek Kolodziej, Clement Fuji Tsang
+*/
+
+#if MXNET_USE_TENSORRT
+
+
+#include "../common.h"
+#include "../subgraph_property.h"
+#include "nnvm_to_onnx-inl.h"
+#include "./onnx_to_tensorrt.h"
+
+namespace mxnet {
+namespace op {
+
+using int64 = ::google::protobuf::int64;
+
+struct TRTParam {
+ std::unordered_map inputs_to_idx;
+ std::unordered_map outputs_to_idx;
+ std::unordered_map params_map;
+};
+
+struct TRTEngineParam {
+ nvinfer1::IExecutionContext* trt_executor = nullptr;
+ std::vector > binding_vec;
+};
+
+
+class TensorrtSelector : public SubgraphSelector {
+ public:
+ const std::unordered_set unconditionalTRTops = {
+"Convolution",
+"BatchNorm",
+"elemwise_add",
+"elemwise_sub",
+"elemwise_mul",
+"rsqrt",
+"pad",
+"Pad",
+"mean",
+"FullyConnected",
+"Flatten",
+"SoftmaxOutput",
+ };
+
+ const std::unordered_set withWeightsOps = {
+"Convolution",
+"BatchNorm",
+"FullyConnected"
+ };
+
+ bool isTRTCompatible(const nnvm::Node ) {
+const std::string op_name = n.op()->name;
+if (op_name == "Pooling") {
+ return (n.attrs.dict.at("pool_type") == "avg" ||
+ n.attrs.dict.at("pool_type") == "max");
+}
+
+if (unconditionalTRTops.count(op_name)) {
+ return true;
+}
+
+if (op_name == "Activation") {
+ return n.attrs.dict.at("act_type") == "relu" ||
+n.attrs.dict.at("act_type") == "tanh" ||
+n.attrs.dict.at("act_type") == "sigmoid";
+}
+
+return false;
+ }
+
+ bool Select(const nnvm::Node ) override {
+return !n.is_variable() && isTRTCompatible(n);
+ }
+
+ bool SelectInput(const nnvm::Node , const nnvm::Node _node) override {
+if (new_node.is_variable()) {
+ if (withWeightsOps.count(n.op()->name)) {
+return n.inputs[0].node->attrs.name != new_node.attrs.name;
+ } else {
+return false;
+ }
+}
+if (isTRTCompatible(new_node))
Review comment:
Can be simplified to
```C++
return isTRTCompatible(new_node);
```
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] KellenSunderland commented on a change in pull request #14040: Reformat of TensorRT to use subgraph API
KellenSunderland commented on a change in pull request #14040: Reformat of
TensorRT to use subgraph API
URL: https://github.com/apache/incubator-mxnet/pull/14040#discussion_r253288287
##
File path: src/operator/subgraph/tensorrt/onnx_to_tensorrt.cc
##
@@ -127,16 +128,15 @@ nvinfer1::ICudaEngine* onnxToTrtCtx(
}
throw dmlc::Error("Cannot parse ONNX into TensorRT Engine");
}
-
- bool fp16 = trt_builder->platformHasFastFp16();
-
+ if (dmlc::GetEnv("MXNET_TENSORRT_USE_FP16", true)) {
+if (trt_builder->platformHasFastFp16()) {
+ trt_builder->setFp16Mode(true);
+} else {
+ LOG(INFO) << "WARNING: TensorRT can't use fp16 on this plateform";
Review comment:
Also we're logging INFO level logs but have WARNING in the message. I'd
remove the warning from the message and set log level to warning. (this is a
common issue in our codebase).
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] KellenSunderland commented on a change in pull request #14040: Reformat of TensorRT to use subgraph API
KellenSunderland commented on a change in pull request #14040: Reformat of
TensorRT to use subgraph API
URL: https://github.com/apache/incubator-mxnet/pull/14040#discussion_r253288274
##
File path: src/operator/subgraph/tensorrt/onnx_to_tensorrt.cc
##
@@ -127,16 +128,15 @@ nvinfer1::ICudaEngine* onnxToTrtCtx(
}
throw dmlc::Error("Cannot parse ONNX into TensorRT Engine");
}
-
- bool fp16 = trt_builder->platformHasFastFp16();
-
+ if (dmlc::GetEnv("MXNET_TENSORRT_USE_FP16", true)) {
+if (trt_builder->platformHasFastFp16()) {
+ trt_builder->setFp16Mode(true);
+} else {
+ LOG(INFO) << "WARNING: TensorRT can't use fp16 on this plateform";
Review comment:
plateform -> platform
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] KellenSunderland commented on a change in pull request #14040: Reformat of TensorRT to use subgraph API
KellenSunderland commented on a change in pull request #14040: Reformat of
TensorRT to use subgraph API
URL: https://github.com/apache/incubator-mxnet/pull/14040#discussion_r253288158
##
File path: src/operator/subgraph/tensorrt/nnvm_to_onnx.cc
##
@@ -411,6 +383,49 @@ void ConvertElementwiseAdd(NodeProto* node_proto, const
NodeAttrs& /*attrs*/,
node_proto->set_op_type("Add");
}
+inline TensorProto_DataType ConvertDType(int dtype) {
+ switch (dtype) {
+case mshadow::kFloat64:
+ return TensorProto_DataType_DOUBLE;
+case mshadow::kFloat32:
+ return TensorProto_DataType_FLOAT;
+case mshadow::kFloat16:
+ return TensorProto_DataType_FLOAT16;
+case mshadow::kUint8:
+ return TensorProto_DataType_UINT8;
+case mshadow::kInt32:
+ return TensorProto_DataType_INT32;
+case mshadow::kInt8:
+ return TensorProto_DataType_INT8;
+case mshadow::kInt64:
+ return TensorProto_DataType_INT64;
+default:
+ return TensorProto_DataType_UNDEFINED;
+ }
+}
+
+inline std::string StringDType(int dtype) {
Review comment:
Is this still needed, don't see any references to it.
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
[GitHub] KellenSunderland commented on a change in pull request #14040: Reformat of TensorRT to use subgraph API
KellenSunderland commented on a change in pull request #14040: Reformat of TensorRT to use subgraph API URL: https://github.com/apache/incubator-mxnet/pull/14040#discussion_r253288021 ## File path: include/mxnet/c_api.h ## @@ -1835,12 +1835,6 @@ MXNET_DLL int MXExecutorReshape(int partial_shaping, ExecutorHandle shared_exec, ExecutorHandle *out); -/*! - * \brief get optimized graph from graph executor - */ -MXNET_DLL int MXExecutorGetOptimizedSymbol(ExecutorHandle handle, - SymbolHandle *out); - Review comment: It affect semantic versioning if we remove it (it can break compilation on downstream projects) so if there's not a strong reason to remove it we should leave it in for that reason. Can it still preform a useful function (for example showing the graph after optimization)? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
[GitHub] KellenSunderland commented on a change in pull request #14040: Reformat of TensorRT to use subgraph API
KellenSunderland commented on a change in pull request #14040: Reformat of TensorRT to use subgraph API URL: https://github.com/apache/incubator-mxnet/pull/14040#discussion_r253288021 ## File path: include/mxnet/c_api.h ## @@ -1835,12 +1835,6 @@ MXNET_DLL int MXExecutorReshape(int partial_shaping, ExecutorHandle shared_exec, ExecutorHandle *out); -/*! - * \brief get optimized graph from graph executor - */ -MXNET_DLL int MXExecutorGetOptimizedSymbol(ExecutorHandle handle, - SymbolHandle *out); - Review comment: It affects semantic versioning if we remove it (it can break compilation on downstream projects) so if there's not a strong reason to remove it we should leave it in for that reason. Can it still preform a useful function (for example showing the graph after optimization)? This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
