leandron commented on a change in pull request #6395: URL: https://github.com/apache/incubator-tvm/pull/6395#discussion_r483526749
########## File path: python/tvm/relay/op/contrib/tensorrt.py ########## @@ -0,0 +1,675 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# pylint: disable=invalid-name, unused-argument +"""TensorRT supported operators.""" +import tvm +from tvm import relay +from tvm.relay import transform +from tvm.relay.build_module import bind_params_by_name +from tvm.relay.expr import Call, Constant, Tuple, GlobalVar +from tvm.relay.expr_functor import ExprMutator + +import os +import numpy as np + +# Version to use for annotation when there is no linked TRT. +TENSORRT_VERSION = (6, 0, 1) +USE_IMPLICIT_BATCH = True +REMOVE_NO_MAC_SUBGRAPHS = False + +def is_tensorrt_runtime_enabled(): + """Check if the TensorRT graph runtime is present. + Returns + ------- + ret: bool + True if present, False if not. + """ + check_enabled = tvm.get_global_func("relay.op.is_tensorrt_runtime_enabled", True) + if check_enabled: + return check_enabled() + return False + +def get_tensorrt_version(): + """Gets the version of TensorRT that TVM is built against. + + Returns + ------- + ret: Tuple[int] + TensorRT version as a tuple of major, minor, and patch number. If TVM + is not built with TensorRT, the value set by set_tensorrt_version() is returned instead. + """ + linked_ver = tuple(tvm.get_global_func("relay.op.get_tensorrt_version")()) + if len(linked_ver) == 3: + return linked_ver + return TENSORRT_VERSION + +def set_tensorrt_version(version): + """Override TensorRT version for annotation + + Returns + ------- + version: Tuple[int] + TensorRT version as a tuple of major, minor, and patch number. If TVM + is not built with TensorRT, an empty tuple is returned instead. + """ + global TENSORRT_VERSION + TENSORRT_VERSION = version + +def get_tensorrt_use_implicit_batch_mode(): + return USE_IMPLICIT_BATCH + +def set_tensorrt_use_implicit_batch_mode(use_implicit_batch): + global USE_IMPLICIT_BATCH + USE_IMPLICIT_BATCH = use_implicit_batch + +def get_tensorrt_remove_no_mac_subgraphs(): + return REMOVE_NO_MAC_SUBGRAPHS + +def set_tensorrt_remove_no_mac_subgraphs(remove_no_mac_subgraphs): + global REMOVE_NO_MAC_SUBGRAPHS + REMOVE_NO_MAC_SUBGRAPHS = remove_no_mac_subgraphs + +def partition_for_tensorrt(mod, params=None, version=None, use_implicit_batch=True, remove_no_mac_subgraphs=False, max_workspace_size=1 << 30): + """Partition the graph greedily offloading supported + operators to TensorRT. + Parameters + ---------- + mod : Module + The module to run passes on. + params : Optional[Dict[str, NDArray]] + Constant input parameters. + version : Optional[Tuple(int)] + TensorRT version to target as tuple of (major, minor, patch). Will use linked TRT version if available if version is not specified. + use_implicit_batch : Optional[bool] + + remove_no_mac_subgraphs : Optional[bool] + + Returns + ------- + ret : annotated and partitioned module. + """ + if version: + assert isinstance(version, tuple) and len(version) == 3 + set_tensorrt_version(version) + set_tensorrt_use_implicit_batch_mode(use_implicit_batch) + set_tensorrt_remove_no_mac_subgraphs(remove_no_mac_subgraphs) + if params: + mod['main'] = bind_params_by_name(mod['main'], params) + + seq = tvm.transform.Sequential([transform.InferType(), + RemoveDropoutPass(), + transform.RemoveUnusedFunctions(), + transform.ConvertLayout({'nn.conv2d': ['NCHW', 'default'], + 'nn.conv3d': ['NCDHW', 'default']}), + transform.FoldConstant(), + transform.AnnotateTarget('tensorrt'), + transform.MergeCompilerRegions(), + transform.PartitionGraph(), + transform.InferType()]) + with tvm.transform.PassContext(opt_level=3): + mod = seq(mod) + mod = prune_tensorrt_subgraphs(mod) + # Pass parameters to codegen + os.environ["TVM_TENSORRT_USE_IMPLICIT_BATCH"] = str(int(use_implicit_batch)) + os.environ["TVM_TENSORRT_MAX_WORKSPACE_SIZE"] = str(int(max_workspace_size)) Review comment: I see this is covered on your PR original message :) - so you can ignore this one. ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org
