<!DOCTYPE html> <html> <head> <meta charset="utf-8"/> <meta content="width=device-width, initial-scale=1.0" name="viewport"/> <meta content="width=device-width,initial-scale=1" name="viewport"/> <meta content="ie=edge" http-equiv="x-ua-compatible"/> <meta content="Copy to clipboard" name="lang:clipboard.copy"/> <meta content="Copied to clipboard" name="lang:clipboard.copied"/> <meta content="en" name="lang:search.language"/> <meta content="True" name="lang:search.pipeline.stopwords"/> <meta content="True" name="lang:search.pipeline.trimmer"/> <meta content="No matching documents" name="lang:search.result.none"/> <meta content="1 matching document" name="lang:search.result.one"/> <meta content="# matching documents" name="lang:search.result.other"/> <meta content="[\s\-]+" name="lang:search.tokenizer"/> <link crossorigin="" href="https://fonts.gstatic.com/" rel="preconnect"/> <link href="https://fonts.googleapis.com/css?family=Roboto+Mono:400,500,700|Roboto:300,400,400i,700&display=fallback" rel="stylesheet"/> <style> body, input { font-family: "Roboto", "Helvetica Neue", Helvetica, Arial, sans-serif } code, kbd, pre { font-family: "Roboto Mono", "Courier New", Courier, monospace } </style> <link href="../_static/stylesheets/application.css" rel="stylesheet"/> <link href="../_static/stylesheets/application-palette.css" rel="stylesheet"/> <link href="../_static/stylesheets/application-fixes.css" rel="stylesheet"/> <link href="../_static/fonts/material-icons.css" rel="stylesheet"/> <meta content="84bd00" name="theme-color"/> <script src="../_static/javascripts/modernizr.js"> </script> <title> torchtrtc — Torch-TensorRT v1.0.0 documentation </title> <link href="../_static/material.css" rel="stylesheet" type="text/css"/> <link href="../_static/pygments.css" rel="stylesheet" type="text/css"/> <link href="../_static/collapsible-lists/css/tree_view.css" rel="stylesheet" type="text/css"/> <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"> </script> <script src="../_static/jquery.js"> </script> <script src="../_static/underscore.js"> </script> <script src="../_static/doctools.js"> </script> <script src="../_static/language_data.js"> </script> <script src="../_static/collapsible-lists/js/CollapsibleLists.compressed.js"> </script> <script src="../_static/collapsible-lists/js/apply-collapsible-lists.js"> </script> <script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"> </script> <script async="async" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.7/latest.js?config=TeX-AMS-MML_HTMLorMML"> </script> <script type="text/x-mathjax-config"> MathJax.Hub.Config({"tex2jax": {"inlineMath": [["$", "$"], ["\\(", "\\)"]], "processEscapes": true, "ignoreClass": "document", "processClass": "math|output_area"}}) </script> <link href="../genindex.html" rel="index" title="Index"/> <link href="../search.html" rel="search" title="Search"/> <link href="use_from_pytorch.html" rel="next" title="Using Torch-TensorRT Directly From PyTorch"/> <link href="ptq.html" rel="prev" title="Post Training Quantization (PTQ)"/> </head> <body data-md-color-accent="light-green" data-md-color-primary="light-green" dir="ltr"> <svg class="md-svg"> <defs data-children-count="0"> <svg height="448" id="__github" viewbox="0 0 416 448" width="416" xmlns="http://www.w3.org/2000/svg"> <path d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z" fill="currentColor"> </path> </svg> </defs> </svg> <input class="md-toggle" data-md-toggle="drawer" id="__drawer" type="checkbox"/> <input class="md-toggle" data-md-toggle="search" id="__search" type="checkbox"/> <label class="md-overlay" data-md-component="overlay" for="__drawer"> </label> <a class="md-skip" href="#tutorials/torchtrtc" tabindex="1"> Skip to content </a> <header class="md-header" data-md-component="header"> <nav class="md-header-nav md-grid"> <div class="md-flex navheader"> <div class="md-flex__cell md-flex__cell--shrink"> <a class="md-header-nav__button md-logo" href="../index.html" title="Torch-TensorRT v1.0.0 documentation"> <i class="md-icon"> </i> </a> </div> <div class="md-flex__cell md-flex__cell--shrink"> <label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"> </label> </div> <div class="md-flex__cell md-flex__cell--stretch"> <div class="md-flex__ellipsis md-header-nav__title" data-md-component="title"> <span class="md-header-nav__topic"> Torch-TensorRT </span> <span class="md-header-nav__topic"> torchtrtc </span> </div> </div> <div class="md-flex__cell md-flex__cell--shrink"> <label class="md-icon md-icon--search md-header-nav__button" for="__search"> </label> <div class="md-search" data-md-component="search" role="dialog"> <label class="md-search__overlay" for="__search"> </label> <div class="md-search__inner" role="search"> <form action="../search.html" class="md-search__form" method="get" name="search"> <input autocapitalize="off" autocomplete="off" class="md-search__input" data-md-component="query" data-md-state="active" name="q" placeholder="Search" spellcheck="false" type="text"/> <label class="md-icon md-search__icon" for="__search"> </label> <button class="md-icon md-search__icon" data-md-component="reset" tabindex="-1" type="reset"> </button> </form> <div class="md-search__output"> <div class="md-search__scrollwrap" data-md-scrollfix=""> <div class="md-search-result" data-md-component="result"> <div class="md-search-result__meta"> Type to start searching </div> <ol class="md-search-result__list"> </ol> </div> </div> </div> </div> </div> </div> <div class="md-flex__cell md-flex__cell--shrink"> <div class="md-header-nav__source"> <a class="md-source" data-md-source="github" href="https://github.com/nvidia/Torch-TensorRT/" title="Go to repository"> <div class="md-source__icon"> <svg height="28" viewbox="0 0 24 24" width="28" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"> <use height="24" width="24" xlink:href="#__github"> </use> </svg> </div> <div class="md-source__repository"> Torch-TensorRT </div> </a> </div> </div> <div class="md-flex__cell md-flex__cell--shrink dropdown"> <button class="dropdownbutton"> Versions </button> <div class="dropdown-content md-hero"> <a href="https://nvidia.github.io/Torch-TensorRT/" title="master"> master </a> <a href="https://nvidia.github.io/Torch-TensorRT/v1.0.0/" title="v1.0.0"> v1.0.0 </a> <a href="https://nvidia.github.io/Torch-TensorRT/v0.4.1/" title="v0.4.1"> v0.4.1 </a> <a href="https://nvidia.github.io/Torch-TensorRT/v0.4.0/" title="v0.4.0"> v0.4.0 </a> <a href="https://nvidia.github.io/Torch-TensorRT/v0.3.0/" title="v0.3.0"> v0.3.0 </a> <a href="https://nvidia.github.io/Torch-TensorRT/v0.2.0/" title="v0.2.0"> v0.2.0 </a> <a href="https://nvidia.github.io/Torch-TensorRT/v0.1.0/" title="v0.1.0"> v0.1.0 </a> <a href="https://nvidia.github.io/Torch-TensorRT/v0.0.3/" title="v0.0.3"> v0.0.3 </a> <a href="https://nvidia.github.io/Torch-TensorRT/v0.0.2/" title="v0.0.2"> v0.0.2 </a> <a href="https://nvidia.github.io/Torch-TensorRT/v0.0.1/" title="v0.0.1"> v0.0.1 </a> </div> </div> </div> </nav> </header> <div class="md-container"> <nav class="md-tabs" data-md-component="tabs"> <div class="md-tabs__inner md-grid"> <ul class="md-tabs__list"> <li class="md-tabs__item"> <a class="md-tabs__link" href="../index.html"> Torch-TensorRT v1.0.0 documentation </a> </li> </ul> </div> </nav> <main class="md-main"> <div class="md-main__inner md-grid" data-md-component="container"> <div class="md-sidebar md-sidebar--primary" data-md-component="navigation"> <div class="md-sidebar__scrollwrap"> <div class="md-sidebar__inner"> <nav class="md-nav md-nav--primary" data-md-level="0"> <label class="md-nav__title md-nav__title--site" for="__drawer"> <a class="md-nav__button md-logo" href="../index.html" title="Torch-TensorRT v1.0.0 documentation"> <i class="md-icon"> </i> </a> <a href="../index.html" title="Torch-TensorRT v1.0.0 documentation"> Torch-TensorRT </a> </label> <div class="md-nav__source"> <a class="md-source" data-md-source="github" href="https://github.com/nvidia/Torch-TensorRT/" title="Go to repository"> <div class="md-source__icon"> <svg height="28" viewbox="0 0 24 24" width="28" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"> <use height="24" width="24" xlink:href="#__github"> </use> </svg> </div> <div class="md-source__repository"> Torch-TensorRT </div> </a> </div> <ul class="md-nav__list"> <li class="md-nav__item"> <span class="md-nav__link caption"> <span class="caption-text"> Getting Started </span> </span> </li> <li class="md-nav__item"> <a class="md-nav__link" href="installation.html"> Installation </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="getting_started_with_cpp_api.html"> Getting Started with C++ </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="getting_started_with_python_api.html"> Using Torch-TensorRT in Python </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="creating_torchscript_module_in_python.html"> Creating a TorchScript Module </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="creating_torchscript_module_in_python.html#working-with-torchscript-in-python"> Working with TorchScript in Python </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="creating_torchscript_module_in_python.html#saving-torchscript-module-to-disk"> Saving TorchScript Module to Disk </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="ptq.html"> Post Training Quantization (PTQ) </a> </li> <li class="md-nav__item"> <input class="md-toggle md-nav__toggle" data-md-toggle="toc" id="__toc" type="checkbox"/> <label class="md-nav__link md-nav__link--active" for="__toc"> torchtrtc </label> <a class="md-nav__link md-nav__link--active" href="#"> torchtrtc </a> <nav class="md-nav md-nav--secondary"> <ul class="md-nav__list" data-md-scrollfix=""> <li class="md-nav__item"> <a class="md-nav__extra_link" href="../_sources/tutorials/torchtrtc.rst.txt"> Show Source </a> </li> </ul> </nav> </li> <li class="md-nav__item"> <a class="md-nav__link" href="use_from_pytorch.html"> Using Torch-TensorRT Directly From PyTorch </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="runtime.html"> Deploying Torch-TensorRT Programs </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="using_dla.html"> DLA </a> </li> <li class="md-nav__item"> <span class="md-nav__link caption"> <span class="caption-text"> Notebooks </span> </span> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../_notebooks/lenet-getting-started.html"> Torch-TensorRT Getting Started - LeNet </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../_notebooks/Resnet50-example.html"> Torch-TensorRT Getting Started - ResNet 50 </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../_notebooks/ssd-object-detection-demo.html"> Object Detection with Torch-TensorRT (SSD) </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../_notebooks/vgg-qat.html"> Deploying Quantization Aware Trained models in INT8 using Torch-TensorRT </a> </li> <li class="md-nav__item"> <span class="md-nav__link caption"> <span class="caption-text"> Python API Documenation </span> </span> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../py_api/torch_tensorrt.html"> torch_tensorrt </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../py_api/logging.html"> torch_tensorrt.logging </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../py_api/ptq.html"> torch_tensorrt.ptq </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../py_api/ptq.html#classes"> Classes </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../py_api/ptq.html#enums"> Enums </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../py_api/ts.html"> torch_tensorrt.ts </a> </li> <li class="md-nav__item"> <span class="md-nav__link caption"> <span class="caption-text"> C++ API Documenation </span> </span> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../_cpp_api/torch_tensort_cpp.html"> Torch-TensorRT C++ API </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../_cpp_api/namespace_torch_tensorrt.html"> Namespace torch_tensorrt </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../_cpp_api/namespace_torch_tensorrt__logging.html"> Namespace torch_tensorrt::logging </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../_cpp_api/namespace_torch_tensorrt__torchscript.html"> Namespace torch_tensorrt::torchscript </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../_cpp_api/namespace_torch_tensorrt__ptq.html"> Namespace torch_tensorrt::ptq </a> </li> <li class="md-nav__item"> <span class="md-nav__link caption"> <span class="caption-text"> Contributor Documentation </span> </span> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../contributors/system_overview.html"> System Overview </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../contributors/writing_converters.html"> Writing Converters </a> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../contributors/useful_links.html"> Useful Links for Torch-TensorRT Development </a> </li> <li class="md-nav__item"> <span class="md-nav__link caption"> <span class="caption-text"> Indices </span> </span> </li> <li class="md-nav__item"> <a class="md-nav__link" href="../indices/supported_ops.html"> Operators Supported </a> </li> </ul> </nav> </div> </div> </div> <div class="md-sidebar md-sidebar--secondary" data-md-component="toc"> <div class="md-sidebar__scrollwrap"> <div class="md-sidebar__inner"> <nav class="md-nav md-nav--secondary"> <ul class="md-nav__list" data-md-scrollfix=""> <li class="md-nav__item"> <a class="md-nav__extra_link" href="../_sources/tutorials/torchtrtc.rst.txt"> Show Source </a> </li> <li class="md-nav__item" id="searchbox"> </li> </ul> </nav> </div> </div> </div> <div class="md-content"> <article class="md-content__inner md-typeset" role="main"> <span id="id1"> </span> <h1 id="tutorials-torchtrtc--page-root"> torchtrtc <a class="headerlink" href="#tutorials-torchtrtc--page-root" title="Permalink to this headline"> ¶ </a> </h1> <p> <code class="docutils literal notranslate"> <span class="pre"> torchtrtc </span> </code> is a CLI application for using the Torch-TensorRT compiler. It serves as an easy way to compile a TorchScript Module with Torch-TensorRT from the command-line to quickly check support or as part of a deployment pipeline. All basic features of the compiler are supported including post training quantization (though you must already have a calibration cache file to use the PTQ feature). The compiler can output two formats, either a TorchScript program with the TensorRT engine embedded or the TensorRT engine itself as a PLAN file. </p> <p> All that is required to run the program after compilation is for C++ linking against <code class="docutils literal notranslate"> <span class="pre"> libtorchtrt.so </span> </code> or in Python importing the torch_tensorrt package. All other aspects of using compiled modules are identical to standard TorchScript. Load with <code class="docutils literal notranslate"> <span class="pre"> torch.jit.load() </span> </code> and run like you would run any other module. </p> <div class="highlight-txt notranslate"> <div class="highlight"> <pre><span></span>torchtrtc [input_file_path] [output_file_path] [input_specs...] {OPTIONS} Torch-TensorRT is a compiler for TorchScript, it will compile and optimize TorchScript programs to run on NVIDIA GPUs using TensorRT OPTIONS: -h, --help Display this help menu Verbiosity of the compiler -v, --verbose Dumps debugging information about the compilation process onto the console -w, --warnings Disables warnings generated during compilation onto the console (warnings are on by default) --i, --info Dumps info messages generated during compilation onto the console --build-debuggable-engine Creates a debuggable engine --use-strict-types Restrict operating type to only use set operation precision --allow-gpu-fallback (Only used when targeting DLA (device-type)) Lets engine run layers on GPU if they are not supported on DLA --require-full-compilation Require that the model should be fully compiled to TensorRT or throw an error --disable-tf32 Prevent Float32 layers from using the TF32 data format --sparse-weights Enable sparsity for weights of conv and FC layers -p[precision...], --enabled-precision=[precision...] (Repeatable) Enabling an operating precision for kernels to use when building the engine (Int8 requires a calibration-cache argument) [ float | float32 | f32 | fp32 | half | float16 | f16 | fp16 | int8 | i8 | char ] (default: float) -d[type], --device-type=[type] The type of device the engine should be built for [ gpu | dla ] (default: gpu) --gpu-id=[gpu_id] GPU id if running on multi-GPU platform (defaults to 0) --dla-core=[dla_core] DLACore id if running on available DLA (defaults to 0) --engine-capability=[capability] The type of device the engine should be built for [ standard | safety | dla_standalone ] --calibration-cache-file=[file_path] Path to calibration cache file to use for post training quantization --teo=[torch-executed-ops...], --torch-executed-ops=[torch-executed-ops...] (Repeatable) Operator in the graph that should always be run in PyTorch for execution (partial compilation must be enabled) --tem=[torch-executed-mods...], --torch-executed-mods=[torch-executed-mods...] (Repeatable) Module that should always be run in Pytorch for execution (partial compilation must be enabled) --mbs=[torch-executed-mods...], --min-block-size=[torch-executed-mods...] Minimum number of contiguous TensorRT supported ops to compile a subgraph to TensorRT --embed-engine Whether to treat input file as a serialized TensorRT engine and embed it into a TorchScript module (device spec must be provided) --num-min-timing-iter=[num_iters] Number of minimization timing iterations used to select kernels --num-avg-timing-iters=[num_iters] Number of averaging timing iterations used to select kernels --workspace-size=[workspace_size] Maximum size of workspace given to TensorRT --max-batch-size=[max_batch_size] Maximum batch size (must be >= 1 to be set, 0 means not set) -t[threshold], --threshold=[threshold] Maximum acceptable numerical deviation from standard torchscript output (default 2e-5) --no-threshold-check Skip checking threshold compliance --truncate-long-double, --truncate, --truncate-64bit Truncate weights that are provided in 64bit to 32bit (Long, Double to Int, Float) --save-engine Instead of compiling a full a TorchScript program, save the created engine to the path specified as the output path input_file_path Path to input TorchScript file output_file_path Path for compiled TorchScript (or TensorRT engine) file input_specs... Specs for inputs to engine, can either be a single size or a range defined by Min, Optimal, Max sizes, e.g. "(N,..,C,H,W)" "[(MIN_N,..,MIN_C,MIN_H,MIN_W);(OPT_N,..,OPT_C,OPT_H,OPT_W);(MAX_N,..,MAX_C,MAX_H,MAX_W)]". Data Type and format can be specified by adding an "@" followed by dtype and "%" followed by format to the end of the shape spec. e.g. "(3, 3, 32, 32)@f16%NHWC" "--" can be used to terminate flag options and force all following arguments to be treated as positional options </pre> </div> </div> <p> e.g. </p> <div class="highlight-shell notranslate"> <div class="highlight"> <pre><span></span>torchtrtc tests/modules/ssd_traced.jit.pt ssd_trt.ts <span class="s2">"[(1,3,300,300); (1,3,512,512); (1, 3, 1024, 1024)]@f16%contiguous"</span> -p f16 </pre> </div> </div> </article> </div> </div> </main> </div> <footer class="md-footer"> <div class="md-footer-nav"> <nav class="md-footer-nav__inner md-grid"> <a class="md-flex md-footer-nav__link md-footer-nav__link--prev" href="ptq.html" rel="prev" title="Post Training Quantization (PTQ)"> <div class="md-flex__cell md-flex__cell--shrink"> <i class="md-icon md-icon--arrow-back md-footer-nav__button"> </i> </div> <div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title"> <span class="md-flex__ellipsis"> <span class="md-footer-nav__direction"> Previous </span> Post Training Quantization (PTQ) </span> </div> </a> <a class="md-flex md-footer-nav__link md-footer-nav__link--next" href="use_from_pytorch.html" rel="next" title="Using Torch-TensorRT Directly From PyTorch"> <div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title"> <span class="md-flex__ellipsis"> <span class="md-footer-nav__direction"> Next </span> Using Torch-TensorRT Directly From PyTorch </span> </div> <div class="md-flex__cell md-flex__cell--shrink"> <i class="md-icon md-icon--arrow-forward md-footer-nav__button"> </i> </div> </a> </nav> </div> <div class="md-footer-meta md-typeset"> <div class="md-footer-meta__inner md-grid"> <div class="md-footer-copyright"> <div class="md-footer-copyright__highlight"> © Copyright 2021, NVIDIA Corporation. </div> Created using <a href="http://www.sphinx-doc.org/"> Sphinx </a> 3.1.2. and <a href="https://github.com/bashtage/sphinx-material/"> Material for Sphinx </a> </div> </div> </div> </footer> <script src="../_static/javascripts/application.js"> </script> <script> app.initialize({version: "1.0.4", url: {base: ".."}}) </script> </body> </html>