<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />
<title>NuScenes 2D detection — OpenPifPaf Guide</title>
<!-- Loaded before other Sphinx assets -->
<link href="_static/styles/theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link href="_static/styles/pydata-sphinx-theme.css?digest=1999514e3f237ded88cf" rel="stylesheet">
<link rel="stylesheet"
href="_static/vendor/fontawesome/5.13.0/css/all.min.css">
<link rel="preload" as="font" type="font/woff2" crossorigin
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-solid-900.woff2">
<link rel="preload" as="font" type="font/woff2" crossorigin
href="_static/vendor/fontawesome/5.13.0/webfonts/fa-brands-400.woff2">
<link rel="stylesheet" type="text/css" href="_static/pygments.css" />
<link rel="stylesheet" href="_static/styles/sphinx-book-theme.css?digest=5115cc725059bd94278eecd172e13a965bf8f5a9" type="text/css" />
<link rel="stylesheet" type="text/css" href="_static/togglebutton.css" />
<link rel="stylesheet" type="text/css" href="_static/copybutton.css" />
<link rel="stylesheet" type="text/css" href="_static/mystnb.css" />
<link rel="stylesheet" type="text/css" href="_static/sphinx-thebe.css" />
<link rel="stylesheet" type="text/css" href="_static/mystyle.css" />
<link rel="stylesheet" type="text/css" href="_static/design-style.b7bb847fb20b106c3d81b95245e65545.min.css" />
<!-- Pre-loaded scripts that we'll load fully later -->
<link rel="preload" as="script" href="_static/scripts/pydata-sphinx-theme.js?digest=1999514e3f237ded88cf">
<script data-url_root="./" id="documentation_options" src="_static/documentation_options.js"></script>
<script src="_static/jquery.js"></script>
<script src="_static/underscore.js"></script>
<script src="_static/doctools.js"></script>
<script src="_static/clipboard.min.js"></script>
<script src="_static/copybutton.js"></script>
<script src="_static/scripts/sphinx-book-theme.js?digest=9c920249402e914e316237a7dbc6769907cce411"></script>
<script>let toggleHintShow = 'Click to show';</script>
<script>let toggleHintHide = 'Click to hide';</script>
<script>let toggleOpenOnPrint = 'true';</script>
<script src="_static/togglebutton.js"></script>
<script>var togglebuttonSelector = '.toggle, .admonition.dropdown, .tag_hide_input div.cell_input, .tag_hide-input div.cell_input, .tag_hide_output div.cell_output, .tag_hide-output div.cell_output, .tag_hide_cell.cell, .tag_hide-cell.cell';</script>
<script src="_static/design-tabs.js"></script>
<script>const THEBE_JS_URL = "https://unpkg.com/thebe@0.8.2/lib/index.js"
const thebe_selector = ".thebe,.cell"
const thebe_selector_input = "pre"
const thebe_selector_output = ".output, .cell_output"
</script>
<script async="async" src="_static/sphinx-thebe.js"></script>
<link rel="shortcut icon" href="_static/favicon.png"/>
<link rel="index" title="Index" href="genindex.html" />
<link rel="search" title="Search" href="search.html" />
<link rel="next" title="OpenCV" href="tutorial_opencv.html" />
<link rel="prev" title="CrowdPose" href="plugins_crowdpose.html" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="docsearch:language" content="en">
<!-- Google Analytics -->
</head>
<body data-spy="scroll" data-target="#bd-toc-nav" data-offset="60">
<!-- Checkboxes to toggle the left sidebar -->
<input type="checkbox" class="sidebar-toggle" name="__navigation" id="__navigation" aria-label="Toggle navigation sidebar">
<label class="overlay overlay-navbar" for="__navigation">
<div class="visually-hidden">Toggle navigation sidebar</div>
</label>
<!-- Checkboxes to toggle the in-page toc -->
<input type="checkbox" class="sidebar-toggle" name="__page-toc" id="__page-toc" aria-label="Toggle in-page Table of Contents">
<label class="overlay overlay-pagetoc" for="__page-toc">
<div class="visually-hidden">Toggle in-page Table of Contents</div>
</label>
<!-- Headers at the top -->
<div class="announcement header-item noprint"></div>
<div class="header header-item noprint"></div>
<div class="container-fluid" id="banner"></div>
<div class="container-xl">
<div class="row">
<!-- Sidebar -->
<div class="bd-sidebar noprint" id="site-navigation">
<div class="bd-sidebar__content">
<div class="bd-sidebar__top"><div class="navbar-brand-box">
<a class="navbar-brand text-wrap" href="index.html">
<!-- `logo` is deprecated in Sphinx 4.0, so remove this when we stop supporting 3 -->
<img src="_static/logo.png" class="logo" alt="logo">
<h1 class="site-logo" id="site-title">OpenPifPaf Guide</h1>
</a>
</div><form class="bd-search d-flex align-items-center" action="search.html" method="get">
<i class="icon fas fa-search"></i>
<input type="search" class="form-control" name="q" id="search-input" placeholder="Search the docs ..." aria-label="Search the docs ..." autocomplete="off" >
</form><nav class="bd-links" id="bd-docs-nav" aria-label="Main">
<div class="bd-toc-item active">
<ul class="nav bd-sidenav bd-sidenav__home-link">
<li class="toctree-l1">
<a class="reference internal" href="intro.html">
Introduction
</a>
</li>
</ul>
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
Getting Started
</span>
</p>
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference internal" href="predict_cli.html">
Prediction
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="examples.html">
Examples
</a>
</li>
</ul>
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
Chapters
</span>
</p>
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference internal" href="predict_api.html">
Prediction API
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="compute.html">
Compute
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="models.html">
Models
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="datasets.html">
Datasets
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="train.html">
Training
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="cli_help.html">
Command Line
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="plugins_overview.html">
Plugins
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="faq.html">
FAQ
</a>
</li>
</ul>
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
Tutorials
</span>
</p>
<ul class="current nav bd-sidenav">
<li class="toctree-l1">
<a class="reference internal" href="plugins_custom.html">
Custom Dataset
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="plugins_animalpose.html">
Animal Keypoints
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="plugins_wholebody.html">
WholeBody
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="plugins_apollocar3d.html">
Car Keypoints
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="plugins_crowdpose.html">
CrowdPose
</a>
</li>
<li class="toctree-l1 current active">
<a class="current reference internal" href="#">
NuScenes 2D detection
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="tutorial_opencv.html">
OpenCV
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="plugins_cifar10.html">
Cifar10
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="plugins_extras.html">
Extras
</a>
</li>
</ul>
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
Reference
</span>
</p>
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference internal" href="moduledocs.html">
Modules
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="bibliography.html">
Bibliography
</a>
</li>
</ul>
<p aria-level="2" class="caption" role="heading">
<span class="caption-text">
Development
</span>
</p>
<ul class="nav bd-sidenav">
<li class="toctree-l1">
<a class="reference internal" href="dev.html">
Contribute
</a>
</li>
<li class="toctree-l1">
<a class="reference internal" href="performance.html">
Performance
</a>
</li>
<li class="toctree-l1">
<a class="reference external" href="https://github.com/openpifpaf/openpifpaf">
GitHub
</a>
</li>
</ul>
</div>
</nav></div>
<div class="bd-sidebar__bottom">
<!-- To handle the deprecated key -->
</div>
</div>
<div id="rtd-footer-container"></div>
</div>
<!-- A tiny helper pixel to detect if we've scrolled -->
<div class="sbt-scroll-pixel-helper"></div>
<!-- Main content -->
<div class="col py-0 content-container">
<div class="header-article row sticky-top noprint">
<div class="col py-1 d-flex header-article-main">
<div class="header-article__left">
<label for="__navigation"
class="headerbtn"
data-toggle="tooltip"
data-placement="right"
title="Toggle navigation"
>
<span class="headerbtn__icon-container">
<i class="fas fa-bars"></i>
</span>
</label>
</div>
<div class="header-article__right">
<div class="menu-dropdown menu-dropdown-launch-buttons">
<button class="headerbtn menu-dropdown__trigger"
aria-label="Launch interactive content">
<i class="fas fa-rocket"></i>
</button>
<div class="menu-dropdown__content">
<ul>
<li>
<a href="https://mybinder.org/v2/gh/openpifpaf/openpifpaf/stable?urlpath=tree/guide/plugins_nuscenes.ipynb"
class="headerbtn"
data-toggle="tooltip"
data-placement="left"
title="Launch on Binder"
>
<span class="headerbtn__icon-container">
<img src="_static/images/logo_binder.svg">
</span>
<span class="headerbtn__text-container">Binder</span>
</a>
</li>
</ul>
</div>
</div>
<div class="menu-dropdown menu-dropdown-repository-buttons">
<button class="headerbtn menu-dropdown__trigger"
aria-label="Source repositories">
<i class="fab fa-github"></i>
</button>
<div class="menu-dropdown__content">
<ul>
<li>
<a href="https://github.com/openpifpaf/openpifpaf"
class="headerbtn"
data-toggle="tooltip"
data-placement="left"
title="Source repository"
>
<span class="headerbtn__icon-container">
<i class="fab fa-github"></i>
</span>
<span class="headerbtn__text-container">repository</span>
</a>
</li>
<li>
<a href="https://github.com/openpifpaf/openpifpaf/issues/new?title=Issue%20on%20page%20%2Fplugins_nuscenes.html&body=Your%20issue%20content%20here."
class="headerbtn"
data-toggle="tooltip"
data-placement="left"
title="Open an issue"
>
<span class="headerbtn__icon-container">
<i class="fas fa-lightbulb"></i>
</span>
<span class="headerbtn__text-container">open issue</span>
</a>
</li>
<li>
<a href="https://github.com/openpifpaf/openpifpaf/edit/stable/guide/plugins_nuscenes.ipynb"
class="headerbtn"
data-toggle="tooltip"
data-placement="left"
title="Edit this page"
>
<span class="headerbtn__icon-container">
<i class="fas fa-pencil-alt"></i>
</span>
<span class="headerbtn__text-container">suggest edit</span>
</a>
</li>
</ul>
</div>
</div>
<div class="menu-dropdown menu-dropdown-download-buttons">
<button class="headerbtn menu-dropdown__trigger"
aria-label="Download this page">
<i class="fas fa-download"></i>
</button>
<div class="menu-dropdown__content">
<ul>
<li>
<a href="_sources/plugins_nuscenes.ipynb"
class="headerbtn"
data-toggle="tooltip"
data-placement="left"
title="Download source file"
>
<span class="headerbtn__icon-container">
<i class="fas fa-file"></i>
</span>
<span class="headerbtn__text-container">.ipynb</span>
</a>
</li>
<li>
<button onclick="printPdf(this)"
class="headerbtn"
data-toggle="tooltip"
data-placement="left"
title="Print to PDF"
>
<span class="headerbtn__icon-container">
<i class="fas fa-file-pdf"></i>
</span>
<span class="headerbtn__text-container">.pdf</span>
</button>
</li>
</ul>
</div>
</div>
<label for="__page-toc"
class="headerbtn headerbtn-page-toc"
>
<span class="headerbtn__icon-container">
<i class="fas fa-list"></i>
</span>
</label>
</div>
</div>
<!-- Table of contents -->
<div class="col-md-3 bd-toc show noprint">
<div class="tocsection onthispage pt-5 pb-3">
<i class="fas fa-list"></i> On this page
</div>
<nav id="bd-toc-nav" aria-label="Page">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#prediction">
Prediction
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#data-download-and-preprocessing">
Data download and preprocessing
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#train">
Train
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#evaluation">
Evaluation
</a>
</li>
</ul>
</nav>
</div>
</div>
<div class="article row">
<div class="col pl-md-3 pl-lg-5 content-container">
<!-- Table of contents that is only displayed when printing the page -->
<div id="jb-print-docs-body" class="onlyprint">
<h1>NuScenes 2D detection</h1>
<!-- Table of contents -->
<div id="print-main-content">
<div id="jb-print-toc">
<div>
<h2> On this page </h2>
</div>
<nav aria-label="Page">
<ul class="visible nav section-nav flex-column">
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#prediction">
Prediction
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#data-download-and-preprocessing">
Data download and preprocessing
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#train">
Train
</a>
</li>
<li class="toc-h2 nav-item toc-entry">
<a class="reference internal nav-link" href="#evaluation">
Evaluation
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<main id="main-content" role="main">
<div>
<section class="tex2jax_ignore mathjax_ignore" id="nuscenes-2d-detection">
<h1>NuScenes 2D detection<a class="headerlink" href="#nuscenes-2d-detection" title="Permalink to this headline">#</a></h1>
<div style="text-align: right"> by <a href="https://www.linkedin.com/in/duncan-zauss/">Duncan Zauss</a>, 10/08/2021</div> <br />
This is an extension to OpenPifPaf to make 2D detections with the objects from the NuScenes dataset.<section id="prediction">
<h2>Prediction<a class="headerlink" href="#prediction" title="Permalink to this headline">#</a></h2>
<p>We provide a pretrained checkpoint to predict the objects from NuScenes. The required checkpoint will be automatically downloaded if you provide “–checkpoint=shufflenetv2k16-nuscenes” as a CLI argument. Below an example prediction is shown.</p>
<div class="cell docutils container">
<div class="cell_input docutils container">
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">%%</span><span class="k">bash</span>
python -m openpifpaf.predict nuscenes/n013-2018-08-28-10-41-15+0800__CAM_FRONT__1535424448912407.jpg \
--checkpoint=shufflenetv2k16-nuscenes --image-output
</pre></div>
</div>
</div>
<div class="cell_output docutils container">
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>INFO:__main__:neural network device: cpu (CUDA available: False, count: 0)
INFO:openpifpaf.predictor:neural network device: cpu (CUDA available: False, count: 0)
INFO:openpifpaf.decoder.cifdet:annotations 3, decoder = 19.4ms
INFO:openpifpaf.predictor:batch 0: nuscenes/n013-2018-08-28-10-41-15+0800__CAM_FRONT__1535424448912407.jpg
src/openpifpaf/csrc/src/cif_hr.cpp:102: UserInfo: resizing cifhr buffer
src/openpifpaf/csrc/src/occupancy.cpp:53: UserInfo: resizing occupancy buffer
</pre></div>
</div>
</div>
</div>
<div class="cell docutils container">
<div class="cell_input docutils container">
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="n">IPython</span><span class="o">.</span><span class="n">display</span><span class="o">.</span><span class="n">Image</span><span class="p">(</span><span class="s1">'nuscenes/n013-2018-08-28-10-41-15+0800__CAM_FRONT__1535424448912407.jpg.predictions.jpeg'</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="cell_output docutils container">
<img alt="_images/plugins_nuscenes_4_0.jpg" src="_images/plugins_nuscenes_4_0.jpg" />
</div>
</div>
</section>
<section id="data-download-and-preprocessing">
<h2>Data download and preprocessing<a class="headerlink" href="#data-download-and-preprocessing" title="Permalink to this headline">#</a></h2>
<p>If you wish to train or evaluate a model, you will need create a NuScenes account and then download the NuScenes dataset from <a class="reference external" href="https://www.nuscenes.org/download">this link</a>. Only the metadata and the sample images are required, there is no need to download the Sweep images (unannotated previous and following frames).
Subsequently, you need to process the image annotations to MS COCO style annotations with <a class="reference external" href="https://github.com/open-mmlab/mmdetection3d/blob/master/tools/data_converter/nuimage_converter.py">this script</a>.</p>
</section>
<section id="train">
<h2>Train<a class="headerlink" href="#train" title="Permalink to this headline">#</a></h2>
<p>Our model was trained with the following command:<br/></p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">python3</span> <span class="o">-</span><span class="n">m</span> <span class="n">openpifpaf</span><span class="o">.</span><span class="n">train</span> <span class="o">--</span><span class="n">dataset</span><span class="o">=</span><span class="n">nuscenes</span> <span class="o">--</span><span class="n">lr</span><span class="o">=</span><span class="mf">0.0001</span> <span class="o">--</span><span class="n">momentum</span><span class="o">=</span><span class="mf">0.95</span> <span class="o">--</span><span class="n">b</span><span class="o">-</span><span class="n">scale</span><span class="o">=</span><span class="mf">10.0</span> <span class="o">--</span><span class="n">clip</span><span class="o">-</span><span class="n">grad</span><span class="o">-</span><span class="n">value</span><span class="o">=</span><span class="mi">10</span> <span class="o">--</span><span class="n">epochs</span><span class="o">=</span><span class="mi">150</span> <span class="o">--</span><span class="n">lr</span><span class="o">-</span><span class="n">decay</span> <span class="mi">120</span> <span class="mi">140</span> <span class="o">--</span><span class="n">lr</span><span class="o">-</span><span class="n">decay</span><span class="o">-</span><span class="n">epochs</span><span class="o">=</span><span class="mi">10</span> <span class="o">--</span><span class="n">batch</span><span class="o">-</span><span class="n">size</span><span class="o">=</span><span class="mi">32</span> <span class="o">--</span><span class="n">weight</span><span class="o">-</span><span class="n">decay</span><span class="o">=</span><span class="mf">1e-5</span> <span class="o">--</span><span class="n">nuscenes</span><span class="o">-</span><span class="n">upsample</span><span class="o">=</span><span class="mi">2</span> <span class="o">--</span><span class="n">nuscenes</span><span class="o">-</span><span class="n">extended</span><span class="o">-</span><span class="n">scale</span> <span class="o">--</span><span class="n">nuscenes</span><span class="o">-</span><span class="n">orientation</span><span class="o">-</span><span class="n">invariant</span><span class="o">=</span><span class="mf">0.1</span> <span class="o">--</span><span class="n">basenet</span><span class="o">=</span><span class="n">shufflenetv2k16</span>
</pre></div>
</div>
</section>
<section id="evaluation">
<h2>Evaluation<a class="headerlink" href="#evaluation" title="Permalink to this headline">#</a></h2>
<p>To evaluate your network you can use the following command:<br/></p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">python3</span> <span class="o">-</span><span class="n">m</span> <span class="n">openpifpaf</span><span class="o">.</span><span class="n">eval</span> <span class="o">--</span><span class="n">seed</span><span class="o">-</span><span class="n">threshold</span><span class="o">=</span><span class="mf">0.3</span> <span class="o">--</span><span class="n">instance</span><span class="o">-</span><span class="n">threshold</span><span class="o">=</span><span class="mf">0.3</span> <span class="o">--</span><span class="n">dataset</span><span class="o">=</span><span class="n">nuscenes</span> <span class="o">--</span><span class="n">checkpoint</span><span class="o">=</span><span class="n">shufflenetv2k16</span><span class="o">-</span><span class="n">nuscenes</span>
</pre></div>
</div>
</section>
</section>
<script type="text/x-thebe-config">
{
requestKernel: true,
binderOptions: {
repo: "binder-examples/jupyter-stacks-datascience",
ref: "master",
},
codeMirrorConfig: {
theme: "abcdef",
mode: "python"
},
kernelOptions: {
kernelName: "python3",
path: "./."
},
predefinedOutput: true
}
</script>
<script>kernelName = 'python3'</script>
</div>
</main>
<footer class="footer-article noprint">
<!-- Previous / next buttons -->
<div class='prev-next-area'>
<a class='left-prev' id="prev-link" href="plugins_crowdpose.html" title="previous page">
<i class="fas fa-angle-left"></i>
<div class="prev-next-info">
<p class="prev-next-subtitle">previous</p>
<p class="prev-next-title">CrowdPose</p>
</div>
</a>
<a class='right-next' id="next-link" href="tutorial_opencv.html" title="next page">
<div class="prev-next-info">
<p class="prev-next-subtitle">next</p>
<p class="prev-next-title">OpenCV</p>
</div>
<i class="fas fa-angle-right"></i>
</a>
</div>
</footer>
</div>
</div>
<div class="footer-content row">
<footer class="col footer"><p>
By <a href="https://www.svenkreiss.com">Sven Kreiss</a> and <a href="https://github.com/openpifpaf/openpifpaf/graphs/contributors">contributors</a>.<br/>
© Copyright 2020-2022.<br/>
<div class="extra_footer">
<p>Powered by <a href="https://jupyterbook.org/">Jupyter Book</a>.</p>
</div>
</p>
</footer>
</div>
</div>
</div>
</div>
<!-- Scripts loaded after <body> so the DOM is not blocked -->
<script src="_static/scripts/pydata-sphinx-theme.js?digest=1999514e3f237ded88cf"></script>
</body>
</html>