<!-- HTML header for doxygen 1.8.14-->
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=9"/>
<meta name="generator" content="Doxygen 1.8.11"/>
<meta name="viewport" content="width=device-width, initial-scale=1"/>
<title>Caffe2 - Python API: caffe2/python/dataio.py Source File</title>
<link href="tabs.css" rel="stylesheet" type="text/css"/>
<link rel="icon" href="/static/favicon.png" type="image/x-icon">
<script type="text/javascript" src="jquery.js"></script>
<script type="text/javascript" src="dynsections.js"></script>
<link href="search/search.css" rel="stylesheet" type="text/css"/>
<script type="text/javascript" src="search/searchdata.js"></script>
<script type="text/javascript" src="search/search.js"></script>
<script type="text/javascript">
$(document).ready(function() { init_search(); });
</script>
<link href="stylesheet.css" rel="stylesheet" type="text/css" />
<link href="main.css" rel="stylesheet" type="text/css"/>
</head>
<body>
<div id="top"><!-- do not remove this div, it is closed by doxygen! -->
<div id="titlearea">
<table cellspacing="0" cellpadding="0">
<tbody>
<tr style="height: 56px;">
<td id="projectlogo" width="56"><a href="/"><img alt="Logo" src="Caffe2-with-name-55-tall.png"/></a></td>
<td id="projectalign" style="padding-left: 0.5em;">
<div id="projectname">Caffe2 - Python API
</div>
<div id="projectbrief">A deep learning, cross platform ML framework</div>
</td>
</tr>
</tbody>
</table>
</div>
<!-- end header part -->
<!-- Generated by Doxygen 1.8.11 -->
<script type="text/javascript">
var searchBox = new SearchBox("searchBox", "search",false,'Search');
</script>
<div id="navrow1" class="tabs">
<ul class="tablist">
<li><a href="namespaces.html"><span>Packages</span></a></li>
<li><a href="annotated.html"><span>Classes</span></a></li>
<li class="current"><a href="files.html"><span>Files</span></a></li>
<li><a href="/doxygen-c/html/classes.html"><span>C++ API</span></a></li>
<li><a href="/doxygen-python/html/annotated.html"><span>Python API</span></a></li>
<li><a href="https://github.com/caffe2/caffe2"><span>GitHub</span></a></li>
<li>
<div id="MSearchBox" class="MSearchBoxInactive">
<span class="left">
<img id="MSearchSelect" src="search/mag_sel.png"
onmouseover="return searchBox.OnSearchSelectShow()"
onmouseout="return searchBox.OnSearchSelectHide()"
alt=""/>
<input type="text" id="MSearchField" value="Search" accesskey="S"
onfocus="searchBox.OnSearchFieldFocus(true)"
onblur="searchBox.OnSearchFieldFocus(false)"
onkeyup="searchBox.OnSearchFieldChange(event)"/>
</span><span class="right">
<a id="MSearchClose" href="javascript:searchBox.CloseResultsWindow()"><img id="MSearchCloseImg" border="0" src="search/close.png" alt=""/></a>
</span>
</div>
</li>
</ul>
</div>
<div id="navrow2" class="tabs2">
<ul class="tablist">
<li><a href="files.html"><span>File List</span></a></li>
</ul>
</div>
<!-- window showing the filter options -->
<div id="MSearchSelectWindow"
onmouseover="return searchBox.OnSearchSelectShow()"
onmouseout="return searchBox.OnSearchSelectHide()"
onkeydown="return searchBox.OnSearchSelectKey(event)">
</div>
<!-- iframe showing the search results (closed by default) -->
<div id="MSearchResultsWindow">
<iframe src="javascript:void(0)" frameborder="0"
name="MSearchResults" id="MSearchResults">
</iframe>
</div>
<div id="nav-path" class="navpath">
<ul>
<li class="navelem"><a class="el" href="dir_20697b8f204bdfcab31e6b1a416f3ab8.html">caffe2</a></li><li class="navelem"><a class="el" href="dir_f4fda25fc5253eea1ed54677ae5fa2de.html">python</a></li> </ul>
</div>
</div><!-- top -->
<div class="header">
<div class="headertitle">
<div class="title">dataio.py</div> </div>
</div><!--header-->
<div class="contents">
<div class="fragment"><div class="line"><a name="l00001"></a><span class="lineno"> 1</span> <span class="comment">## @package dataio</span></div><div class="line"><a name="l00002"></a><span class="lineno"> 2</span> <span class="comment"># Module caffe2.python.dataio</span></div><div class="line"><a name="l00003"></a><span class="lineno"> 3</span> <span class="stringliteral">"""</span></div><div class="line"><a name="l00004"></a><span class="lineno"> 4</span> <span class="stringliteral">Defines the base interface for reading and writing operations.</span></div><div class="line"><a name="l00005"></a><span class="lineno"> 5</span> <span class="stringliteral"></span></div><div class="line"><a name="l00006"></a><span class="lineno"> 6</span> <span class="stringliteral">Readers/Writers are objects that produce operations that read/write sequences</span></div><div class="line"><a name="l00007"></a><span class="lineno"> 7</span> <span class="stringliteral">of data. Each operation reads or writes a list of BlobReferences.</span></div><div class="line"><a name="l00008"></a><span class="lineno"> 8</span> <span class="stringliteral"></span></div><div class="line"><a name="l00009"></a><span class="lineno"> 9</span> <span class="stringliteral">Readers and Writers must be implemented such that read and write operations</span></div><div class="line"><a name="l00010"></a><span class="lineno"> 10</span> <span class="stringliteral">are atomic and thread safe.</span></div><div class="line"><a name="l00011"></a><span class="lineno"> 11</span> <span class="stringliteral"></span></div><div class="line"><a name="l00012"></a><span class="lineno"> 12</span> <span class="stringliteral">Examples of possible Readers and Writers:</span></div><div class="line"><a name="l00013"></a><span class="lineno"> 13</span> <span class="stringliteral"> QueueReader, QueueWriter,</span></div><div class="line"><a name="l00014"></a><span class="lineno"> 14</span> <span class="stringliteral"> DatasetReader, DatasetWriter,</span></div><div class="line"><a name="l00015"></a><span class="lineno"> 15</span> <span class="stringliteral"></span></div><div class="line"><a name="l00016"></a><span class="lineno"> 16</span> <span class="stringliteral">See `dataset.py` for an example of implementation.</span></div><div class="line"><a name="l00017"></a><span class="lineno"> 17</span> <span class="stringliteral">"""</span></div><div class="line"><a name="l00018"></a><span class="lineno"> 18</span> <span class="keyword">from</span> __future__ <span class="keyword">import</span> absolute_import</div><div class="line"><a name="l00019"></a><span class="lineno"> 19</span> <span class="keyword">from</span> __future__ <span class="keyword">import</span> division</div><div class="line"><a name="l00020"></a><span class="lineno"> 20</span> <span class="keyword">from</span> __future__ <span class="keyword">import</span> print_function</div><div class="line"><a name="l00021"></a><span class="lineno"> 21</span> <span class="keyword">from</span> __future__ <span class="keyword">import</span> unicode_literals</div><div class="line"><a name="l00022"></a><span class="lineno"> 22</span> </div><div class="line"><a name="l00023"></a><span class="lineno"> 23</span> <span class="keyword">from</span> <a class="code" href="namespacecaffe2_1_1python.html">caffe2.python</a> <span class="keyword">import</span> core</div><div class="line"><a name="l00024"></a><span class="lineno"> 24</span> <span class="keyword">from</span> <a class="code" href="namespacecaffe2_1_1python_1_1schema.html">caffe2.python.schema</a> <span class="keyword">import</span> Field, Struct, from_blob_list</div><div class="line"><a name="l00025"></a><span class="lineno"> 25</span> <span class="keyword">import</span> numpy <span class="keyword">as</span> np</div><div class="line"><a name="l00026"></a><span class="lineno"> 26</span> </div><div class="line"><a name="l00027"></a><span class="lineno"> 27</span> </div><div class="line"><a name="l00028"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html"> 28</a></span> <span class="keyword">class </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html">Reader</a>(object):</div><div class="line"><a name="l00029"></a><span class="lineno"> 29</span>  <span class="stringliteral">"""</span></div><div class="line"><a name="l00030"></a><span class="lineno"> 30</span> <span class="stringliteral"> Reader is an abstract class to be implemented in order to provide</span></div><div class="line"><a name="l00031"></a><span class="lineno"> 31</span> <span class="stringliteral"> operations capable of iterating through a dataset or stream of data.</span></div><div class="line"><a name="l00032"></a><span class="lineno"> 32</span> <span class="stringliteral"></span></div><div class="line"><a name="l00033"></a><span class="lineno"> 33</span> <span class="stringliteral"> A Reader must implement at least one operation, `read`, which</span></div><div class="line"><a name="l00034"></a><span class="lineno"> 34</span> <span class="stringliteral"> adds operations to a net that read the next batch of data. Readers can</span></div><div class="line"><a name="l00035"></a><span class="lineno"> 35</span> <span class="stringliteral"> optionally support the `reset` operation, which is useful when multiple</span></div><div class="line"><a name="l00036"></a><span class="lineno"> 36</span> <span class="stringliteral"> passes over the data are required.</span></div><div class="line"><a name="l00037"></a><span class="lineno"> 37</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00038"></a><span class="lineno"> 38</span>  <span class="keyword">def </span>__init__(self, schema=None):</div><div class="line"><a name="l00039"></a><span class="lineno"> 39</span>  <span class="keywordflow">if</span> schema <span class="keywordflow">is</span> <span class="keywordflow">not</span> <span class="keywordtype">None</span>:</div><div class="line"><a name="l00040"></a><span class="lineno"> 40</span>  <span class="keyword">assert</span> isinstance(schema, Field)</div><div class="line"><a name="l00041"></a><span class="lineno"> 41</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a67e4d228c8017789e3331bff2b5d2537">_schema</a> = schema</div><div class="line"><a name="l00042"></a><span class="lineno"> 42</span> </div><div class="line"><a name="l00043"></a><span class="lineno"> 43</span>  <span class="keyword">def </span><a class="code" href="namespaceschema.html">schema</a>(self):</div><div class="line"><a name="l00044"></a><span class="lineno"> 44</span>  <span class="keyword">assert</span> self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a67e4d228c8017789e3331bff2b5d2537">_schema</a> <span class="keywordflow">is</span> <span class="keywordflow">not</span> <span class="keywordtype">None</span>, <span class="stringliteral">'Schema not provided for this reader.'</span></div><div class="line"><a name="l00045"></a><span class="lineno"> 45</span>  <span class="keywordflow">return</span> self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a67e4d228c8017789e3331bff2b5d2537">_schema</a></div><div class="line"><a name="l00046"></a><span class="lineno"> 46</span> </div><div class="line"><a name="l00047"></a><span class="lineno"> 47</span>  <span class="keyword">def </span>_set_schema(self, schema):</div><div class="line"><a name="l00048"></a><span class="lineno"> 48</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a67e4d228c8017789e3331bff2b5d2537">_schema</a> = schema</div><div class="line"><a name="l00049"></a><span class="lineno"> 49</span> </div><div class="line"><a name="l00050"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a8d8909ca962957a5eb60f9d4007ef745"> 50</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a8d8909ca962957a5eb60f9d4007ef745">setup_ex</a>(self, init_net, finish_net):</div><div class="line"><a name="l00051"></a><span class="lineno"> 51</span>  <span class="stringliteral">"""Setup nets to run at task initialization and cleanup time.</span></div><div class="line"><a name="l00052"></a><span class="lineno"> 52</span> <span class="stringliteral"></span></div><div class="line"><a name="l00053"></a><span class="lineno"> 53</span> <span class="stringliteral"> Args:</span></div><div class="line"><a name="l00054"></a><span class="lineno"> 54</span> <span class="stringliteral"> global_init_net: A net invoked at task init time.</span></div><div class="line"><a name="l00055"></a><span class="lineno"> 55</span> <span class="stringliteral"> global_finish_net: A net invoked at task cleanup time.</span></div><div class="line"><a name="l00056"></a><span class="lineno"> 56</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00057"></a><span class="lineno"> 57</span>  <span class="keywordflow">pass</span></div><div class="line"><a name="l00058"></a><span class="lineno"> 58</span> </div><div class="line"><a name="l00059"></a><span class="lineno"> 59</span>  <span class="keyword">def </span>read_ex(self, local_init_net, local_finish_net):</div><div class="line"><a name="l00060"></a><span class="lineno"> 60</span>  read_net = core.Net(<span class="stringliteral">'reader_body'</span>)</div><div class="line"><a name="l00061"></a><span class="lineno"> 61</span>  <span class="keywordflow">return</span> ([read_net], ) + self.read(read_net)</div><div class="line"><a name="l00062"></a><span class="lineno"> 62</span> </div><div class="line"><a name="l00063"></a><span class="lineno"> 63</span>  <span class="keyword">def </span>read_record_ex(self, local_init_net, local_finish_net):</div><div class="line"><a name="l00064"></a><span class="lineno"> 64</span>  nets, should_stop, fields = self.read_ex(</div><div class="line"><a name="l00065"></a><span class="lineno"> 65</span>  local_init_net, local_finish_net)</div><div class="line"><a name="l00066"></a><span class="lineno"> 66</span>  <span class="keywordflow">if</span> self._schema:</div><div class="line"><a name="l00067"></a><span class="lineno"> 67</span>  fields = from_blob_list(self._schema, fields)</div><div class="line"><a name="l00068"></a><span class="lineno"> 68</span>  <span class="keywordflow">return</span> nets, should_stop, fields</div><div class="line"><a name="l00069"></a><span class="lineno"> 69</span> </div><div class="line"><a name="l00070"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a024602c0217e1c49b436d5d592a9ff70"> 70</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a024602c0217e1c49b436d5d592a9ff70">read</a>(self, read_net):</div><div class="line"><a name="l00071"></a><span class="lineno"> 71</span>  <span class="stringliteral">"""Append operations to read_net that will read a batch from the</span></div><div class="line"><a name="l00072"></a><span class="lineno"> 72</span> <span class="stringliteral"> underlying data soruce.</span></div><div class="line"><a name="l00073"></a><span class="lineno"> 73</span> <span class="stringliteral"></span></div><div class="line"><a name="l00074"></a><span class="lineno"> 74</span> <span class="stringliteral"> Operations added to `read_net` must be thread safe and atomic, that is,</span></div><div class="line"><a name="l00075"></a><span class="lineno"> 75</span> <span class="stringliteral"> it should be possible to clone `read_net` and run multiple instances of</span></div><div class="line"><a name="l00076"></a><span class="lineno"> 76</span> <span class="stringliteral"> it in parallel.</span></div><div class="line"><a name="l00077"></a><span class="lineno"> 77</span> <span class="stringliteral"></span></div><div class="line"><a name="l00078"></a><span class="lineno"> 78</span> <span class="stringliteral"> Args:</span></div><div class="line"><a name="l00079"></a><span class="lineno"> 79</span> <span class="stringliteral"> read_net: the net that will be appended with read operations</span></div><div class="line"><a name="l00080"></a><span class="lineno"> 80</span> <span class="stringliteral"></span></div><div class="line"><a name="l00081"></a><span class="lineno"> 81</span> <span class="stringliteral"> Returns:</span></div><div class="line"><a name="l00082"></a><span class="lineno"> 82</span> <span class="stringliteral"> A tuple (should_stop, fields), with:</span></div><div class="line"><a name="l00083"></a><span class="lineno"> 83</span> <span class="stringliteral"> should_stop: BlobReference pointing to a boolean scalar</span></div><div class="line"><a name="l00084"></a><span class="lineno"> 84</span> <span class="stringliteral"> blob that indicates whether the read operation</span></div><div class="line"><a name="l00085"></a><span class="lineno"> 85</span> <span class="stringliteral"> was succesfull or whether the end of data has</span></div><div class="line"><a name="l00086"></a><span class="lineno"> 86</span> <span class="stringliteral"> been reached.</span></div><div class="line"><a name="l00087"></a><span class="lineno"> 87</span> <span class="stringliteral"> fields: A tuple of BlobReference containing the latest batch</span></div><div class="line"><a name="l00088"></a><span class="lineno"> 88</span> <span class="stringliteral"> of data that was read.</span></div><div class="line"><a name="l00089"></a><span class="lineno"> 89</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00090"></a><span class="lineno"> 90</span>  <span class="keywordflow">raise</span> NotImplementedError(<span class="stringliteral">'Readers must implement `read`.'</span>)</div><div class="line"><a name="l00091"></a><span class="lineno"> 91</span> </div><div class="line"><a name="l00092"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a3e24214ecea0850be812df3296b3796b"> 92</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a3e24214ecea0850be812df3296b3796b">reset</a>(self, net):</div><div class="line"><a name="l00093"></a><span class="lineno"> 93</span>  <span class="stringliteral">"""Append operations to `net` that will reset the reader.</span></div><div class="line"><a name="l00094"></a><span class="lineno"> 94</span> <span class="stringliteral"></span></div><div class="line"><a name="l00095"></a><span class="lineno"> 95</span> <span class="stringliteral"> This can be used to read the data multiple times.</span></div><div class="line"><a name="l00096"></a><span class="lineno"> 96</span> <span class="stringliteral"> Not all readers support this operation.</span></div><div class="line"><a name="l00097"></a><span class="lineno"> 97</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00098"></a><span class="lineno"> 98</span>  <span class="keywordflow">raise</span> NotImplementedError(<span class="stringliteral">'This reader cannot be resetted.'</span>)</div><div class="line"><a name="l00099"></a><span class="lineno"> 99</span> </div><div class="line"><a name="l00100"></a><span class="lineno"> 100</span>  <span class="keyword">def </span>read_record(self, read_net):</div><div class="line"><a name="l00101"></a><span class="lineno"> 101</span>  should_stop, fields = self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a024602c0217e1c49b436d5d592a9ff70">read</a>(read_net)</div><div class="line"><a name="l00102"></a><span class="lineno"> 102</span>  <span class="keywordflow">if</span> self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a67e4d228c8017789e3331bff2b5d2537">_schema</a>:</div><div class="line"><a name="l00103"></a><span class="lineno"> 103</span>  fields = from_blob_list(self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a67e4d228c8017789e3331bff2b5d2537">_schema</a>, fields)</div><div class="line"><a name="l00104"></a><span class="lineno"> 104</span>  <span class="keywordflow">return</span> should_stop, fields</div><div class="line"><a name="l00105"></a><span class="lineno"> 105</span> </div><div class="line"><a name="l00106"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#ad6c5e15f2482793bebac92089d050bbc"> 106</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#ad6c5e15f2482793bebac92089d050bbc">execution_step</a>(self, reader_net_name=None, external_should_stop=None):</div><div class="line"><a name="l00107"></a><span class="lineno"> 107</span>  <span class="stringliteral">"""Create an execution step with a net containing read operators.</span></div><div class="line"><a name="l00108"></a><span class="lineno"> 108</span> <span class="stringliteral"></span></div><div class="line"><a name="l00109"></a><span class="lineno"> 109</span> <span class="stringliteral"> The execution step will contain a `stop_blob` that knows how to stop</span></div><div class="line"><a name="l00110"></a><span class="lineno"> 110</span> <span class="stringliteral"> the execution loop when end of data was reached.</span></div><div class="line"><a name="l00111"></a><span class="lineno"> 111</span> <span class="stringliteral"></span></div><div class="line"><a name="l00112"></a><span class="lineno"> 112</span> <span class="stringliteral"> E.g.:</span></div><div class="line"><a name="l00113"></a><span class="lineno"> 113</span> <span class="stringliteral"></span></div><div class="line"><a name="l00114"></a><span class="lineno"> 114</span> <span class="stringliteral"> read_step, fields = reader.execution_step()</span></div><div class="line"><a name="l00115"></a><span class="lineno"> 115</span> <span class="stringliteral"> consume_net = core.Net('consume')</span></div><div class="line"><a name="l00116"></a><span class="lineno"> 116</span> <span class="stringliteral"> consume_net.Print(fields[0], [])</span></div><div class="line"><a name="l00117"></a><span class="lineno"> 117</span> <span class="stringliteral"> p = core.Plan('reader')</span></div><div class="line"><a name="l00118"></a><span class="lineno"> 118</span> <span class="stringliteral"> p.AddStep(read_step.AddNet(consume_net))</span></div><div class="line"><a name="l00119"></a><span class="lineno"> 119</span> <span class="stringliteral"> core.RunPlan(p)</span></div><div class="line"><a name="l00120"></a><span class="lineno"> 120</span> <span class="stringliteral"></span></div><div class="line"><a name="l00121"></a><span class="lineno"> 121</span> <span class="stringliteral"> Args:</span></div><div class="line"><a name="l00122"></a><span class="lineno"> 122</span> <span class="stringliteral"> reader_net_name: (optional) the name of the reader_net to be</span></div><div class="line"><a name="l00123"></a><span class="lineno"> 123</span> <span class="stringliteral"> created. The execution step will</span></div><div class="line"><a name="l00124"></a><span class="lineno"> 124</span> <span class="stringliteral"> be named accordingly.</span></div><div class="line"><a name="l00125"></a><span class="lineno"> 125</span> <span class="stringliteral"></span></div><div class="line"><a name="l00126"></a><span class="lineno"> 126</span> <span class="stringliteral"> Returns:</span></div><div class="line"><a name="l00127"></a><span class="lineno"> 127</span> <span class="stringliteral"> A tuple (read_step, fields), with:</span></div><div class="line"><a name="l00128"></a><span class="lineno"> 128</span> <span class="stringliteral"> read_step: A newly created execution step containing a net with</span></div><div class="line"><a name="l00129"></a><span class="lineno"> 129</span> <span class="stringliteral"> read operations. The step will have `stop_blob` set,</span></div><div class="line"><a name="l00130"></a><span class="lineno"> 130</span> <span class="stringliteral"> in order to stop the loop on end of data.</span></div><div class="line"><a name="l00131"></a><span class="lineno"> 131</span> <span class="stringliteral"> fields: A tuple of BlobReference containing the latest batch</span></div><div class="line"><a name="l00132"></a><span class="lineno"> 132</span> <span class="stringliteral"> of data that was read.</span></div><div class="line"><a name="l00133"></a><span class="lineno"> 133</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00134"></a><span class="lineno"> 134</span>  reader_net = <a class="code" href="classcaffe2_1_1python_1_1core_1_1_net.html">core.Net</a>(reader_net_name <span class="keywordflow">or</span> <span class="stringliteral">'reader'</span>)</div><div class="line"><a name="l00135"></a><span class="lineno"> 135</span>  should_stop, fields = self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a272213d199828188fff4e1f006285372">read_record</a>(reader_net)</div><div class="line"><a name="l00136"></a><span class="lineno"> 136</span>  <span class="keywordflow">if</span> external_should_stop <span class="keywordflow">is</span> <span class="keywordflow">not</span> <span class="keywordtype">None</span>:</div><div class="line"><a name="l00137"></a><span class="lineno"> 137</span>  should_stop = reader_net.Or([external_should_stop, should_stop])</div><div class="line"><a name="l00138"></a><span class="lineno"> 138</span>  read_step = core.execution_step(</div><div class="line"><a name="l00139"></a><span class="lineno"> 139</span>  <span class="stringliteral">'{}_step'</span>.format(reader_net_name),</div><div class="line"><a name="l00140"></a><span class="lineno"> 140</span>  reader_net,</div><div class="line"><a name="l00141"></a><span class="lineno"> 141</span>  should_stop_blob=should_stop)</div><div class="line"><a name="l00142"></a><span class="lineno"> 142</span>  <span class="keywordflow">return</span> (read_step, fields)</div><div class="line"><a name="l00143"></a><span class="lineno"> 143</span> </div><div class="line"><a name="l00144"></a><span class="lineno"> 144</span> </div><div class="line"><a name="l00145"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html"> 145</a></span> <span class="keyword">class </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html">Writer</a>(object):</div><div class="line"><a name="l00146"></a><span class="lineno"> 146</span>  <span class="stringliteral">"""</span></div><div class="line"><a name="l00147"></a><span class="lineno"> 147</span> <span class="stringliteral"> Writer is an abstract class to be implemented in order to provide</span></div><div class="line"><a name="l00148"></a><span class="lineno"> 148</span> <span class="stringliteral"> operations capable of feeding a data stream or a dataset.</span></div><div class="line"><a name="l00149"></a><span class="lineno"> 149</span> <span class="stringliteral"></span></div><div class="line"><a name="l00150"></a><span class="lineno"> 150</span> <span class="stringliteral"> A Writer must implement 2 operations:</span></div><div class="line"><a name="l00151"></a><span class="lineno"> 151</span> <span class="stringliteral"> `write`, which adds operations to a net that write the write batch of</span></div><div class="line"><a name="l00152"></a><span class="lineno"> 152</span> <span class="stringliteral"> data, and `commit`, which adds operations to a net in order to indicate</span></div><div class="line"><a name="l00153"></a><span class="lineno"> 153</span> <span class="stringliteral"> that no more data will be written.</span></div><div class="line"><a name="l00154"></a><span class="lineno"> 154</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00155"></a><span class="lineno"> 155</span>  _schema = <span class="keywordtype">None</span></div><div class="line"><a name="l00156"></a><span class="lineno"> 156</span> </div><div class="line"><a name="l00157"></a><span class="lineno"> 157</span>  <span class="keyword">def </span><a class="code" href="namespaceschema.html">schema</a>(self):</div><div class="line"><a name="l00158"></a><span class="lineno"> 158</span>  <span class="keywordflow">return</span> self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a9d70c665219c46da12b0cfafdae8060a">_schema</a></div><div class="line"><a name="l00159"></a><span class="lineno"> 159</span> </div><div class="line"><a name="l00160"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a77a746b74ca09dd9aee005330cc6916d"> 160</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a77a746b74ca09dd9aee005330cc6916d">write</a>(self, writer_net, fields):</div><div class="line"><a name="l00161"></a><span class="lineno"> 161</span>  <span class="stringliteral">"""Add operations to `writer_net` that write the next batch of data.</span></div><div class="line"><a name="l00162"></a><span class="lineno"> 162</span> <span class="stringliteral"></span></div><div class="line"><a name="l00163"></a><span class="lineno"> 163</span> <span class="stringliteral"> Operations added to the net must be thread-safe and unique, that is:</span></div><div class="line"><a name="l00164"></a><span class="lineno"> 164</span> <span class="stringliteral"> multiple writers must be able to write to the dataset in parallel.</span></div><div class="line"><a name="l00165"></a><span class="lineno"> 165</span> <span class="stringliteral"></span></div><div class="line"><a name="l00166"></a><span class="lineno"> 166</span> <span class="stringliteral"> Args:</span></div><div class="line"><a name="l00167"></a><span class="lineno"> 167</span> <span class="stringliteral"> fields: a tuple of BlobReference containing the batch of data to</span></div><div class="line"><a name="l00168"></a><span class="lineno"> 168</span> <span class="stringliteral"> write.</span></div><div class="line"><a name="l00169"></a><span class="lineno"> 169</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00170"></a><span class="lineno"> 170</span>  <span class="keywordflow">raise</span> NotImplementedError(<span class="stringliteral">'Writers must implement write.'</span>)</div><div class="line"><a name="l00171"></a><span class="lineno"> 171</span> </div><div class="line"><a name="l00172"></a><span class="lineno"> 172</span>  <span class="keyword">def </span>write_record(self, writer_net, fields):</div><div class="line"><a name="l00173"></a><span class="lineno"> 173</span>  <span class="keywordflow">if</span> isinstance(fields, Field):</div><div class="line"><a name="l00174"></a><span class="lineno"> 174</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a9d70c665219c46da12b0cfafdae8060a">_schema</a> = fields</div><div class="line"><a name="l00175"></a><span class="lineno"> 175</span>  fields = fields.field_blobs()</div><div class="line"><a name="l00176"></a><span class="lineno"> 176</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a77a746b74ca09dd9aee005330cc6916d">write</a>(writer_net, fields)</div><div class="line"><a name="l00177"></a><span class="lineno"> 177</span> </div><div class="line"><a name="l00178"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a3e270a50d9e2d65db196cddc56132c00"> 178</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a3e270a50d9e2d65db196cddc56132c00">setup_ex</a>(self, init_net, finish_net):</div><div class="line"><a name="l00179"></a><span class="lineno"> 179</span>  <span class="stringliteral">"""Experimental, don't use yet"""</span></div><div class="line"><a name="l00180"></a><span class="lineno"> 180</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a7fb018f843ba304edbd23f0e82328ece">commit</a>(finish_net)</div><div class="line"><a name="l00181"></a><span class="lineno"> 181</span> </div><div class="line"><a name="l00182"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a8c0252988fae6e79ed51fa5407e244f6"> 182</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a8c0252988fae6e79ed51fa5407e244f6">write_ex</a>(self, fields, local_init_net, local_finish_net, stop_blob):</div><div class="line"><a name="l00183"></a><span class="lineno"> 183</span>  <span class="stringliteral">"""Experimental extension to the interface. Don't use yet"""</span></div><div class="line"><a name="l00184"></a><span class="lineno"> 184</span>  write_net = <a class="code" href="classcaffe2_1_1python_1_1core_1_1_net.html">core.Net</a>(<span class="stringliteral">'write_net'</span>)</div><div class="line"><a name="l00185"></a><span class="lineno"> 185</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a77a746b74ca09dd9aee005330cc6916d">write</a>(write_net, fields)</div><div class="line"><a name="l00186"></a><span class="lineno"> 186</span>  <span class="keywordflow">return</span> [write_net]</div><div class="line"><a name="l00187"></a><span class="lineno"> 187</span> </div><div class="line"><a name="l00188"></a><span class="lineno"> 188</span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a55b403eba95e44ab19e68dd164a9aaf1">write_record_ex</a>(</div><div class="line"><a name="l00189"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a55b403eba95e44ab19e68dd164a9aaf1"> 189</a></span>  self, fields, local_init_net, local_finish_net, stop_blob=<span class="keywordtype">None</span>):</div><div class="line"><a name="l00190"></a><span class="lineno"> 190</span>  <span class="stringliteral">"""Experimental extension to the interface. Don't use yet."""</span></div><div class="line"><a name="l00191"></a><span class="lineno"> 191</span>  <span class="keywordflow">if</span> isinstance(fields, Field):</div><div class="line"><a name="l00192"></a><span class="lineno"> 192</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a9d70c665219c46da12b0cfafdae8060a">_schema</a> = fields</div><div class="line"><a name="l00193"></a><span class="lineno"> 193</span>  fields = fields.field_blobs()</div><div class="line"><a name="l00194"></a><span class="lineno"> 194</span>  <span class="keywordflow">if</span> stop_blob <span class="keywordflow">is</span> <span class="keywordtype">None</span>:</div><div class="line"><a name="l00195"></a><span class="lineno"> 195</span>  stop_blob = local_init_net.NextName(<span class="stringliteral">"dequeue_status"</span>)</div><div class="line"><a name="l00196"></a><span class="lineno"> 196</span>  write_nets = self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a8c0252988fae6e79ed51fa5407e244f6">write_ex</a>(</div><div class="line"><a name="l00197"></a><span class="lineno"> 197</span>  fields, local_init_net, local_finish_net, stop_blob)</div><div class="line"><a name="l00198"></a><span class="lineno"> 198</span>  <span class="keywordflow">return</span> (write_nets, stop_blob)</div><div class="line"><a name="l00199"></a><span class="lineno"> 199</span> </div><div class="line"><a name="l00200"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a7fb018f843ba304edbd23f0e82328ece"> 200</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a7fb018f843ba304edbd23f0e82328ece">commit</a>(self, finish_net):</div><div class="line"><a name="l00201"></a><span class="lineno"> 201</span>  <span class="stringliteral">"""Add operations to `finish_net` that signal end of data.</span></div><div class="line"><a name="l00202"></a><span class="lineno"> 202</span> <span class="stringliteral"></span></div><div class="line"><a name="l00203"></a><span class="lineno"> 203</span> <span class="stringliteral"> This must be implemented by all Writers, but may be no-op for some</span></div><div class="line"><a name="l00204"></a><span class="lineno"> 204</span> <span class="stringliteral"> of them.</span></div><div class="line"><a name="l00205"></a><span class="lineno"> 205</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00206"></a><span class="lineno"> 206</span>  <span class="keywordflow">pass</span></div><div class="line"><a name="l00207"></a><span class="lineno"> 207</span> </div><div class="line"><a name="l00208"></a><span class="lineno"> 208</span> </div><div class="line"><a name="l00209"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader_builder.html"> 209</a></span> <span class="keyword">class </span>ReaderBuilder(object):</div><div class="line"><a name="l00210"></a><span class="lineno"> 210</span>  <span class="stringliteral">""" Allow usage of a reader in distributed fashion. """</span></div><div class="line"><a name="l00211"></a><span class="lineno"> 211</span>  <span class="keyword">def </span><a class="code" href="namespaceschema.html">schema</a>(self):</div><div class="line"><a name="l00212"></a><span class="lineno"> 212</span>  <span class="keywordflow">raise</span> NotImplementedError()</div><div class="line"><a name="l00213"></a><span class="lineno"> 213</span> </div><div class="line"><a name="l00214"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader_builder.html#a45191bc578de11a88715b49faec44d03"> 214</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_builder.html#a45191bc578de11a88715b49faec44d03">setup</a>(self, **kwargs):</div><div class="line"><a name="l00215"></a><span class="lineno"> 215</span>  <span class="stringliteral">"""</span></div><div class="line"><a name="l00216"></a><span class="lineno"> 216</span> <span class="stringliteral"> Optionally, perform one-time setup before calling new_reader().</span></div><div class="line"><a name="l00217"></a><span class="lineno"> 217</span> <span class="stringliteral"> Subclass should make sure this function is only called once.</span></div><div class="line"><a name="l00218"></a><span class="lineno"> 218</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00219"></a><span class="lineno"> 219</span>  <span class="keywordflow">raise</span> NotImplementedError()</div><div class="line"><a name="l00220"></a><span class="lineno"> 220</span> </div><div class="line"><a name="l00221"></a><span class="lineno"> 221</span>  <span class="keyword">def </span>new_reader(self, **kwargs):</div><div class="line"><a name="l00222"></a><span class="lineno"> 222</span>  <span class="keywordflow">raise</span> NotImplementedError()</div><div class="line"><a name="l00223"></a><span class="lineno"> 223</span> </div><div class="line"><a name="l00224"></a><span class="lineno"> 224</span> </div><div class="line"><a name="l00225"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_piped_reader_builder.html"> 225</a></span> <span class="keyword">class </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_piped_reader_builder.html">PipedReaderBuilder</a>(<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_builder.html">ReaderBuilder</a>):</div><div class="line"><a name="l00226"></a><span class="lineno"> 226</span>  <span class="stringliteral">"""ReaderBuilder that modifies underlying builder by calling `piper`</span></div><div class="line"><a name="l00227"></a><span class="lineno"> 227</span> <span class="stringliteral"> function on each new reader produced, and return the result of</span></div><div class="line"><a name="l00228"></a><span class="lineno"> 228</span> <span class="stringliteral"> the function. This way, it is possible to append data processing</span></div><div class="line"><a name="l00229"></a><span class="lineno"> 229</span> <span class="stringliteral"> pipelines that will be replicated for each reader that gets created.</span></div><div class="line"><a name="l00230"></a><span class="lineno"> 230</span> <span class="stringliteral"></span></div><div class="line"><a name="l00231"></a><span class="lineno"> 231</span> <span class="stringliteral"> E.g.:</span></div><div class="line"><a name="l00232"></a><span class="lineno"> 232</span> <span class="stringliteral"></span></div><div class="line"><a name="l00233"></a><span class="lineno"> 233</span> <span class="stringliteral"> PipedReaderBuilder(</span></div><div class="line"><a name="l00234"></a><span class="lineno"> 234</span> <span class="stringliteral"> ReaderBuilder(...),</span></div><div class="line"><a name="l00235"></a><span class="lineno"> 235</span> <span class="stringliteral"> lambda reader: pipe(reader, processor=my_proc))</span></div><div class="line"><a name="l00236"></a><span class="lineno"> 236</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00237"></a><span class="lineno"> 237</span> </div><div class="line"><a name="l00238"></a><span class="lineno"> 238</span>  <span class="keyword">def </span>__init__(self, builder, piper):</div><div class="line"><a name="l00239"></a><span class="lineno"> 239</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_piped_reader_builder.html#a5b880dde8089504e6a1d59d961211510">_builder</a> = builder</div><div class="line"><a name="l00240"></a><span class="lineno"> 240</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_piped_reader_builder.html#afa8acb4ffa740a238fcb64d307d245a2">_piper</a> = piper</div><div class="line"><a name="l00241"></a><span class="lineno"> 241</span> </div><div class="line"><a name="l00242"></a><span class="lineno"> 242</span>  <span class="keyword">def </span><a class="code" href="namespaceschema.html">schema</a>(self):</div><div class="line"><a name="l00243"></a><span class="lineno"> 243</span>  <span class="keywordflow">return</span> self._builder.schema()</div><div class="line"><a name="l00244"></a><span class="lineno"> 244</span> </div><div class="line"><a name="l00245"></a><span class="lineno"> 245</span>  <span class="keyword">def </span>setup(self, **kwargs):</div><div class="line"><a name="l00246"></a><span class="lineno"> 246</span>  self._builder.setup(**kwargs)</div><div class="line"><a name="l00247"></a><span class="lineno"> 247</span> </div><div class="line"><a name="l00248"></a><span class="lineno"> 248</span>  <span class="keyword">def </span>new_reader(self, **kwargs):</div><div class="line"><a name="l00249"></a><span class="lineno"> 249</span>  <span class="comment"># Passing everything down since you could wrap a PipedReaderBuilder in</span></div><div class="line"><a name="l00250"></a><span class="lineno"> 250</span>  <span class="comment"># another PipedReaderBuilder</span></div><div class="line"><a name="l00251"></a><span class="lineno"> 251</span>  output = self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_piped_reader_builder.html#afa8acb4ffa740a238fcb64d307d245a2">_piper</a>(</div><div class="line"><a name="l00252"></a><span class="lineno"> 252</span>  reader=self._builder.new_reader(**kwargs),</div><div class="line"><a name="l00253"></a><span class="lineno"> 253</span>  **kwargs</div><div class="line"><a name="l00254"></a><span class="lineno"> 254</span>  )</div><div class="line"><a name="l00255"></a><span class="lineno"> 255</span>  <span class="keywordflow">return</span> output <span class="keywordflow">if</span> isinstance(output, Reader) <span class="keywordflow">else</span> output.reader()</div><div class="line"><a name="l00256"></a><span class="lineno"> 256</span> </div><div class="line"><a name="l00257"></a><span class="lineno"> 257</span> </div><div class="line"><a name="l00258"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_pipe.html"> 258</a></span> <span class="keyword">class </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_pipe.html">Pipe</a>(object):</div><div class="line"><a name="l00259"></a><span class="lineno"> 259</span>  <span class="keyword">def </span>__init__(self, schema=None, obj_key=None):</div><div class="line"><a name="l00260"></a><span class="lineno"> 260</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_pipe.html#a67fa37befe204abe20469aefcc435b5d">_num_writers</a> = 0</div><div class="line"><a name="l00261"></a><span class="lineno"> 261</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_pipe.html#aeb582e1dc5b200f59799a85c7162aed9">_num_readers</a> = 0</div><div class="line"><a name="l00262"></a><span class="lineno"> 262</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_pipe.html#a1583d0c5bad66affcfe3def7ccf48bf0">_schema</a> = schema</div><div class="line"><a name="l00263"></a><span class="lineno"> 263</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_pipe.html#a978c380d579e6ec821fe71e1be9d574d">_obj_key</a> = obj_key</div><div class="line"><a name="l00264"></a><span class="lineno"> 264</span> </div><div class="line"><a name="l00265"></a><span class="lineno"> 265</span>  <span class="keyword">def </span><a class="code" href="namespaceschema.html">schema</a>(self):</div><div class="line"><a name="l00266"></a><span class="lineno"> 266</span>  <span class="keywordflow">return</span> self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_pipe.html#a1583d0c5bad66affcfe3def7ccf48bf0">_schema</a></div><div class="line"><a name="l00267"></a><span class="lineno"> 267</span> </div><div class="line"><a name="l00268"></a><span class="lineno"> 268</span>  <span class="keyword">def </span>setup(self, global_init_net):</div><div class="line"><a name="l00269"></a><span class="lineno"> 269</span>  <span class="keywordflow">pass</span></div><div class="line"><a name="l00270"></a><span class="lineno"> 270</span> </div><div class="line"><a name="l00271"></a><span class="lineno"> 271</span>  <span class="keyword">def </span>reader(self):</div><div class="line"><a name="l00272"></a><span class="lineno"> 272</span>  <span class="keywordflow">raise</span> NotImplementedError()</div><div class="line"><a name="l00273"></a><span class="lineno"> 273</span> </div><div class="line"><a name="l00274"></a><span class="lineno"> 274</span>  <span class="keyword">def </span>writer(self):</div><div class="line"><a name="l00275"></a><span class="lineno"> 275</span>  <span class="keywordflow">raise</span> NotImplementedError()</div><div class="line"><a name="l00276"></a><span class="lineno"> 276</span> </div><div class="line"><a name="l00277"></a><span class="lineno"> 277</span>  <span class="keyword">def </span>num_readers(self):</div><div class="line"><a name="l00278"></a><span class="lineno"> 278</span>  <span class="keywordflow">return</span> self._num_readers</div><div class="line"><a name="l00279"></a><span class="lineno"> 279</span> </div><div class="line"><a name="l00280"></a><span class="lineno"> 280</span>  <span class="keyword">def </span>num_writers(self):</div><div class="line"><a name="l00281"></a><span class="lineno"> 281</span>  <span class="keywordflow">return</span> self._num_writers</div><div class="line"><a name="l00282"></a><span class="lineno"> 282</span> </div><div class="line"><a name="l00283"></a><span class="lineno"> 283</span>  <span class="keyword">def </span>_new_writer(self, writer_schema, writer_init_net):</div><div class="line"><a name="l00284"></a><span class="lineno"> 284</span>  <span class="keywordflow">if</span> writer_schema <span class="keywordflow">is</span> <span class="keywordflow">not</span> <span class="keywordtype">None</span> <span class="keywordflow">and</span> self._schema <span class="keywordflow">is</span> <span class="keywordtype">None</span>:</div><div class="line"><a name="l00285"></a><span class="lineno"> 285</span>  self._schema = writer_schema</div><div class="line"><a name="l00286"></a><span class="lineno"> 286</span>  self._num_writers += 1</div><div class="line"><a name="l00287"></a><span class="lineno"> 287</span>  <span class="keywordflow">if</span> self._obj_key <span class="keywordflow">is</span> <span class="keywordflow">not</span> <span class="keywordtype">None</span>:</div><div class="line"><a name="l00288"></a><span class="lineno"> 288</span>  writer_init_net.add_attribute(self._obj_key, self)</div><div class="line"><a name="l00289"></a><span class="lineno"> 289</span> </div><div class="line"><a name="l00290"></a><span class="lineno"> 290</span>  <span class="keyword">def </span>_new_reader(self, reader_init_net):</div><div class="line"><a name="l00291"></a><span class="lineno"> 291</span>  self._num_readers += 1</div><div class="line"><a name="l00292"></a><span class="lineno"> 292</span>  <span class="keywordflow">if</span> self._obj_key <span class="keywordflow">is</span> <span class="keywordflow">not</span> <span class="keywordtype">None</span>:</div><div class="line"><a name="l00293"></a><span class="lineno"> 293</span>  reader_init_net.add_attribute(self._obj_key, self)</div><div class="line"><a name="l00294"></a><span class="lineno"> 294</span> </div><div class="line"><a name="l00295"></a><span class="lineno"> 295</span> </div><div class="line"><a name="l00296"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_counter_reader.html"> 296</a></span> <span class="keyword">class </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_counter_reader.html">CounterReader</a>(<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html">Reader</a>):</div><div class="line"><a name="l00297"></a><span class="lineno"> 297</span>  <span class="stringliteral">""" Reader that produces increasing integers. """</span></div><div class="line"><a name="l00298"></a><span class="lineno"> 298</span>  <span class="keyword">def </span>__init__(self):</div><div class="line"><a name="l00299"></a><span class="lineno"> 299</span>  Reader.__init__(self, schema=<a class="code" href="classcaffe2_1_1python_1_1schema_1_1_struct.html">Struct</a>((<span class="stringliteral">'iter'</span>, np.int64)))</div><div class="line"><a name="l00300"></a><span class="lineno"> 300</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_counter_reader.html#aef4fa9db5d30aa1f08f744490ce8021f">counter</a> = <span class="keywordtype">None</span></div><div class="line"><a name="l00301"></a><span class="lineno"> 301</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_counter_reader.html#a3b43f6f116b22f46e62dac1541b335c1">should_stop</a> = <span class="keywordtype">None</span></div><div class="line"><a name="l00302"></a><span class="lineno"> 302</span> </div><div class="line"><a name="l00303"></a><span class="lineno"> 303</span>  <span class="keyword">def </span>setup_ex(self, global_init_net, global_finish_net):</div><div class="line"><a name="l00304"></a><span class="lineno"> 304</span>  <span class="keywordflow">if</span> self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_counter_reader.html#aef4fa9db5d30aa1f08f744490ce8021f">counter</a> <span class="keywordflow">is</span> <span class="keywordtype">None</span>:</div><div class="line"><a name="l00305"></a><span class="lineno"> 305</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_counter_reader.html#aef4fa9db5d30aa1f08f744490ce8021f">counter</a> = global_init_net.CreateCounter([], init_count=0)</div><div class="line"><a name="l00306"></a><span class="lineno"> 306</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_counter_reader.html#a3b43f6f116b22f46e62dac1541b335c1">should_stop</a> = global_init_net.ConstantFill(</div><div class="line"><a name="l00307"></a><span class="lineno"> 307</span>  [], shape=[], dtype=core.DataType.BOOL, value=<span class="keyword">False</span>)</div><div class="line"><a name="l00308"></a><span class="lineno"> 308</span> </div><div class="line"><a name="l00309"></a><span class="lineno"> 309</span>  <span class="keyword">def </span>read_ex(self, local_init_net, local_finish_net):</div><div class="line"><a name="l00310"></a><span class="lineno"> 310</span>  count_net = <a class="code" href="classcaffe2_1_1python_1_1core_1_1_net.html">core.Net</a>(<span class="stringliteral">'limited_reader_counter'</span>)</div><div class="line"><a name="l00311"></a><span class="lineno"> 311</span>  value = count_net.CountUp([self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_counter_reader.html#aef4fa9db5d30aa1f08f744490ce8021f">counter</a>], 1)</div><div class="line"><a name="l00312"></a><span class="lineno"> 312</span>  <span class="keywordflow">return</span> [count_net], self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_counter_reader.html#a3b43f6f116b22f46e62dac1541b335c1">should_stop</a>, [value]</div><div class="line"><a name="l00313"></a><span class="lineno"> 313</span> </div><div class="line"><a name="l00314"></a><span class="lineno"> 314</span> </div><div class="line"><a name="l00315"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html"> 315</a></span> <span class="keyword">class </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html">ReaderWithLimitBase</a>(<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html">Reader</a>):</div><div class="line"><a name="l00316"></a><span class="lineno"> 316</span>  <span class="stringliteral">"""Abstract Reader constrained by certain conditions.</span></div><div class="line"><a name="l00317"></a><span class="lineno"> 317</span> <span class="stringliteral"></span></div><div class="line"><a name="l00318"></a><span class="lineno"> 318</span> <span class="stringliteral"> Base class for Reader classes which check for certain conditions to stop</span></div><div class="line"><a name="l00319"></a><span class="lineno"> 319</span> <span class="stringliteral"> further processing (e.g. max number of iterations or time limit).</span></div><div class="line"><a name="l00320"></a><span class="lineno"> 320</span> <span class="stringliteral"> Also produces a boolean blob (data_finished) that can be used to see if</span></div><div class="line"><a name="l00321"></a><span class="lineno"> 321</span> <span class="stringliteral"> the reader exausted all input data (true) or stopped for another reason</span></div><div class="line"><a name="l00322"></a><span class="lineno"> 322</span> <span class="stringliteral"> (false).</span></div><div class="line"><a name="l00323"></a><span class="lineno"> 323</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00324"></a><span class="lineno"> 324</span> </div><div class="line"><a name="l00325"></a><span class="lineno"> 325</span>  <span class="keyword">def </span>__init__(self, reader):</div><div class="line"><a name="l00326"></a><span class="lineno"> 326</span>  Reader.__init__(self, schema=reader._schema)</div><div class="line"><a name="l00327"></a><span class="lineno"> 327</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a4280bdbee45637532cc46f379ceebe36">reader</a> = reader</div><div class="line"><a name="l00328"></a><span class="lineno"> 328</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a2695eb809e444deca3fa9c1d1e6a45a0">net</a> = <a class="code" href="classcaffe2_1_1python_1_1core_1_1_net.html">core.Net</a>(<span class="stringliteral">'reader_with_limit'</span>)</div><div class="line"><a name="l00329"></a><span class="lineno"> 329</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a40e3b66209543cb9501574bc529c55a2">_data_finished</a> = self.net.AddExternalInput(</div><div class="line"><a name="l00330"></a><span class="lineno"> 330</span>  self.net.NextName(<span class="stringliteral">'data_finished'</span>))</div><div class="line"><a name="l00331"></a><span class="lineno"> 331</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#afe0a7c91ce61267f4966715d97897f86">should_stop</a> = <span class="keywordtype">None</span></div><div class="line"><a name="l00332"></a><span class="lineno"> 332</span> </div><div class="line"><a name="l00333"></a><span class="lineno"> 333</span>  <span class="keyword">def </span>setup_ex(self, global_init_net, global_finish_net):</div><div class="line"><a name="l00334"></a><span class="lineno"> 334</span>  global_init_net.ConstantFill(</div><div class="line"><a name="l00335"></a><span class="lineno"> 335</span>  [], [self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a40e3b66209543cb9501574bc529c55a2">_data_finished</a>],</div><div class="line"><a name="l00336"></a><span class="lineno"> 336</span>  shape=[], value=<span class="keyword">False</span>, dtype=core.DataType.BOOL)</div><div class="line"><a name="l00337"></a><span class="lineno"> 337</span>  self.reader.setup_ex(global_init_net, global_finish_net)</div><div class="line"><a name="l00338"></a><span class="lineno"> 338</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a969c829309710eced9c46c7372267f43">setup_limiter</a>(global_init_net, global_finish_net)</div><div class="line"><a name="l00339"></a><span class="lineno"> 339</span> </div><div class="line"><a name="l00340"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a947fb82fe17fd625685cad6f63874c50"> 340</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a947fb82fe17fd625685cad6f63874c50">read_ex</a>(self, local_init_net, local_finish_net):</div><div class="line"><a name="l00341"></a><span class="lineno"> 341</span>  <span class="stringliteral">"""Reads from an underlying Reader class, but may stop due to additional</span></div><div class="line"><a name="l00342"></a><span class="lineno"> 342</span> <span class="stringliteral"> constraints.</span></div><div class="line"><a name="l00343"></a><span class="lineno"> 343</span> <span class="stringliteral"></span></div><div class="line"><a name="l00344"></a><span class="lineno"> 344</span> <span class="stringliteral"> Build and return network(s) to read data from a Reader with</span></div><div class="line"><a name="l00345"></a><span class="lineno"> 345</span> <span class="stringliteral"> additional constraints, depending on which derived class is used.</span></div><div class="line"><a name="l00346"></a><span class="lineno"> 346</span> <span class="stringliteral"> Derived classes implement setup_limited and check_limiter_condition</span></div><div class="line"><a name="l00347"></a><span class="lineno"> 347</span> <span class="stringliteral"> which determine the nature of the constraint imposed on the reader,</span></div><div class="line"><a name="l00348"></a><span class="lineno"> 348</span> <span class="stringliteral"> e.g. iteration limits or time limit.</span></div><div class="line"><a name="l00349"></a><span class="lineno"> 349</span> <span class="stringliteral"></span></div><div class="line"><a name="l00350"></a><span class="lineno"> 350</span> <span class="stringliteral"> Args:</span></div><div class="line"><a name="l00351"></a><span class="lineno"> 351</span> <span class="stringliteral"> local_init_net: A net invoked at task instance init time (Once per</span></div><div class="line"><a name="l00352"></a><span class="lineno"> 352</span> <span class="stringliteral"> parallel thread).</span></div><div class="line"><a name="l00353"></a><span class="lineno"> 353</span> <span class="stringliteral"> local_finish_net: A net invoked at task instance cleanup time (Once</span></div><div class="line"><a name="l00354"></a><span class="lineno"> 354</span> <span class="stringliteral"> per parallel thread).</span></div><div class="line"><a name="l00355"></a><span class="lineno"> 355</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00356"></a><span class="lineno"> 356</span> </div><div class="line"><a name="l00357"></a><span class="lineno"> 357</span>  <span class="comment"># Check if limiting constraint is met.</span></div><div class="line"><a name="l00358"></a><span class="lineno"> 358</span>  stop_condition_net = <a class="code" href="classcaffe2_1_1python_1_1core_1_1_net.html">core.Net</a>(<span class="stringliteral">'limited_reader_condition'</span>)</div><div class="line"><a name="l00359"></a><span class="lineno"> 359</span>  should_stop = self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a77b30909960c74b18df486f3e727da23">check_limiter_condition</a>(stop_condition_net)</div><div class="line"><a name="l00360"></a><span class="lineno"> 360</span> </div><div class="line"><a name="l00361"></a><span class="lineno"> 361</span>  <span class="comment"># Call original reader.</span></div><div class="line"><a name="l00362"></a><span class="lineno"> 362</span>  nets, local_data_finished, fields = self.reader.read_ex(</div><div class="line"><a name="l00363"></a><span class="lineno"> 363</span>  local_init_net, local_finish_net)</div><div class="line"><a name="l00364"></a><span class="lineno"> 364</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#aab1108a3c40f87d3a28a8e195a68f76e">_set_schema</a>(self.reader._schema)</div><div class="line"><a name="l00365"></a><span class="lineno"> 365</span> </div><div class="line"><a name="l00366"></a><span class="lineno"> 366</span>  <span class="comment"># Check if original reader is done.</span></div><div class="line"><a name="l00367"></a><span class="lineno"> 367</span>  check_done_net = <a class="code" href="classcaffe2_1_1python_1_1core_1_1_net.html">core.Net</a>(<span class="stringliteral">'limited_reader_post'</span>)</div><div class="line"><a name="l00368"></a><span class="lineno"> 368</span>  <span class="comment"># Copy to the same blob as the counter output to trigger reader</span></div><div class="line"><a name="l00369"></a><span class="lineno"> 369</span>  <span class="comment"># stopping - this is ok because execution will check should_stop_blob</span></div><div class="line"><a name="l00370"></a><span class="lineno"> 370</span>  <span class="comment"># after every single operation, so it has already been checked on this</span></div><div class="line"><a name="l00371"></a><span class="lineno"> 371</span>  <span class="comment"># iteration by this point.</span></div><div class="line"><a name="l00372"></a><span class="lineno"> 372</span>  check_done_net.Copy(local_data_finished, should_stop)</div><div class="line"><a name="l00373"></a><span class="lineno"> 373</span>  <span class="comment"># Update externally-accessible flag indicating if reader is done</span></div><div class="line"><a name="l00374"></a><span class="lineno"> 374</span>  check_done_net.Or([self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a40e3b66209543cb9501574bc529c55a2">_data_finished</a>, local_data_finished],</div><div class="line"><a name="l00375"></a><span class="lineno"> 375</span>  [self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a40e3b66209543cb9501574bc529c55a2">_data_finished</a>])</div><div class="line"><a name="l00376"></a><span class="lineno"> 376</span> </div><div class="line"><a name="l00377"></a><span class="lineno"> 377</span>  <span class="keywordflow">return</span> [stop_condition_net] + nets + [check_done_net], should_stop, fields</div><div class="line"><a name="l00378"></a><span class="lineno"> 378</span> </div><div class="line"><a name="l00379"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a969c829309710eced9c46c7372267f43"> 379</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a969c829309710eced9c46c7372267f43">setup_limiter</a>(self, global_init_net, global_finish_net):</div><div class="line"><a name="l00380"></a><span class="lineno"> 380</span>  <span class="stringliteral">"""Configure task level init/cleanup nets required to implement limit</span></div><div class="line"><a name="l00381"></a><span class="lineno"> 381</span> <span class="stringliteral"> condition. Must be implemented by subclass.</span></div><div class="line"><a name="l00382"></a><span class="lineno"> 382</span> <span class="stringliteral"></span></div><div class="line"><a name="l00383"></a><span class="lineno"> 383</span> <span class="stringliteral"> Args:</span></div><div class="line"><a name="l00384"></a><span class="lineno"> 384</span> <span class="stringliteral"> global_init_net: A net invoked at task init time.</span></div><div class="line"><a name="l00385"></a><span class="lineno"> 385</span> <span class="stringliteral"> global_finish_net: A net invoked at task cleanup time.</span></div><div class="line"><a name="l00386"></a><span class="lineno"> 386</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00387"></a><span class="lineno"> 387</span>  <span class="keywordflow">raise</span> NotImplementedError(<span class="stringliteral">"Subclass must implement `setup_limiter`"</span>)</div><div class="line"><a name="l00388"></a><span class="lineno"> 388</span> </div><div class="line"><a name="l00389"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a77b30909960c74b18df486f3e727da23"> 389</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a77b30909960c74b18df486f3e727da23">check_limiter_condition</a>(self, stop_condition_net):</div><div class="line"><a name="l00390"></a><span class="lineno"> 390</span>  <span class="stringliteral">"""Configure a net that is invoked between reading batches to see if</span></div><div class="line"><a name="l00391"></a><span class="lineno"> 391</span> <span class="stringliteral"> limit condition is met. Must be implemented by subclass.</span></div><div class="line"><a name="l00392"></a><span class="lineno"> 392</span> <span class="stringliteral"></span></div><div class="line"><a name="l00393"></a><span class="lineno"> 393</span> <span class="stringliteral"> Args:</span></div><div class="line"><a name="l00394"></a><span class="lineno"> 394</span> <span class="stringliteral"> stop_condition_net: A net invoked to evaluate an early termination</span></div><div class="line"><a name="l00395"></a><span class="lineno"> 395</span> <span class="stringliteral"> condition.</span></div><div class="line"><a name="l00396"></a><span class="lineno"> 396</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00397"></a><span class="lineno"> 397</span>  <span class="keywordflow">raise</span> NotImplementedError(<span class="stringliteral">"Subclass must implement `check_limiter_condition"</span>)</div><div class="line"><a name="l00398"></a><span class="lineno"> 398</span> </div><div class="line"><a name="l00399"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#aac12472533f0d43e0cb4b8a5df6489d1"> 399</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#aac12472533f0d43e0cb4b8a5df6489d1">data_finished</a>(self):</div><div class="line"><a name="l00400"></a><span class="lineno"> 400</span>  <span class="stringliteral">"""</span></div><div class="line"><a name="l00401"></a><span class="lineno"> 401</span> <span class="stringliteral"> Return a blob that can be checked after the end of the reading task,</span></div><div class="line"><a name="l00402"></a><span class="lineno"> 402</span> <span class="stringliteral"> which will contain a scalar float indicating whether the underlying</span></div><div class="line"><a name="l00403"></a><span class="lineno"> 403</span> <span class="stringliteral"> reader has been exhausted (True) or whether we stopped because reached</span></div><div class="line"><a name="l00404"></a><span class="lineno"> 404</span> <span class="stringliteral"> the limit of iterations (False).</span></div><div class="line"><a name="l00405"></a><span class="lineno"> 405</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00406"></a><span class="lineno"> 406</span>  <span class="keywordflow">return</span> self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a40e3b66209543cb9501574bc529c55a2">_data_finished</a></div><div class="line"><a name="l00407"></a><span class="lineno"> 407</span> </div><div class="line"><a name="l00408"></a><span class="lineno"> 408</span> </div><div class="line"><a name="l00409"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html"> 409</a></span> <span class="keyword">class </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html">ReaderWithLimit</a>(<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html">ReaderWithLimitBase</a>):</div><div class="line"><a name="l00410"></a><span class="lineno"> 410</span>  <span class="stringliteral">"""Reader that stops after `num_iter` batches.</span></div><div class="line"><a name="l00411"></a><span class="lineno"> 411</span> <span class="stringliteral"></span></div><div class="line"><a name="l00412"></a><span class="lineno"> 412</span> <span class="stringliteral"> If `num_iter` <= 0 or is None, reverts to an unconstrained reader that</span></div><div class="line"><a name="l00413"></a><span class="lineno"> 413</span> <span class="stringliteral"> exports a boolean blob indicating that the reader has exhausted</span></div><div class="line"><a name="l00414"></a><span class="lineno"> 414</span> <span class="stringliteral"> the data steam.</span></div><div class="line"><a name="l00415"></a><span class="lineno"> 415</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00416"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#a70002bd48f0f4d114520015b12bfdfbd"> 416</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#a70002bd48f0f4d114520015b12bfdfbd">__init__</a>(self, reader, num_iter=1):</div><div class="line"><a name="l00417"></a><span class="lineno"> 417</span>  <span class="stringliteral">"""Class initializer.</span></div><div class="line"><a name="l00418"></a><span class="lineno"> 418</span> <span class="stringliteral"></span></div><div class="line"><a name="l00419"></a><span class="lineno"> 419</span> <span class="stringliteral"> Args:</span></div><div class="line"><a name="l00420"></a><span class="lineno"> 420</span> <span class="stringliteral"> reader: The underlying reader object doing the actual read.</span></div><div class="line"><a name="l00421"></a><span class="lineno"> 421</span> <span class="stringliteral"> num_iter: Number of batches to read. If `None`,</span></div><div class="line"><a name="l00422"></a><span class="lineno"> 422</span> <span class="stringliteral"> the class reverts to a normal reader except that it also</span></div><div class="line"><a name="l00423"></a><span class="lineno"> 423</span> <span class="stringliteral"> produces a data_finished blob as a side effect to indicate</span></div><div class="line"><a name="l00424"></a><span class="lineno"> 424</span> <span class="stringliteral"> whether the input stream is exhausted.</span></div><div class="line"><a name="l00425"></a><span class="lineno"> 425</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00426"></a><span class="lineno"> 426</span>  super(ReaderWithLimit, self).<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#a70002bd48f0f4d114520015b12bfdfbd">__init__</a>(reader)</div><div class="line"><a name="l00427"></a><span class="lineno"> 427</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#af3e6f05bed28a9630dc7a148cbb95098">counter</a> = <span class="keywordtype">None</span></div><div class="line"><a name="l00428"></a><span class="lineno"> 428</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#affce11133c16f0bb732989888c221f67">num_iter</a> = num_iter</div><div class="line"><a name="l00429"></a><span class="lineno"> 429</span>  <span class="keywordflow">if</span> self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#affce11133c16f0bb732989888c221f67">num_iter</a> <span class="keywordflow">is</span> <span class="keywordflow">not</span> <span class="keywordtype">None</span>:</div><div class="line"><a name="l00430"></a><span class="lineno"> 430</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#af3e6f05bed28a9630dc7a148cbb95098">counter</a> = self.net.AddExternalInput(</div><div class="line"><a name="l00431"></a><span class="lineno"> 431</span>  self.net.NextName(<span class="stringliteral">'counter'</span>))</div><div class="line"><a name="l00432"></a><span class="lineno"> 432</span> </div><div class="line"><a name="l00433"></a><span class="lineno"> 433</span>  <span class="keyword">def </span>setup_limiter(self, global_init_net, global_finish_net):</div><div class="line"><a name="l00434"></a><span class="lineno"> 434</span>  <span class="keywordflow">if</span> self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#af3e6f05bed28a9630dc7a148cbb95098">counter</a>:</div><div class="line"><a name="l00435"></a><span class="lineno"> 435</span>  global_init_net.CreateCounter(</div><div class="line"><a name="l00436"></a><span class="lineno"> 436</span>  [], [self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#af3e6f05bed28a9630dc7a148cbb95098">counter</a>], init_count=int(self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#affce11133c16f0bb732989888c221f67">num_iter</a>))</div><div class="line"><a name="l00437"></a><span class="lineno"> 437</span> </div><div class="line"><a name="l00438"></a><span class="lineno"> 438</span>  <span class="keyword">def </span>check_limiter_condition(self, stop_condition_net):</div><div class="line"><a name="l00439"></a><span class="lineno"> 439</span>  <span class="keywordflow">if</span> self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#af3e6f05bed28a9630dc7a148cbb95098">counter</a>:</div><div class="line"><a name="l00440"></a><span class="lineno"> 440</span>  <span class="keywordflow">return</span> stop_condition_net.CountDown([self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#af3e6f05bed28a9630dc7a148cbb95098">counter</a>], 1)</div><div class="line"><a name="l00441"></a><span class="lineno"> 441</span>  <span class="keywordflow">else</span>:</div><div class="line"><a name="l00442"></a><span class="lineno"> 442</span>  <span class="keywordflow">return</span> stop_condition_net.ConstantFill(</div><div class="line"><a name="l00443"></a><span class="lineno"> 443</span>  [], 1,</div><div class="line"><a name="l00444"></a><span class="lineno"> 444</span>  shape=[], value=<span class="keyword">False</span>, dtype=core.DataType.BOOL)</div><div class="line"><a name="l00445"></a><span class="lineno"> 445</span> </div><div class="line"><a name="l00446"></a><span class="lineno"> 446</span> </div><div class="line"><a name="l00447"></a><span class="lineno"> 447</span> <span class="keyword">def </span>CountUntil(num_iter):</div><div class="line"><a name="l00448"></a><span class="lineno"> 448</span>  <span class="keywordflow">return</span> <a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html">ReaderWithLimit</a>(<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_counter_reader.html">CounterReader</a>(), num_iter)</div><div class="line"><a name="l00449"></a><span class="lineno"> 449</span> </div><div class="line"><a name="l00450"></a><span class="lineno"> 450</span> </div><div class="line"><a name="l00451"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html"> 451</a></span> <span class="keyword">class </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html">ReaderWithTimeLimit</a>(<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html">ReaderWithLimitBase</a>):</div><div class="line"><a name="l00452"></a><span class="lineno"> 452</span>  <span class="stringliteral">"""Reader that stops after `duration` seconds.</span></div><div class="line"><a name="l00453"></a><span class="lineno"> 453</span> <span class="stringliteral"></span></div><div class="line"><a name="l00454"></a><span class="lineno"> 454</span> <span class="stringliteral"> If `duration` <= 0 or is None, reverts to an unconstrained reader that</span></div><div class="line"><a name="l00455"></a><span class="lineno"> 455</span> <span class="stringliteral"> exports a boolean blob indicating that the reader has exhausted</span></div><div class="line"><a name="l00456"></a><span class="lineno"> 456</span> <span class="stringliteral"> the data steam.</span></div><div class="line"><a name="l00457"></a><span class="lineno"> 457</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00458"></a><span class="lineno"><a class="line" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#a71add0ae34dd8e3caf4acae256eca151"> 458</a></span>  <span class="keyword">def </span><a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#a71add0ae34dd8e3caf4acae256eca151">__init__</a>(self, reader, duration=0):</div><div class="line"><a name="l00459"></a><span class="lineno"> 459</span>  <span class="stringliteral">"""Class initializer.</span></div><div class="line"><a name="l00460"></a><span class="lineno"> 460</span> <span class="stringliteral"></span></div><div class="line"><a name="l00461"></a><span class="lineno"> 461</span> <span class="stringliteral"> Args:</span></div><div class="line"><a name="l00462"></a><span class="lineno"> 462</span> <span class="stringliteral"> reader: The underlying reader object doing the actual read.</span></div><div class="line"><a name="l00463"></a><span class="lineno"> 463</span> <span class="stringliteral"> duration: Number of seconds to read. If un-specified, None, or <= 0,</span></div><div class="line"><a name="l00464"></a><span class="lineno"> 464</span> <span class="stringliteral"> the class reverts to a normal reader except that it also</span></div><div class="line"><a name="l00465"></a><span class="lineno"> 465</span> <span class="stringliteral"> produces a data_finished blob as a side effect to indicate</span></div><div class="line"><a name="l00466"></a><span class="lineno"> 466</span> <span class="stringliteral"> whether the input stream is exhausted.</span></div><div class="line"><a name="l00467"></a><span class="lineno"> 467</span> <span class="stringliteral"> """</span></div><div class="line"><a name="l00468"></a><span class="lineno"> 468</span>  super(ReaderWithTimeLimit, self).<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#a71add0ae34dd8e3caf4acae256eca151">__init__</a>(reader)</div><div class="line"><a name="l00469"></a><span class="lineno"> 469</span> </div><div class="line"><a name="l00470"></a><span class="lineno"> 470</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#abee021655958ec4fba6c3a611eb43ec4">timer</a> = <span class="keywordtype">None</span></div><div class="line"><a name="l00471"></a><span class="lineno"> 471</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#a0ff62afd3c822f5bd0cfd7ba5c1b859a">duration</a> = duration</div><div class="line"><a name="l00472"></a><span class="lineno"> 472</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#a224e4dc2dbf1d25e2a4fc4c7c3a26095">duration_ns_blob</a> = <span class="keywordtype">None</span></div><div class="line"><a name="l00473"></a><span class="lineno"> 473</span> </div><div class="line"><a name="l00474"></a><span class="lineno"> 474</span>  <span class="keyword">def </span>setup_limiter(self, global_init_net, global_finish_net):</div><div class="line"><a name="l00475"></a><span class="lineno"> 475</span>  <span class="keywordflow">if</span> self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#a0ff62afd3c822f5bd0cfd7ba5c1b859a">duration</a> <span class="keywordflow">is</span> <span class="keywordflow">not</span> <span class="keywordtype">None</span> <span class="keywordflow">and</span> self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#a0ff62afd3c822f5bd0cfd7ba5c1b859a">duration</a> > 0:</div><div class="line"><a name="l00476"></a><span class="lineno"> 476</span>  duration_ns = int(self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#a0ff62afd3c822f5bd0cfd7ba5c1b859a">duration</a> * (10**9))</div><div class="line"><a name="l00477"></a><span class="lineno"> 477</span> </div><div class="line"><a name="l00478"></a><span class="lineno"> 478</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#abee021655958ec4fba6c3a611eb43ec4">timer</a> = global_init_net.TimerBegin(</div><div class="line"><a name="l00479"></a><span class="lineno"> 479</span>  [], counter_name=<span class="stringliteral">'epoch_timer'</span>)</div><div class="line"><a name="l00480"></a><span class="lineno"> 480</span>  start_time = global_init_net.TimerGet(self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#abee021655958ec4fba6c3a611eb43ec4">timer</a>)</div><div class="line"><a name="l00481"></a><span class="lineno"> 481</span>  self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#a224e4dc2dbf1d25e2a4fc4c7c3a26095">duration_ns_blob</a> = global_init_net.ConstantFill(</div><div class="line"><a name="l00482"></a><span class="lineno"> 482</span>  [start_time], value=duration_ns)</div><div class="line"><a name="l00483"></a><span class="lineno"> 483</span> </div><div class="line"><a name="l00484"></a><span class="lineno"> 484</span>  global_finish_net.TimerEnd([self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#abee021655958ec4fba6c3a611eb43ec4">timer</a>], [])</div><div class="line"><a name="l00485"></a><span class="lineno"> 485</span> </div><div class="line"><a name="l00486"></a><span class="lineno"> 486</span>  <span class="keyword">def </span>check_limiter_condition(self, stop_condition_net):</div><div class="line"><a name="l00487"></a><span class="lineno"> 487</span>  <span class="keywordflow">if</span> self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#a0ff62afd3c822f5bd0cfd7ba5c1b859a">duration</a>:</div><div class="line"><a name="l00488"></a><span class="lineno"> 488</span>  time_elapsed = stop_condition_net.TimerGet(self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#abee021655958ec4fba6c3a611eb43ec4">timer</a>)</div><div class="line"><a name="l00489"></a><span class="lineno"> 489</span>  <span class="keywordflow">return</span> stop_condition_net.GE(</div><div class="line"><a name="l00490"></a><span class="lineno"> 490</span>  [time_elapsed, self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#a224e4dc2dbf1d25e2a4fc4c7c3a26095">duration_ns_blob</a>], str(self.<a class="code" href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#afe0a7c91ce61267f4966715d97897f86">should_stop</a>))</div><div class="line"><a name="l00491"></a><span class="lineno"> 491</span>  <span class="keywordflow">else</span>:</div><div class="line"><a name="l00492"></a><span class="lineno"> 492</span>  <span class="keywordflow">return</span> stop_condition_net.ConstantFill(</div><div class="line"><a name="l00493"></a><span class="lineno"> 493</span>  [], 1, shape=[], value=<span class="keyword">False</span>, dtype=core.DataType.BOOL</div><div class="line"><a name="l00494"></a><span class="lineno"> 494</span>  )</div><div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base_html_a40e3b66209543cb9501574bc529c55a2"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a40e3b66209543cb9501574bc529c55a2">caffe2.python.dataio.ReaderWithLimitBase._data_finished</a></div><div class="ttdeci">_data_finished</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00329">dataio.py:329</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_html_affce11133c16f0bb732989888c221f67"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#affce11133c16f0bb732989888c221f67">caffe2.python.dataio.ReaderWithLimit.num_iter</a></div><div class="ttdeci">num_iter</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00428">dataio.py:428</a></div></div>
<div class="ttc" id="namespaceschema_html"><div class="ttname"><a href="namespaceschema.html">schema</a></div><div class="ttdoc">Module caffe2.python.schema. </div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_builder_html_a45191bc578de11a88715b49faec44d03"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_builder.html#a45191bc578de11a88715b49faec44d03">caffe2.python.dataio.ReaderBuilder.setup</a></div><div class="ttdeci">def setup(self, kwargs)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00214">dataio.py:214</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base_html_afe0a7c91ce61267f4966715d97897f86"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#afe0a7c91ce61267f4966715d97897f86">caffe2.python.dataio.ReaderWithLimitBase.should_stop</a></div><div class="ttdeci">should_stop</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00331">dataio.py:331</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_builder_html"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_builder.html">caffe2.python.dataio.ReaderBuilder</a></div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00209">dataio.py:209</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_pipe_html_a1583d0c5bad66affcfe3def7ccf48bf0"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_pipe.html#a1583d0c5bad66affcfe3def7ccf48bf0">caffe2.python.dataio.Pipe._schema</a></div><div class="ttdeci">_schema</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00262">dataio.py:262</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base_html_a4280bdbee45637532cc46f379ceebe36"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a4280bdbee45637532cc46f379ceebe36">caffe2.python.dataio.ReaderWithLimitBase.reader</a></div><div class="ttdeci">reader</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00327">dataio.py:327</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_counter_reader_html_a3b43f6f116b22f46e62dac1541b335c1"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_counter_reader.html#a3b43f6f116b22f46e62dac1541b335c1">caffe2.python.dataio.CounterReader.should_stop</a></div><div class="ttdeci">should_stop</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00301">dataio.py:301</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_html_a024602c0217e1c49b436d5d592a9ff70"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a024602c0217e1c49b436d5d592a9ff70">caffe2.python.dataio.Reader.read</a></div><div class="ttdeci">def read(self, read_net)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00070">dataio.py:70</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_html_aab1108a3c40f87d3a28a8e195a68f76e"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#aab1108a3c40f87d3a28a8e195a68f76e">caffe2.python.dataio.Reader._set_schema</a></div><div class="ttdeci">def _set_schema(self, schema)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00047">dataio.py:47</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base_html_aac12472533f0d43e0cb4b8a5df6489d1"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#aac12472533f0d43e0cb4b8a5df6489d1">caffe2.python.dataio.ReaderWithLimitBase.data_finished</a></div><div class="ttdeci">def data_finished(self)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00399">dataio.py:399</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_piped_reader_builder_html_afa8acb4ffa740a238fcb64d307d245a2"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_piped_reader_builder.html#afa8acb4ffa740a238fcb64d307d245a2">caffe2.python.dataio.PipedReaderBuilder._piper</a></div><div class="ttdeci">_piper</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00240">dataio.py:240</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit_html_a224e4dc2dbf1d25e2a4fc4c7c3a26095"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#a224e4dc2dbf1d25e2a4fc4c7c3a26095">caffe2.python.dataio.ReaderWithTimeLimit.duration_ns_blob</a></div><div class="ttdeci">duration_ns_blob</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00472">dataio.py:472</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_html"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader.html">caffe2.python.dataio.Reader</a></div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00028">dataio.py:28</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_html"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html">caffe2.python.dataio.ReaderWithLimit</a></div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00409">dataio.py:409</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_html_a67e4d228c8017789e3331bff2b5d2537"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a67e4d228c8017789e3331bff2b5d2537">caffe2.python.dataio.Reader._schema</a></div><div class="ttdeci">_schema</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00041">dataio.py:41</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base_html_a2695eb809e444deca3fa9c1d1e6a45a0"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a2695eb809e444deca3fa9c1d1e6a45a0">caffe2.python.dataio.ReaderWithLimitBase.net</a></div><div class="ttdeci">net</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00328">dataio.py:328</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_pipe_html"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_pipe.html">caffe2.python.dataio.Pipe</a></div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00258">dataio.py:258</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_writer_html_a55b403eba95e44ab19e68dd164a9aaf1"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a55b403eba95e44ab19e68dd164a9aaf1">caffe2.python.dataio.Writer.write_record_ex</a></div><div class="ttdeci">def write_record_ex(self, fields, local_init_net, local_finish_net, stop_blob=None)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00189">dataio.py:189</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit_html_a71add0ae34dd8e3caf4acae256eca151"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#a71add0ae34dd8e3caf4acae256eca151">caffe2.python.dataio.ReaderWithTimeLimit.__init__</a></div><div class="ttdeci">def __init__(self, reader, duration=0)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00458">dataio.py:458</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_writer_html_a3e270a50d9e2d65db196cddc56132c00"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a3e270a50d9e2d65db196cddc56132c00">caffe2.python.dataio.Writer.setup_ex</a></div><div class="ttdeci">def setup_ex(self, init_net, finish_net)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00178">dataio.py:178</a></div></div>
<div class="ttc" id="namespacecaffe2_1_1python_html"><div class="ttname"><a href="namespacecaffe2_1_1python.html">caffe2.python</a></div><div class="ttdef"><b>Definition:</b> <a href="python_2____init_____8py_source.html#l00001">__init__.py:1</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_pipe_html_a67fa37befe204abe20469aefcc435b5d"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_pipe.html#a67fa37befe204abe20469aefcc435b5d">caffe2.python.dataio.Pipe._num_writers</a></div><div class="ttdeci">_num_writers</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00260">dataio.py:260</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_pipe_html_a978c380d579e6ec821fe71e1be9d574d"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_pipe.html#a978c380d579e6ec821fe71e1be9d574d">caffe2.python.dataio.Pipe._obj_key</a></div><div class="ttdeci">_obj_key</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00263">dataio.py:263</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_piped_reader_builder_html_a5b880dde8089504e6a1d59d961211510"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_piped_reader_builder.html#a5b880dde8089504e6a1d59d961211510">caffe2.python.dataio.PipedReaderBuilder._builder</a></div><div class="ttdeci">_builder</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00239">dataio.py:239</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_piped_reader_builder_html"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_piped_reader_builder.html">caffe2.python.dataio.PipedReaderBuilder</a></div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00225">dataio.py:225</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit_html_abee021655958ec4fba6c3a611eb43ec4"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#abee021655958ec4fba6c3a611eb43ec4">caffe2.python.dataio.ReaderWithTimeLimit.timer</a></div><div class="ttdeci">timer</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00470">dataio.py:470</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_writer_html_a7fb018f843ba304edbd23f0e82328ece"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a7fb018f843ba304edbd23f0e82328ece">caffe2.python.dataio.Writer.commit</a></div><div class="ttdeci">def commit(self, finish_net)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00200">dataio.py:200</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_writer_html"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_writer.html">caffe2.python.dataio.Writer</a></div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00145">dataio.py:145</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_pipe_html_aeb582e1dc5b200f59799a85c7162aed9"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_pipe.html#aeb582e1dc5b200f59799a85c7162aed9">caffe2.python.dataio.Pipe._num_readers</a></div><div class="ttdeci">_num_readers</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00261">dataio.py:261</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_writer_html_a9d70c665219c46da12b0cfafdae8060a"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a9d70c665219c46da12b0cfafdae8060a">caffe2.python.dataio.Writer._schema</a></div><div class="ttdeci">_schema</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00155">dataio.py:155</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_counter_reader_html"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_counter_reader.html">caffe2.python.dataio.CounterReader</a></div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00296">dataio.py:296</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_html_af3e6f05bed28a9630dc7a148cbb95098"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#af3e6f05bed28a9630dc7a148cbb95098">caffe2.python.dataio.ReaderWithLimit.counter</a></div><div class="ttdeci">counter</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00427">dataio.py:427</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1core_1_1_net_html"><div class="ttname"><a href="classcaffe2_1_1python_1_1core_1_1_net.html">caffe2.python.core.Net</a></div><div class="ttdef"><b>Definition:</b> <a href="core_8py_source.html#l01352">core.py:1352</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_html_ad6c5e15f2482793bebac92089d050bbc"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#ad6c5e15f2482793bebac92089d050bbc">caffe2.python.dataio.Reader.execution_step</a></div><div class="ttdeci">def execution_step(self, reader_net_name=None, external_should_stop=None)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00106">dataio.py:106</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base_html_a77b30909960c74b18df486f3e727da23"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a77b30909960c74b18df486f3e727da23">caffe2.python.dataio.ReaderWithLimitBase.check_limiter_condition</a></div><div class="ttdeci">def check_limiter_condition(self, stop_condition_net)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00389">dataio.py:389</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_html_a3e24214ecea0850be812df3296b3796b"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a3e24214ecea0850be812df3296b3796b">caffe2.python.dataio.Reader.reset</a></div><div class="ttdeci">def reset(self, net)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00092">dataio.py:92</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_html_a272213d199828188fff4e1f006285372"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a272213d199828188fff4e1f006285372">caffe2.python.dataio.Reader.read_record</a></div><div class="ttdeci">def read_record(self, read_net)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00100">dataio.py:100</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base_html_a969c829309710eced9c46c7372267f43"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a969c829309710eced9c46c7372267f43">caffe2.python.dataio.ReaderWithLimitBase.setup_limiter</a></div><div class="ttdeci">def setup_limiter(self, global_init_net, global_finish_net)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00379">dataio.py:379</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1schema_1_1_struct_html"><div class="ttname"><a href="classcaffe2_1_1python_1_1schema_1_1_struct.html">caffe2.python.schema.Struct</a></div><div class="ttdef"><b>Definition:</b> <a href="schema_8py_source.html#l00278">schema.py:278</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit_html"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html">caffe2.python.dataio.ReaderWithTimeLimit</a></div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00451">dataio.py:451</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base_html_a947fb82fe17fd625685cad6f63874c50"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html#a947fb82fe17fd625685cad6f63874c50">caffe2.python.dataio.ReaderWithLimitBase.read_ex</a></div><div class="ttdeci">def read_ex(self, local_init_net, local_finish_net)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00340">dataio.py:340</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base_html"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_base.html">caffe2.python.dataio.ReaderWithLimitBase</a></div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00315">dataio.py:315</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_writer_html_a77a746b74ca09dd9aee005330cc6916d"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a77a746b74ca09dd9aee005330cc6916d">caffe2.python.dataio.Writer.write</a></div><div class="ttdeci">def write(self, writer_net, fields)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00160">dataio.py:160</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit_html_a70002bd48f0f4d114520015b12bfdfbd"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_limit.html#a70002bd48f0f4d114520015b12bfdfbd">caffe2.python.dataio.ReaderWithLimit.__init__</a></div><div class="ttdeci">def __init__(self, reader, num_iter=1)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00416">dataio.py:416</a></div></div>
<div class="ttc" id="namespacecaffe2_1_1python_1_1schema_html"><div class="ttname"><a href="namespacecaffe2_1_1python_1_1schema.html">caffe2.python.schema</a></div><div class="ttdef"><b>Definition:</b> <a href="schema_8py_source.html#l00001">schema.py:1</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_writer_html_a8c0252988fae6e79ed51fa5407e244f6"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_writer.html#a8c0252988fae6e79ed51fa5407e244f6">caffe2.python.dataio.Writer.write_ex</a></div><div class="ttdeci">def write_ex(self, fields, local_init_net, local_finish_net, stop_blob)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00182">dataio.py:182</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit_html_a0ff62afd3c822f5bd0cfd7ba5c1b859a"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader_with_time_limit.html#a0ff62afd3c822f5bd0cfd7ba5c1b859a">caffe2.python.dataio.ReaderWithTimeLimit.duration</a></div><div class="ttdeci">duration</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00471">dataio.py:471</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_reader_html_a8d8909ca962957a5eb60f9d4007ef745"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_reader.html#a8d8909ca962957a5eb60f9d4007ef745">caffe2.python.dataio.Reader.setup_ex</a></div><div class="ttdeci">def setup_ex(self, init_net, finish_net)</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00050">dataio.py:50</a></div></div>
<div class="ttc" id="classcaffe2_1_1python_1_1dataio_1_1_counter_reader_html_aef4fa9db5d30aa1f08f744490ce8021f"><div class="ttname"><a href="classcaffe2_1_1python_1_1dataio_1_1_counter_reader.html#aef4fa9db5d30aa1f08f744490ce8021f">caffe2.python.dataio.CounterReader.counter</a></div><div class="ttdeci">counter</div><div class="ttdef"><b>Definition:</b> <a href="dataio_8py_source.html#l00300">dataio.py:300</a></div></div>
</div><!-- fragment --></div><!-- contents -->
<!-- HTML footer for doxygen 1.8.14-->
<!-- start footer part -->
<hr class="footer"/><address class="footer"><small>
Generated on Thu Apr 19 2018 13:04:04 for Caffe2 - Python API by  <a href="http://www.doxygen.org/index.html">
<img class="footer" src="doxygen.png" alt="doxygen"/>
</a> 1.8.11
</small></address>
<div class="footerContainer">
<div id="footer_wrap" class="wrapper footerWrapper">
<div class="footerBlocks">
<div id="fb_oss" class="footerSection fbOpenSourceFooter">
<svg class="facebookOSSLogoSvg" viewBox="0 0 1133.9 1133.9" x="0px" y="0px" height=50 width=50>
<g>
<path class="logoRing outerRing" d="M 498.3 3.7 c 153.6 88.9 307.3 177.7 461.1 266.2 c 7.6 4.4 10.3 9.1 10.3 17.8 c -0.3 179.1 -0.2 358.3 0 537.4 c 0 8.1 -2.4 12.8 -9.7 17.1 c -154.5 88.9 -308.8 178.1 -462.9 267.5 c -9 5.2 -15.5 5.3 -24.6 0.1 c -153.9 -89.2 -307.9 -178 -462.1 -266.8 C 3 838.8 0 833.9 0 825.1 c 0.3 -179.1 0.2 -358.3 0 -537.4 c 0 -8.6 2.6 -13.6 10.2 -18 C 164.4 180.9 318.4 92 472.4 3 C 477 -1.5 494.3 -0.7 498.3 3.7 Z M 48.8 555.3 c 0 79.9 0.2 159.9 -0.2 239.8 c -0.1 10 3 15.6 11.7 20.6 c 137.2 78.8 274.2 157.8 411 237.3 c 9.9 5.7 17 5.7 26.8 0.1 c 137.5 -79.8 275.2 -159.2 412.9 -238.5 c 7.4 -4.3 10.5 -8.9 10.5 -17.8 c -0.3 -160.2 -0.3 -320.5 0 -480.7 c 0 -8.8 -2.8 -13.6 -10.3 -18 C 772.1 218 633.1 137.8 494.2 57.4 c -6.5 -3.8 -11.5 -4.5 -18.5 -0.5 C 336.8 137.4 197.9 217.7 58.8 297.7 c -7.7 4.4 -10.2 9.2 -10.2 17.9 C 48.9 395.5 48.8 475.4 48.8 555.3 Z" />
<path class="logoRing middleRing" d="M 184.4 555.9 c 0 -33.3 -1 -66.7 0.3 -100 c 1.9 -48 24.1 -86 64.7 -110.9 c 54.8 -33.6 110.7 -65.5 167 -96.6 c 45.7 -25.2 92.9 -24.7 138.6 1 c 54.4 30.6 108.7 61.5 162.2 93.7 c 44 26.5 67.3 66.8 68 118.4 c 0.9 63.2 0.9 126.5 0 189.7 c -0.7 50.6 -23.4 90.7 -66.6 116.9 c -55 33.4 -110.8 65.4 -167.1 96.5 c -43.4 24 -89 24.2 -132.3 0.5 c -57.5 -31.3 -114.2 -64 -170 -98.3 c -41 -25.1 -62.9 -63.7 -64.5 -112.2 C 183.5 621.9 184.3 588.9 184.4 555.9 Z M 232.9 556.3 c 0 29.5 0.5 59.1 -0.1 88.6 c -0.8 39.2 16.9 67.1 50.2 86.2 c 51.2 29.4 102.2 59.2 153.4 88.4 c 31.4 17.9 63.6 18.3 95 0.6 c 53.7 -30.3 107.1 -61.2 160.3 -92.5 c 29.7 -17.5 45 -44.5 45.3 -78.8 c 0.6 -61.7 0.5 -123.5 0 -185.2 c -0.3 -34.4 -15.3 -61.5 -44.9 -79 C 637.7 352.6 583 320.8 527.9 290 c -27.5 -15.4 -57.2 -16.1 -84.7 -0.7 c -56.9 31.6 -113.4 64 -169.1 97.6 c -26.4 15.9 -40.7 41.3 -41.1 72.9 C 232.6 491.9 232.9 524.1 232.9 556.3 Z" />
<path class="logoRing innerRing" d="M 484.9 424.4 c 69.8 -2.8 133.2 57.8 132.6 132 C 617 630 558.5 688.7 484.9 689.1 c -75.1 0.4 -132.6 -63.6 -132.7 -132.7 C 352.1 485 413.4 421.5 484.9 424.4 Z M 401.3 556.7 c -3.4 37.2 30.5 83.6 83 84.1 c 46.6 0.4 84.8 -37.6 84.9 -84 c 0.1 -46.6 -37.2 -84.4 -84.2 -84.6 C 432.2 472.1 397.9 518.3 401.3 556.7 Z" />
</g>
</svg>
<h2>Facebook Open Source</h2>
</div>
<div class="footerSection">
<a class="footerLink" href="https://code.facebook.com/projects/" target="_blank">Open Source Projects</a>
<a class="footerLink" href="https://github.com/facebook/" target="_blank">GitHub</a>
<a class="footerLink" href="https://twitter.com/fbOpenSource" target="_blank">Twitter</a>
</div>
<div class="footerSection rightAlign">
<a class="footerLink" href="https://github.com/caffe2/caffe2" target="_blank">Contribute to this project on GitHub</a>
</div>
</div>
</div>
</div>
<script type="text/javascript" src="/js/jekyll-link-anchors.js"></script>
<script>
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
ga('create', '{{ site.gacode }}', 'auto');
ga('send', 'pageview');
</script>
</body>
</html>