3 from __future__
import absolute_import
4 from __future__
import division
5 from __future__
import print_function
6 from __future__
import unicode_literals
19 Reader with persistent in-file cache. 22 cached_reader = CachedReader(reader) 23 build_cache_step = cached_reader.build_cache('/tmp/cache.db') 24 with LocalSession() as session: 25 session.run(build_cache_step) 27 Every time new reader is created, it's expected that build_cache will be 28 called before setup_ex and usage of the reader. build_cache will check 29 existence of provided file path and in case it's missing will initialize it 30 by reading data from original reader. All consequent attempts to read will 31 ignore original reader (i.e. no additional data will be read from it). 34 def __init__(self, reader, db_type='leveldb', name='cached_reader'):
35 super(CachedReader, self).__init__(reader.schema())
44 def setup_ex(self, init_net, finish_net):
45 assert self.
cache_path,
'build_cache must be called first' 48 self.
ds_reader = self.ds.reader(init_net, batch_size=100)
50 def read(self, read_net):
51 assert self.
ds_reader,
'setup must be called first' 52 return self.ds_reader.read(read_net)
57 def build_cache(self, cache_path, overwrite=False):
62 return core.execution_step(
'build_step', [])
68 copy_step = copy_tg.to_task().get_step()
72 return core.execution_step(
'build_cache', [init_net, copy_step, save_net])
74 def _init_dataset(self, init_net):
75 with core.NameScope(self.
name):
76 self.ds.init_empty(init_net)
78 def _save_to_file(self, net):
80 self.ds.content().field_blobs(),
88 def _load_from_file(self, net):
91 self.ds.content().field_blobs(),
def _init_dataset(self, init_net)
def _save_to_file(self, net)
def _load_from_file(self, net)