3 from __future__
import absolute_import
4 from __future__
import division
5 from __future__
import print_function
6 from __future__
import unicode_literals
8 from caffe2.python import core, model_helper, schema, scope, utils
13 parameter_sharing_context,
20 from caffe2.proto
import caffe2_pb2
21 from future.utils
import viewitems, viewvalues
27 logger = logging.getLogger(__name__)
32 Model helper for building models on top of layers abstractions. 34 Each layer is the abstraction that is higher level than Operator. Layer 35 is responsible for ownership of it's own parameters and can easily be 36 instantiated in multiple nets possible with different sets of ops. 37 As an example: one can easily instantiate predict and train nets from 38 the same set of layers, where predict net will have subset of the 39 operators from train net. 42 def __init__(self, name, input_feature_schema, trainer_extra_schema,
44 ''' TODO(amalevich): more documnetation on input args 47 super(LayerModelHelper, self).
__init__(name=name)
77 )
if not keep_blobs
else input_feature_schema.clone()
81 )
if not keep_blobs
else trainer_extra_schema.clone()
90 def clear_output_schema(self):
93 def set_initialize_params(self, initialize_params):
96 def add_metric_field(self, name, value):
97 assert name
not in self._metrics_schema.fields, (
98 "Try to add metric field twice: {}".format(name))
104 def _get_global_constant_initializer_op(
105 blob_name, array=
None, dtype=
None, initializer=
None 109 if array
is not None:
110 assert initializer
is None,\
111 "Only one from array and initializer should be specified" 113 array = np.array(array)
115 array = np.array(array, dtype=dtype)
119 if array.dtype == np.int32:
120 op_name =
'GivenTensorIntFill' 121 elif array.dtype == np.int64:
122 op_name =
'GivenTensorInt64Fill' 123 elif array.dtype == np.str:
124 op_name =
'GivenTensorStringFill' 125 elif array.dtype == np.bool:
126 op_name =
'GivenTensorBoolFill' 128 op_name =
'GivenTensorFill' 130 def initializer(blob_name):
131 return core.CreateOperator(
135 values=array.flatten().tolist()
138 assert initializer
is not None 139 initializer_op = initializer(blob_name)
140 return initializer_op
142 def add_global_constant(
143 self, name, array=
None, dtype=
None, initializer=
None 145 assert isinstance(name, six.string_types), (
146 'name should be a string as we are using it as map key')
150 "%s already added in global_constants" % name
151 blob_name = self.net.NextBlob(name)
153 initializer_op = LayerModelHelper._get_global_constant_initializer_op(
154 blob_name, array, dtype, initializer
157 "there is already a initializer op associated with blob %s" % \
162 def maybe_add_global_constant(self, name, *args, **kwargs):
170 LayerModelHelper._get_global_constant_initializer_op(
171 blob_name, *args, **kwargs
175 assert utils.OpAlmostEqual(
180 "conflict initializers for global constant %s, " \
181 "previous %s, now %s" % (
182 blob_name, str(initializer_op),
187 def _init_global_constants(self):
194 def _add_global_constants(self, init_net):
196 init_net._net.op.extend([initializer_op])
198 def create_init_net(self, name):
203 def _validate_param_shape(self, param_name, shape):
209 if shape != ref_shape:
211 "Got inconsistent shapes between shared parameters " 212 "when trying to map a blob in scope {0} to {1}. ref_shape : " 213 " {2}, shape : {3}".format(
214 scope.CurrentNameScope(), param_name, ref_shape, shape)
217 def create_param(self, param_name, shape, initializer, optimizer=None,
218 ps_param=
None, regularizer=
None):
220 param_name = str(param_name)
221 elif isinstance(param_name, six.string_types):
224 param_name = parameter_sharing_context.get_parameter_name(
227 raise "Unsupported type for param_name" 231 if len(initializer) == 1:
234 assert len(initializer) == 2
235 init_op_args = copy.deepcopy(initializer[1])
236 if shape
is not None:
237 assert 'shape' not in init_op_args
238 init_op_args.update({
'shape': shape})
240 initializer_op =
None 242 initializer_op = core.CreateOperator(
249 param = layers.LayerParameter(
250 parameter=param_blob,
251 initializer=initializer_op,
254 regularizer=regularizer
263 def next_layer_name(self, prefix):
264 base_name = core.ScopedName(prefix)
268 name = base_name +
'_auto_' + str(index)
271 self._layer_names.add(name)
274 def add_layer(self, layer):
275 self._layers.append(layer)
276 for param
in layer.get_parameters():
282 self.params.append(param.parameter)
283 if isinstance(param, layers.LayerParameter):
285 elif isinstance(param, ParameterInfo):
292 logger.info(
'regularization is unsupported for ParameterInfo object')
295 'unknown object type besides ParameterInfo and LayerParameter: {}' 303 return layer.output_schema
305 def get_parameter_blobs(self):
308 for param
in layer.get_parameters():
309 param_blobs.append(param.parameter)
313 def add_post_grad_net_modifiers(self, modifier):
316 assert isinstance(modifier, NetModifier),\
317 "{} has to be a NetModifier instance".format(modifier)
318 self._post_grad_net_modifiers.append(modifier)
320 def add_final_net_modifiers(self, modifier):
323 assert isinstance(modifier, NetModifier),\
324 "{} has to be a NetModifier instance".format(modifier)
325 self._final_net_modifiers.append(modifier)
332 def sequence_seed(self):
335 def store_seed(self, seed, sequence_seed=True):
341 def apply_seed(self, net):
346 def default_optimizer(self):
349 @default_optimizer.setter
350 def default_optimizer(self, optimizer):
354 def input_feature_schema(self):
358 def trainer_extra_schema(self):
364 Returns the schema that represents model output that should be used for 367 During the training/evaluation this schema will be appended to the 368 schema that represents model output. 373 def output_schema(self):
377 @output_schema.setter
378 def output_schema(self, schema):
383 def preproc_output_schema(self):
387 @preproc_output_schema.setter
388 def preproc_output_schema(self, schema):
394 assert self.
_loss is not None 398 def loss(self, loss):
399 assert self.
_loss is None 403 return self.
_loss is not None 405 def add_loss(self, loss, name='unnamed'):
406 assert loss
is not None,
"Added loss should not be None" 409 ),
"Added loss should be a scalar or a struct" 410 if self.
_loss is None:
413 prefix_base = name +
'_auto_' 416 while prefix
in self.
_loss:
417 prefix = prefix_base + str(index)
422 def add_output_schema(self, name, value):
423 assert value
is not None, \
424 'Added output schema {} should not be None'.format(name)
427 'Added output schema {} should be a scalar or a struct.\n\ 428 Now it is {}.'.format(name, type(value))
432 assert name
not in self._output_schema.fields, \
433 'Output Schema Field {} already exists'.format(name)
437 def add_trainer_extra_schema(self, trainer_extra_schema):
438 trainer_extra_record = schema.NewRecord(self.
net, trainer_extra_schema)
441 def __getattr__(self, layer):
442 def is_functional_layer(layer):
443 if core.IsOperator(layer):
445 elif layer.startswith(
'FunctionalLayer'):
450 def resolve_functional_layer(layer):
451 if core.IsOperator(layer):
453 elif layer.startswith(
'FunctionalLayer'):
454 return layer[len(
'FunctionalLayer'):]
457 '%s cannot be resolved as functional layer' % layer
460 if layer.startswith(
'__'):
461 raise AttributeError(layer)
464 if layers.layer_exists(layer):
465 def wrapper(*args, **kwargs):
466 new_layer = layers.create_layer(layer, self, *args, **kwargs)
467 if kwargs.get(
"output_to_metrics",
False):
468 new_layer.export_output_for_metrics()
469 if kwargs.get(
"params_to_metrics",
False):
470 new_layer.export_params_for_metrics()
473 elif is_functional_layer(layer):
477 layer = resolve_functional_layer(layer)
478 def wrapper(*args, **kwargs):
479 def apply_operator(net, in_record, out_record, **kwargs):
482 net.__getattr__(layer)(in_record.field_blobs(),
483 out_record.field_blobs(),
486 if 'name' not in kwargs:
487 kwargs[
'name'] = layer
489 new_layer = layers.create_layer(
491 self, *args, function=apply_operator,
495 if kwargs.get(
"output_to_metrics",
False):
496 new_layer.export_output_for_metrics()
497 if kwargs.get(
"params_to_metrics",
False):
498 new_layer.export_params_for_metrics()
504 "Trying to create non-registered layer: {}".format(layer))
510 def apply_regularizers_on_loss(
517 if regularizer
is None or regularizer.apply_after_optimizer:
519 assert isinstance(regularizer, Regularizer)
520 added_loss_blob = regularizer(train_net, train_init_net, param)
526 def apply_regularizers_after_optimizer(
534 if regularizer
is None or not regularizer.apply_after_optimizer:
536 assert isinstance(regularizer, Regularizer)
538 train_net, train_init_net, param, grad_map.get(str(param)))
540 def apply_post_grad_net_modifiers(
547 param_grad_map = {param: grad_map[param]
548 for param
in self.param_to_optim.keys()
if param
in grad_map}
551 modifier(trainer_net, trainer_init_net, param_grad_map,
552 blob_to_device=blob_to_device)
554 def apply_final_net_modifiers(
562 modifier(trainer_net, trainer_init_net, grad_map,
563 blob_to_device=blob_to_device)
565 def apply_optimizers(
572 CPU = core.DeviceOption(caffe2_pb2.CPU)
574 blob_to_device = blob_to_device
or {}
576 assert optimizer
is not None, \
577 "default optimizer must have been set in add_layer" 580 device = get_param_device(
582 grad_map.get(str(param)),
583 param_to_device=blob_to_device,
586 with core.DeviceScope(device):
588 train_net, train_init_net, param, grad_map.get(str(param)))
594 def NoOptim(self, *args, **kwargs):
598 def breakdown_map(self):
601 @breakdown_map.setter
602 def breakdown_map(self, breakdown_map):
605 assert isinstance(breakdown_map, dict)
606 assert all(isinstance(k, six.string_types)
for k
in breakdown_map)
607 assert sorted(list(breakdown_map.values())) == range(len(breakdown_map))
Module caffe2.python.optimizer.
def default_optimizer(self)
def add_global_constant(self, name, array=None, dtype=None, initializer=None)
def _validate_param_shape(self, param_name, shape)
global_constant_initializers
Module caffe2.python.layers.layers.
def _init_global_constants(self)
def add_loss(self, loss, name='unnamed')
def add_layer(self, layer)
def _add_global_constants(self, init_net)
def __init__(self, name, input_feature_schema, trainer_extra_schema, keep_blobs=False)
def create_init_net(self, name)