Probability: It is not possible to save a PixelCNN model

Created on 24 Feb 2020  路  5Comments  路  Source: tensorflow/probability

Code:

imgsize=64

dist = tfp.distributions.PixelCNN(
    image_shape=(imgsize, imgsize, 3),
    num_resnet=1,
    num_hierarchies=2,
    num_filters=160,
    num_logistic_mix=5,
    dropout_p=.3,
    high=1, low=0
)

image_input = layers.Input(shape=(imgsize, imgsize, 3))
log_prob = dist.log_prob(image_input)

model = tf.keras.Model(inputs=image_input, outputs=log_prob)
model.add_loss(-tf.reduce_mean(log_prob))

model.compile(
    optimizer=tf.keras.optimizers.Adam(.0001),
    metrics=[])

model_json = model.to_json()

Error:

NotImplementedError                       Traceback (most recent call last)
<ipython-input-12-d9399ed0ac2a> in <module>
----> 1 model_json = model.to_json()

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in to_json(self, **kwargs)
   1252         A JSON string.
   1253     """
-> 1254     model_config = self._updated_config()
   1255     return json.dumps(
   1256         model_config, default=serialization.get_json_type, **kwargs)

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in _updated_config(self)
   1230     from tensorflow.python.keras import __version__ as keras_version  # pylint: disable=g-import-not-at-top
   1231 
-> 1232     config = self.get_config()
   1233     model_config = {
   1234         'class_name': self.__class__.__name__,

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in get_config(self)
    916     if not self._is_graph_network:
    917       raise NotImplementedError
--> 918     return copy.deepcopy(get_network_config(self))
    919 
    920   @classmethod

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in get_network_config(network, serialize_layer_fn)
   1991           filtered_inbound_nodes.append(node_data)
   1992 
-> 1993     layer_config = serialize_layer_fn(layer)
   1994     layer_config['name'] = layer.name
   1995     layer_config['inbound_nodes'] = filtered_inbound_nodes

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/utils/generic_utils.py in serialize_keras_object(instance)
    196 
    197   if hasattr(instance, 'get_config'):
--> 198     config = instance.get_config()
    199     serialization_config = {}
    200     for key, item in config.items():

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer.py in get_config(self)
    497     # or that `get_config` has been overridden:
    498     if len(extra_args) > 1 and hasattr(self.get_config, '_is_default'):
--> 499       raise NotImplementedError('Layers with arguments in `__init__` must '
    500                                 'override `get_config`.')
    501     return config

NotImplementedError: Layers with arguments in `__init__` must override `get_config`.
picture AliKhoda
👀1

All 5 comments

Would adding this to "tensorflow_probability/python/distributions/pixel_cnn.py" solve the issue?

def get_config(self):

  config = super().get_config().copy()
  config.update({
      '_dropout_p': self._dropout_p,
      '_num_resnet': self._num_resnet,
      '_num_hierarchies': self._num_hierarchies,
      '_num_filters': self._num_filters,
      '_num_logistic_mix': self._num_logistic_mix,
      '_receptive_field_dims': self._receptive_field_dims,
      '_resnet_activation': self._resnet_activation
      '_layer_wrapper': self._layer_wrapper
  })
  return config
AliKhoda picture AliKhoda on 24 Feb 2020
😕1

It does fix the saving, but not the loading:

Code:

loaded_model = tf.keras.models.model_from_json(model_json)

Error:

ValueError                                Traceback (most recent call last)
<ipython-input-12-3120867be721> in <module>
      2     model_json = json_file.read()
      3 
----> 4 loaded_model = tf.keras.models.model_from_json(model_json)
      5 loaded_model.summary()
      6 

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/saving/model_config.py in model_from_json(json_string, custom_objects)
     94   config = json.loads(json_string)
     95   from tensorflow.python.keras.layers import deserialize  # pylint: disable=g-import-not-at-top
---> 96   return deserialize(config, custom_objects=custom_objects)

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/layers/serialization.py in deserialize(config, custom_objects)
    104       module_objects=globs,
    105       custom_objects=custom_objects,
--> 106       printable_module_name='layer')

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
    301             custom_objects=dict(
    302                 list(_GLOBAL_CUSTOM_OBJECTS.items()) +
--> 303                 list(custom_objects.items())))
    304       with CustomObjectScope(custom_objects):
    305         return cls.from_config(cls_config)

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in from_config(cls, config, custom_objects)
    935     """
    936     input_tensors, output_tensors, created_layers = reconstruct_from_config(
--> 937         config, custom_objects)
    938     model = cls(inputs=input_tensors, outputs=output_tensors,
    939                 name=config.get('name'))

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in reconstruct_from_config(config, custom_objects, created_layers)
   1891   # First, we create all layers and enqueue nodes to be processed
   1892   for layer_data in config['layers']:
-> 1893     process_layer(layer_data)
   1894   # Then we process nodes in order of layer depth.
   1895   # Nodes that cannot yet be processed (if the inbound node

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in process_layer(layer_data)
   1873       from tensorflow.python.keras.layers import deserialize as deserialize_layer  # pylint: disable=g-import-not-at-top
   1874 
-> 1875       layer = deserialize_layer(layer_data, custom_objects=custom_objects)
   1876       created_layers[layer_name] = layer
   1877 

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/layers/serialization.py in deserialize(config, custom_objects)
    104       module_objects=globs,
    105       custom_objects=custom_objects,
--> 106       printable_module_name='layer')

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
    290     config = identifier
    291     (cls, cls_config) = class_and_config_for_serialized_keras_object(
--> 292         config, module_objects, custom_objects, printable_module_name)
    293 
    294     if hasattr(cls, 'from_config'):

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/utils/generic_utils.py in class_and_config_for_serialized_keras_object(config, module_objects, custom_objects, printable_module_name)
    248     cls = module_objects.get(class_name)
    249     if cls is None:
--> 250       raise ValueError('Unknown ' + printable_module_name + ': ' + class_name)
    251 
    252   cls_config = config['config']

ValueError: Unknown layer: _PixelCNNNetwork
AliKhoda picture AliKhoda on 24 Feb 2020
😕1

Code:

loaded_model = tf.keras.models.model_from_json(model_json, custom_objects={'_PixelCNNNetwork': tfp.distributions.PixelCNN})

Error:

TypeError                                 Traceback (most recent call last)
<ipython-input-15-e742f0ee369b> in <module>
      2     model_json = json_file.read()
      3 
----> 4 loaded_model = tf.keras.models.model_from_json(model_json, custom_objects={'_PixelCNNNetwork': tfp.distributions.PixelCNN})
      5 loaded_model.summary()
      6 

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/saving/model_config.py in model_from_json(json_string, custom_objects)
     94   config = json.loads(json_string)
     95   from tensorflow.python.keras.layers import deserialize  # pylint: disable=g-import-not-at-top
---> 96   return deserialize(config, custom_objects=custom_objects)

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/layers/serialization.py in deserialize(config, custom_objects)
    104       module_objects=globs,
    105       custom_objects=custom_objects,
--> 106       printable_module_name='layer')

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
    301             custom_objects=dict(
    302                 list(_GLOBAL_CUSTOM_OBJECTS.items()) +
--> 303                 list(custom_objects.items())))
    304       with CustomObjectScope(custom_objects):
    305         return cls.from_config(cls_config)

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in from_config(cls, config, custom_objects)
    936     """
    937     input_tensors, output_tensors, created_layers = reconstruct_from_config(
--> 938         config, custom_objects)
    939     model = cls(inputs=input_tensors, outputs=output_tensors,
    940                 name=config.get('name'))

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in reconstruct_from_config(config, custom_objects, created_layers)
   1892   # First, we create all layers and enqueue nodes to be processed
   1893   for layer_data in config['layers']:
-> 1894     process_layer(layer_data)
   1895   # Then we process nodes in order of layer depth.
   1896   # Nodes that cannot yet be processed (if the inbound node

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in process_layer(layer_data)
   1874       from tensorflow.python.keras.layers import deserialize as deserialize_layer  # pylint: disable=g-import-not-at-top
   1875 
-> 1876       layer = deserialize_layer(layer_data, custom_objects=custom_objects)
   1877       created_layers[layer_name] = layer
   1878 

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/layers/serialization.py in deserialize(config, custom_objects)
    104       module_objects=globs,
    105       custom_objects=custom_objects,
--> 106       printable_module_name='layer')

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
    310       custom_objects = custom_objects or {}
    311       with CustomObjectScope(custom_objects):
--> 312         return cls(**cls_config)
    313   elif isinstance(identifier, six.string_types):
    314     object_name = identifier

TypeError: __init__() got an unexpected keyword argument 'trainable'
AliKhoda picture AliKhoda on 24 Feb 2020
😕1

Modifications:

def get_config(self):

    config = super().get_config().copy()
    config.update({
      'dropout_p': self._dropout_p,
      'num_resnet': self._num_resnet,
      'num_hierarchies': self._num_hierarchies,
      'num_filters': self._num_filters,
      'num_logistic_mix': self._num_logistic_mix,
      'receptive_field_dims': self._receptive_field_dims,
      'resnet_activation': self._resnet_activation,
    })
    return config

```
def __init__(
self,
dropout_p=0.5,
num_resnet=5,
num_hierarchies=3,
num_filters=160,
num_logistic_mix=10,
receptive_field_dims=(3, 3),
resnet_activation='concat_elu',
use_weight_norm=True,
use_data_init=True,
dtype=tf.float32,
**kwargs):

`super(_PixelCNNNetwork, self).__init__(dtype=dtype, **kwargs)`

Code:
-----------------------------

from tensorflow_probability.python.distributions.pixel_cnn import _PixelCNNNetwork

loaded_model = tf.keras.models.model_from_json(model_json, custom_objects={'_PixelCNNNetwork': _PixelCNNNetwork})

Error:
---------------------------------------------------------------------------

InvalidArgumentError Traceback (most recent call last)
in
4 from tensorflow_probability.python.distributions.pixel_cnn import _PixelCNNNetwork
5
----> 6 loaded_model = tf.keras.models.model_from_json(model_json, custom_objects={'_PixelCNNNetwork': _PixelCNNNetwork})
7 loaded_model.summary()
8

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/saving/model_config.py in model_from_json(json_string, custom_objects)
94 config = json.loads(json_string)
95 from tensorflow.python.keras.layers import deserialize # pylint: disable=g-import-not-at-top
---> 96 return deserialize(config, custom_objects=custom_objects)

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/layers/serialization.py in deserialize(config, custom_objects)
104 module_objects=globs,
105 custom_objects=custom_objects,
--> 106 printable_module_name='layer')

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
301 custom_objects=dict(
302 list(_GLOBAL_CUSTOM_OBJECTS.items()) +
--> 303 list(custom_objects.items())))
304 with CustomObjectScope(custom_objects):
305 return cls.from_config(cls_config)

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in from_config(cls, config, custom_objects)
936 """
937 input_tensors, output_tensors, created_layers = reconstruct_from_config(
--> 938 config, custom_objects)
939 model = cls(inputs=input_tensors, outputs=output_tensors,
940 name=config.get('name'))

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in reconstruct_from_config(config, custom_objects, created_layers)
1902 if layer in unprocessed_nodes:
1903 for node_data in unprocessed_nodes.pop(layer):
-> 1904 process_node(layer, node_data)
1905
1906 input_tensors = []

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in process_node(layer, node_data)
1850 if not isinstance(input_tensors, dict) and len(flat_input_tensors) == 1:
1851 input_tensors = flat_input_tensors[0]
-> 1852 output_tensors = layer(input_tensors, **kwargs)
1853
1854 # Update node index map.

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer.py in __call__(self, inputs, args, *kwargs)
746 # Build layer if applicable (if the build method has been
747 # overridden).
--> 748 self._maybe_build(inputs)
749 cast_inputs = self._maybe_cast_inputs(inputs)
750

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer.py in _maybe_build(self, inputs)
2114 # operations.
2115 with tf_utils.maybe_init_scope(self):
-> 2116 self.build(input_shapes)
2117 # We must set self.built since user defined build functions are not
2118 # constrained to set self.built.

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_probability/python/distributions/pixel_cnn.py in build(self, input_shape)
950 inputs = (image_input if conditional_input is None
951 else [image_input, conditional_input])
--> 952 self._network = tf.keras.Model(inputs=inputs, outputs=outputs)
953 super(_PixelCNNNetwork, self).build(input_shape)
954

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/training.py in __init__(self, args, *kwargs)
144
145 def __init__(self, args, *kwargs):
--> 146 super(Model, self).__init__(args, *kwargs)
147 _keras_api_gauge.get_cell('model').set(True)
148 # initializing _distribution_strategy here since it is possible to call

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in __init__(self, args, *kwargs)
168 'inputs' in kwargs and 'outputs' in kwargs):
169 # Graph network
--> 170 self._init_graph_network(args, *kwargs)
171 else:
172 # Subclassed network

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/training/tracking/base.py in _method_wrapper(self, args, *kwargs)
455 self._self_setattr_tracking = False # pylint: disable=protected-access
456 try:
--> 457 result = method(self, args, *kwargs)
458 finally:
459 self._self_setattr_tracking = previous_value # pylint: disable=protected-access

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/network.py in _init_graph_network(self, inputs, outputs, name, *kwargs)
271
272 if any(not hasattr(tensor, '_keras_history') for tensor in self.outputs):
--> 273 base_layer_utils.create_keras_history(self._nested_outputs)
274
275 self._base_init(name=name, *kwargs)

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer_utils.py in create_keras_history(tensors)
185 keras_tensors: The Tensors found that came from a Keras Layer.
186 """
--> 187 _, created_layers = _create_keras_history_helper(tensors, set(), [])
188 return created_layers
189

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer_utils.py in _create_keras_history_helper(tensors, processed_ops, created_layers)
247 constants[i] = backend.function([], op_input)([])
248 processed_ops, created_layers = _create_keras_history_helper(
--> 249 layer_inputs, processed_ops, created_layers)
250 name = op.name
251 node_def = op.node_def.SerializeToString()

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer_utils.py in _create_keras_history_helper(tensors, processed_ops, created_layers)
247 constants[i] = backend.function([], op_input)([])
248 processed_ops, created_layers = _create_keras_history_helper(
--> 249 layer_inputs, processed_ops, created_layers)
250 name = op.name
251 node_def = op.node_def.SerializeToString()

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer_utils.py in _create_keras_history_helper(tensors, processed_ops, created_layers)
247 constants[i] = backend.function([], op_input)([])
248 processed_ops, created_layers = _create_keras_history_helper(
--> 249 layer_inputs, processed_ops, created_layers)
250 name = op.name
251 node_def = op.node_def.SerializeToString()

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer_utils.py in _create_keras_history_helper(tensors, processed_ops, created_layers)
247 constants[i] = backend.function([], op_input)([])
248 processed_ops, created_layers = _create_keras_history_helper(
--> 249 layer_inputs, processed_ops, created_layers)
250 name = op.name
251 node_def = op.node_def.SerializeToString()

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/engine/base_layer_utils.py in _create_keras_history_helper(tensors, processed_ops, created_layers)
245 else:
246 with ops.init_scope():
--> 247 constants[i] = backend.function([], op_input)([])
248 processed_ops, created_layers = _create_keras_history_helper(
249 layer_inputs, processed_ops, created_layers)

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/backend.py in __call__(self, inputs)
3733 return nest.pack_sequence_as(
3734 self._outputs_structure,
-> 3735 [x._numpy() for x in outputs], # pylint: disable=protected-access
3736 expand_composites=True)
3737

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/keras/backend.py in (.0)
3733 return nest.pack_sequence_as(
3734 self._outputs_structure,
-> 3735 [x._numpy() for x in outputs], # pylint: disable=protected-access
3736 expand_composites=True)
3737

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/tensorflow_core/python/framework/ops.py in _numpy(self)
908 return self._numpy_internal()
909 except core._NotOkStatusException as e:
--> 910 six.raise_from(core._status_to_exception(e.code, e.message), None)
911
912 @property

~/anaconda3/envs/tf2-gpu/lib/python3.6/site-packages/six.py in raise_from(value, from_value)

InvalidArgumentError: Cannot convert a Tensor of dtype resource to a NumPy array.
```

AliKhoda picture AliKhoda on 24 Feb 2020
😕1

Solution: Not to save the architecture, but to save only the weights.

AliKhoda picture AliKhoda on 27 Feb 2020
Was this page helpful?
0 / 5 - 0 ratings

Related issues

xht033 picture xht033  路  5Comments
Hoeze picture Hoeze  路  3Comments
jalajthanaki picture jalajthanaki  路  3Comments
bkazi picture bkazi  路  5Comments
wolfiex picture wolfiex  路  3Comments