Tensorflowtts: Fast Speech 2 Inference

[DanBigioi]

Hi! I'm getting some errors when trying to run inference using fast speech 2. This is the code I'm using:

//initialize fastspeech2 model.
with open('./examples/fastspeech2/conf/fastspeech2.v1.yaml') as f:
    config = yaml.load(f, Loader=yaml.Loader)
config = FastSpeech2Config(**config["fastspeech_params"])
fastspeech2 = TFFastSpeech2(config=config, name="fastspeech2")
fastspeech2._build()
fastspeech2.load_weights("C://Users//dabigioi//Downloads//model-150000 (1).h5")


//initialize melgan model
with open('./examples/melgan/conf/melgan.v1.yaml') as f:
    melgan_config = yaml.load(f, Loader=yaml.Loader)
melgan_config = MelGANGeneratorConfig(**melgan_config["generator_params"])
melgan = TFMelGANGenerator(config=melgan_config, name='melgan_generator')
melgan._build()
melgan.load_weights("C://Users//dabigioi//Downloads//generator-1500000.h5")


//fastspeech2 inference
processor = LJSpeechProcessor(None, cleaner_names="english_cleaners")
input_text = "This is what it should actually sound like. Still doesnt sound great, but its a step in the right direction."
input_ids = processor.text_to_sequence(input_text)

mel_before, masked_mel_after, duration_outputs = fastspeech2.inference(
    input_ids=tf.expand_dims(tf.convert_to_tensor(input_ids, dtype=tf.int32), 0),
    attention_mask=tf.math.not_equal(tf.expand_dims(tf.convert_to_tensor(input_ids, dtype=tf.int32), 0), 0),
    speaker_ids=tf.convert_to_tensor([0], dtype=tf.int32),
    speed_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
)

And this is the error I get:

ValueError                                Traceback (most recent call last)
~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\eager\function.py in _convert_inputs_to_signature(inputs, input_signature, flat_input_signature)
   2279         expand_composites=True,
-> 2280         check_types=False)  # lists are convert to tuples for `tf.data`.
   2281   except ValueError:

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\util\nest.py in flatten_up_to(shallow_tree, input_tree, check_types, expand_composites)
    931                            check_types=check_types,
--> 932                            expand_composites=expand_composites)
    933   # Discard paths returned by _yield_flat_up_to.

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\util\nest.py in assert_shallow_structure(shallow_tree, input_tree, check_types, expand_composites)
    834             _STRUCTURES_HAVE_MISMATCHING_LENGTHS.format(
--> 835                 input_length=len(input_tree), shallow_length=len(shallow_tree)))
    836       elif len(input_tree) < len(shallow_tree):

ValueError: The two structures don't have the same sequence length. Input structure has length 4, while shallow structure has length 6.

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-116-a80e8fc6bb76> in <module>
      7     attention_mask=tf.math.not_equal(tf.expand_dims(tf.convert_to_tensor(input_ids, dtype=tf.int32), 0), 0),
      8     speaker_ids=tf.convert_to_tensor([0], dtype=tf.int32),
----> 9     speed_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
     10 )

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\eager\def_function.py in __call__(self, *args, **kwds)
    578         xla_context.Exit()
    579     else:
--> 580       result = self._call(*args, **kwds)
    581 
    582     if tracing_count == self._get_tracing_count():

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\eager\def_function.py in _call(self, *args, **kwds)
    616       # In this case we have not created variables on the first call. So we can
    617       # run the first trace but we should fail if variables are created.
--> 618       results = self._stateful_fn(*args, **kwds)
    619       if self._created_variables:
    620         raise ValueError("Creating variables on a non-first call to a function"

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\eager\function.py in __call__(self, *args, **kwargs)
   2417     """Calls a graph function specialized to the inputs."""
   2418     with self._lock:
-> 2419       graph_function, args, kwargs = self._maybe_define_function(args, kwargs)
   2420     return graph_function._filtered_call(args, kwargs)  # pylint: disable=protected-access
   2421 

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\eager\function.py in _maybe_define_function(self, args, kwargs)
   2733     if self.input_signature is None or args is not None or kwargs is not None:
   2734       args, kwargs = self._function_spec.canonicalize_function_inputs(
-> 2735           *args, **kwargs)
   2736 
   2737     cache_key = self._cache_key(args, kwargs)

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\eager\function.py in canonicalize_function_inputs(self, *args, **kwargs)
   2236           inputs,
   2237           self._input_signature,
-> 2238           self._flat_input_signature)
   2239       return inputs, {}
   2240 

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\eager\function.py in _convert_inputs_to_signature(inputs, input_signature, flat_input_signature)
   2282     raise ValueError("Structure of Python function inputs does not match "
   2283                      "input_signature:\n%s" %
-> 2284                      format_error_message(inputs, input_signature))
   2285 
   2286   need_packing = False

ValueError: Structure of Python function inputs does not match input_signature:
  inputs: (
    tf.Tensor(
[[57 45 46 56 11 46 56 11 60 45 38 57 11 46 57 11 56 45 52 58 49 41 11 38
  40 57 58 38 49 49 62 11 56 52 58 51 41 11 49 46 48 42  7 11 56 57 46 49
  49 11 41 52 42 56 51 57 11 56 52 58 51 41 11 44 55 42 38 57  6 11 39 58
  57 11 46 57 56 11 38 11 56 57 42 53 11 46 51 11 57 45 42 11 55 46 44 45
  57 11 41 46 55 42 40 57 46 52 51  7]], shape=(1, 108), dtype=int32),
    tf.Tensor(
[[ True  True  True  True  True  True  True  True  True  True  True  True
   True  True  True  True  True  True  True  True  True  True  True  True
   True  True  True  True  True  True  True  True  True  True  True  True
   True  True  True  True  True  True  True  True  True  True  True  True
   True  True  True  True  True  True  True  True  True  True  True  True
   True  True  True  True  True  True  True  True  True  True  True  True
   True  True  True  True  True  True  True  True  True  True  True  True
   True  True  True  True  True  True  True  True  True  True  True  True
   True  True  True  True  True  True  True  True  True  True  True  True]], shape=(1, 108), dtype=bool),
    tf.Tensor([0], shape=(1,), dtype=int32),
    tf.Tensor([1.], shape=(1,), dtype=float32))
  input_signature: (
    TensorSpec(shape=(None, None), dtype=tf.int32, name=None),
    TensorSpec(shape=(None, None), dtype=tf.bool, name=None),
    TensorSpec(shape=(None,), dtype=tf.int32, name=None),
    TensorSpec(shape=(None,), dtype=tf.float32, name=None),
    TensorSpec(shape=(None,), dtype=tf.float32, name=None),
    TensorSpec(shape=(None,), dtype=tf.float32, name=None))

I'm not really sure whats going on, so any sort of help is appreciated! Thank you!

[manmay-nakhashi]

@DanBigioi you need to add two more params in inference call f0_ratios, energy_ratios
f0_ratios=tf.ones(shape=[tf.shape(input_ids)[0]], dtype=tf.float32),
energy_ratios=tf.ones(shape=[tf.shape(input_ids)[0]], dtype=tf.float32)

[DanBigioi]

@DanBigioi you need to add two more params in inference call f0_ratios, energy_ratios
f0_ratios=tf.ones(shape=[tf.shape(input_ids)[0]], dtype=tf.float32),
energy_ratios=tf.ones(shape=[tf.shape(input_ids)[0]], dtype=tf.float32)

Oh I see. Thanks a lot for the help!

[DanBigioi]

@DanBigioi you need to add two more params in inference call f0_ratios, energy_ratios
f0_ratios=tf.ones(shape=[tf.shape(input_ids)[0]], dtype=tf.float32),
energy_ratios=tf.ones(shape=[tf.shape(input_ids)[0]], dtype=tf.float32)

@manmay-nakhashi I'm now getting an invalid argument error to do with the energy_ratio param. Have you any idea what could be causing it?

InvalidArgumentError                      Traceback (most recent call last)
<ipython-input-27-32971132e34d> in <module>
      5     speed_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
      6     f0_ratios=tf.ones(shape=[tf.shape(input_ids)[0]], dtype=tf.float32),
----> 7     energy_ratios=tf.ones(shape=[tf.shape(input_ids)[0]], dtype=tf.float32)
      8 )

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\eager\def_function.py in __call__(self, *args, **kwds)
    578         xla_context.Exit()
    579     else:
--> 580       result = self._call(*args, **kwds)
    581 
    582     if tracing_count == self._get_tracing_count():

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\eager\def_function.py in _call(self, *args, **kwds)
    616       # In this case we have not created variables on the first call. So we can
    617       # run the first trace but we should fail if variables are created.
--> 618       results = self._stateful_fn(*args, **kwds)
    619       if self._created_variables:
    620         raise ValueError("Creating variables on a non-first call to a function"

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\eager\function.py in __call__(self, *args, **kwargs)
   2418     with self._lock:
   2419       graph_function, args, kwargs = self._maybe_define_function(args, kwargs)
-> 2420     return graph_function._filtered_call(args, kwargs)  # pylint: disable=protected-access
   2421 
   2422   @property

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\eager\function.py in _filtered_call(self, args, kwargs)
   1663          if isinstance(t, (ops.Tensor,
   1664                            resource_variable_ops.BaseResourceVariable))),
-> 1665         self.captured_inputs)
   1666 
   1667   def _call_flat(self, args, captured_inputs, cancellation_manager=None):

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\eager\function.py in _call_flat(self, args, captured_inputs, cancellation_manager)
   1744       # No tape is watching; skip to running the function.
   1745       return self._build_call_outputs(self._inference_function.call(
-> 1746           ctx, args, cancellation_manager=cancellation_manager))
   1747     forward_backward = self._select_forward_and_backward_functions(
   1748         args,

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\eager\function.py in call(self, ctx, args, cancellation_manager)
    596               inputs=args,
    597               attrs=attrs,
--> 598               ctx=ctx)
    599         else:
    600           outputs = execute.execute_with_cancellation(

~\AppData\Local\Continuum\anaconda3\envs\Fast Speech\lib\site-packages\tensorflow\python\eager\execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
     58     ctx.ensure_initialized()
     59     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
---> 60                                         inputs, attrs, num_outputs)
     61   except core._NotOkStatusException as e:
     62     if name is not None:

InvalidArgumentError: 2 root error(s) found.
  (0) Invalid argument:  slice index 1 of dimension 0 out of bounds.
     [[{{node length_regulator/while/body/_1/strided_slice}}]]
     [[mul_3/_120]]
  (1) Invalid argument:  slice index 1 of dimension 0 out of bounds.
     [[{{node length_regulator/while/body/_1/strided_slice}}]]
0 successful operations.
0 derived errors ignored. [Op:__inference_inference_9250]

Function call stack:
inference -> inference

[dathudeptrai]

@DanBigioi can you check our colab notebook, it should run fine :D (https://colab.research.google.com/drive/1akxtrLZHKuMiQup00tzO2olCaN-y3KiD?usp=sharing)

[DanBigioi]

@DanBigioi can you check our colab notebook, it should run fine :D (https://colab.research.google.com/drive/1akxtrLZHKuMiQup00tzO2olCaN-y3KiD?usp=sharing)

Thanks a ton!! Didnt know that notebook existed :D That solves the issue, cheers!!