Keras: custom generator for fit_generator() that yields multiple inputs with different shapes

[bparaj]

I have a model which takes two inputs of different shapes. To use fit_generator(), I tried to write a generator which yields batches of samples and their corresponding labels but I get error messages.

In data_gen() defined below, I am unsure how the multiple inputs per sample (top_img, bot_img) should be packed. I tried list, tuple and numpy array but all gave errors.

Current version of Keras: 2.0.8

from keras.layers import Input, concatenate, Dense
from keras.models import Model
from keras.optimizers import Adam
import numpy as np

def data_gen(top_dim, bot_dim):
    """
    Generator to yield inputs and their labels in batches.
    """
    batch_size = 16
    while True:
        batch_imgs = []
        batch_labels = []
        for i in range(batch_size):
            # Create random arrays
            rand_pix = np.random.randint(100, 256)
            top_img = np.full(top_dim, rand_pix)
            bot_img = np.full(bot_dim, rand_pix)

            # Set a label
            label = np.random.choice([0, 1])

            batch_imgs.append([top_img, bot_img])
            batch_labels.append(label)

        yield batch_imgs, batch_labels


def get_compiled_model(top_dim, bot_dim):
    """
    Return a two input one output model.
    """
    # Get the top and bottom networks
    top = get_part_model(top_dim)
    bot = get_part_model(bot_dim)

    # Prepare inputs and unify the top and the bottom networks
    inp_top = Input(shape=top_dim)
    inp_bot = Input(shape=bot_dim)
    fusion = concatenate([top(inp_top), bot(inp_bot)])

    predictions = Dense(1, activation='sigmoid')(fusion)

    # Final full model
    model = Model([inp_top, inp_bot], outputs=predictions)

    model.compile(optimizer=Adam(lr=0.001),
                  loss='binary_crossentropy',
                  metrics=["accuracy"])

    return model


if __name__ == "__main__":
    top_dim, bot_dim = (16, 16, 1), (16, 16, 3)
    model = get_compiled_model(top_dim, bot_dim)
    my_gen = data_gen(top_dim, bot_dim)
    model.fit_generator(my_gen, steps_per_epoch=10)

Error message:

Traceback (most recent call last):
  File "test_keras_fit_generator.py", line 83, in <module>
    model.fit_generator(data_gen, steps_per_epoch=4)
  File "/home/bparaj/.venvs/py361/lib/python3.6/site-packages/keras/legacy/interfaces.py", line 87, in wrapper
    return func(*args, **kwargs)
  File "/home/bparaj/.venvs/py361/lib/python3.6/site-packages/keras/engine/training.py", line 2031, in fit_generator
    batch_size = x[0].shape[0]
AttributeError: 'tuple' object has no attribute 'shape'

[bparaj]

I think I figured it out. Samples of each input in the model should be packed separately. The following generator gave no errors:

def data_gen(top_dim, bot_dim):
    """
    Generator to yield batches of two inputs (per sample) with shapes top_dim and 
    bot_dim along with their labels.
    """
    batch_size = 264
    while True:
        top_batch = []
        bot_batch = []
        batch_labels = []
        for i in range(batch_size):
            # Create random arrays
            rand_pix = np.random.randint(100, 256)
            top_img = np.full(top_dim, rand_pix)
            bot_img = np.full(bot_dim, rand_pix)

            # Set a label
            label = np.random.choice([0, 1])
            batch_labels.append(label)

            # Pack each input image separately
            top_batch.append(top_img)
            bot_batch.append(bot_img)

        yield [np.array(top_batch), np.array(bot_batch)], np.array(batch_labels)

[kusumlata123]

I am getting error:AttributeError: 'tuple' object has no attribute 'shape'

word_in = (Input(shape=(max_len,), ))
print(word_in)
emb_word = Embedding(MAX_NB_WORDS+1, VECTOR_DIM, weights=[embedding_matrix], input_length= max_len, trainable= False, mask_zero=True)(word_in)
emb_word = Flatten()(emb_word)

[kusumlata123]

File "MDLSTM2.py", line 105, in model_build
emb_word = Embedding(MAX_NB_WORDS+1, VECTOR_DIM, weights=[embedding_matrix], input_length= max_len, trainable= False, mask_zero=True)(word_in)
File "/home/kusum/mycoref/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py", line 75, in symbolic_fn_wrapper
return func(args, *kwargs)
File "/home/kusum/mycoref/lib/python3.7/site-packages/keras/engine/base_layer.py", line 468, in __call__
self.set_weights(self._initial_weights)
File "/home/kusum/mycoref/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py", line 105, in eager_fn_wrapper
out = func(args, *kwargs)
File "/home/kusum/mycoref/lib/python3.7/site-packages/keras/engine/base_layer.py", line 1122, in set_weights
if pv.shape != w.shape:
AttributeError: 'tuple' object has no attribute 'shape'