Keras: how to use fit_generator with multiple image inputs
Hello, I'm trying to use a model with paired input images through (in their own similar directory trees), augmented through ImageDataGenerator using also flow_from_directory (so the method infers the labels by the folder structure). I'm getting an error because keras can't handle it in this way.
How can I combine the generators (using flow_from_directory) to be accepted by fit_generator?
Here is a sample code
Model definition *
#two classic CNN blocks are defined before these lines and then cocatenated
#create model
model = Model([input_img_1,input_img_2], out, name='colliculus_proto')
#compile model
model.compile(optimizer=Adam(lr=1e-3),
loss='categorical_crossentropy',
metrics=['accuracy'])
# image data generators for image inputs
def input_generator(train_dir,batchsize,img_height,img_width):
train_generator = ImageDataGenerator(rescale = 1./255,
shear_range = 0.2,
zoom_range = 0.2,
rotation_range=5.,
horizontal_flip = True)
training_set = train_generator.flow_from_directory(train_dir,
target_size = (img_height,img_width),
class_mode = 'categorical',
batch_size = batchsize,
shuffle=False)
return training_set
def test_generator(test_dir,batchsize,img_height,img_width):
test_gen = ImageDataGenerator(rescale = 1./255)
test_set = test_gen.flow_from_directory(test_dir,
target_size = (img_height,img_width),
class_mode = 'categorical',
batch_size = batchsize,
shuffle=False)
return test_set
#fit model
input1=input_generator(train_dir_1,batchsize,img_height,img_width)
input2=input_generator(train_dir_2,batchsize,img_height,img_width)
test1=test_generator(test_dir_1,batchsize,img_height,img_width)
test2=test_generator(test_dir_2,batchsize,img_height,img_width)
model.fit_generator([input1,input2],
steps_per_epoch=trainsetsize/batchsize,
epochs = epochs,
validation_data = [test1,test2],
validation_steps = testsetsize/batchsize,
use_multiprocessing=True,
shuffle=False)
The error I get is the following:
TypeError: Error when checking model input: data should be a Numpy array, or list/dict of Numpy arrays. Found:
aendrs
All 22 comments
https://github.com/fchollet/keras/issues/3386
this might help.
I think you need a generator that yields something of the form ([x1, x2], y).
wangqianwen0418
on 13 Oct 2017
Thanks. Here is how I solved it following some ideas from issue 3386, maybe someone might find it useful
```python
input_imgen = ImageDataGenerator(rescale = 1./255,
shear_range = 0.2,
zoom_range = 0.2,
rotation_range=5.,
horizontal_flip = True)
test_imgen = ImageDataGenerator(rescale = 1./255)
def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
genX1 = generator.flow_from_directory(dir1,
target_size = (img_height,img_width),
class_mode = 'categorical',
batch_size = batch_size,
shuffle=False,
seed=7)
genX2 = generator.flow_from_directory(dir2,
target_size = (img_height,img_width),
class_mode = 'categorical',
batch_size = batch_size,
shuffle=False,
seed=7)
while True:
X1i = genX1.next()
X2i = genX2.next()
yield [X1i[0], X2i[0]], X2i[1] #Yield both images and their mutual label
inputgenerator=generate_generator_multiple(generator=input_imgen,
dir1=train_dir_1,
dir2=train_dir_2,
batch_size=batch_size,
img_height=img_height,
img_width=img_height)
testgenerator=generate_generator_multiple(test_imgen,
dir1=train_dir_1,
dir2=train_dir_2,
batch_size=batch_size,
img_height=img_height,
img_width=img_height)
history=model.fit_generator(inputgenerator,
steps_per_epoch=trainsetsize/batch_size,
epochs = epochs,
validation_data = testgenerator,
validation_steps = testsetsize/batch_size,
use_multiprocessing=True,
shuffle=False)
aendrs
on 16 Oct 2017
That was great. How would you find the class_indices for the inputgenerator and testgenerator? I'm not able to use this syntax: inputgenerator.class_indices() here since now, the generator works with multiple inputs.
ghost
on 17 Apr 2018
@aendrs I would like to ask how to give the two inputs same pre-processing everytime? As in the function of input_generator, input_imgen is called twice, for each time randomly selected preprocessing steps are applied on the input.
laukun
on 9 May 2018
@aendrs @raaju-shiv @wangqianwen0418 @laukun @bmabey Could anyone of you guys kindly help me solve this problem https://github.com/keras-team/keras/issues/10499. I tried implementing the same generator as in this post but i dont seem to figure out where is my mistake. any help is very much appreciated.
MjdMahasneh
on 22 Jun 2018
@laukun I would suggest write your own data generator like in this example:
class DataGenerator(keras.utils.Sequence):
"""Generates data for Keras."""
def __init__(self, img_files, clinical_info, labels, ave=None, std=None, batch_size=32, dim=(300, 300), n_channels=3,
n_classes=2, shuffle=True):
"""Initialization.
Args:
img_files: A list of path to image files.
clinical_info: A dictionary of corresponding clinical variables.
labels: A dictionary of corresponding labels.
"""
self.img_files = img_files
self.clinical_info = clinical_info
self.labels = labels
self.batch_size = batch_size
self.dim = dim
if ave is None:
self.ave = np.zeros(n_channels)
else:
self.ave = ave
if std is None:
self.std = np.zeros(n_channels) + 1
else:
self.std = std
self.n_channels = n_channels
self.n_classes = n_classes
self.shuffle = shuffle
self.on_epoch_end()
def __len__(self):
"""Denotes the number of batches per epoch."""
return int(np.floor(len(self.img_files) / self.batch_size))
def __getitem__(self, index):
"""Generate one batch of data."""
# Generate indexes of the batch
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
# Find list of IDs
img_files_temp = [self.img_files[k] for k in indexes]
# Generate data
X, y = self.__data_generation(img_files_temp)
return X, y
def on_epoch_end(self):
"""Updates indexes after each epoch."""
self.indexes = np.arange(len(self.img_files))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __data_generation(self, img_files_temp):
"""Generates data containing batch_size samples."""
# X : (n_samples, *dim, n_channels)
# X = [np.empty((self.batch_size, self.dim[0], self.dim[1], self.n_channels))]
X_img = []
X_clinical = []
y = np.empty((self.batch_size), dtype=int)
# Generate data
for i, img_file in enumerate(img_files_temp):
# Read image
img = skimage.io.imread(img_file)
# Resize
img = skimage.transform.resize(img, output_shape=self.dim, mode='constant', preserve_range=True)
# Normalization
for ch in range(self.n_channels):
img[:, :, ch] = (img[:, :, ch] - self.ave[ch])/self.std[ch]
if self.shuffle:
# Some image augmentation codes
###### You can put your preprocessing codes here. #####
X_img.append(img)
X_clinical.append(self.clinical_info[img_file])
y[i] = self.labels[img_file]
X = [np.array(X_img), np.array(X_clinical)]
return X, keras.utils.to_categorical(y, num_classes=self.n_classes)
And to call the generator:
train_datagen = DataGenerator(img_files=img_files, clinical_info=clinical_info, labels=labels, ave=ave, std=std, batch_size=batch_size)
val_datagen = DataGenerator(img_files=img_files, clinical_info=clinical_info, labels=labels, ave=ave, std=std, batch_size=batch_size)
hist = model.fit_generator(train_datagen,
steps_per_epoch=len(img_files) / batch_size,
epochs=num_epochs,
verbose=1,
validation_data=val_datagen,
validation_steps=1)
The different thing I'm doing here is using two different kinds of input, an image and a numpy array. I saved image paths as a list, and created two dictionaries whose key words are the image paths. When you want to do the preprocessing, you can easily apply them right after reading the image using the image paths.
But this way, the speed will be definitely slower than loading data directly from .npy
sdw95927
on 18 Jul 2018
I'm trying to train a two streams CNN architecture, but unfortunately I got an unexpected result whitch is almost the same as one of the two streams (I have already trained every one separately):
here is my code, please tell me what I'm doing wrong :
@aendrs @sdw95927 @wangkechn
`
Arch
def fusionNet(nb_classes, inputs=(3, 224, 224)):
input_img = Input(shape=inputs, name='RGB_input')
conv1 = Conv2D(96, (7, 7), activation="relu", name="conv1", strides=(2, 2), padding="same",
## ....
## ....
## ....
firstCNN_output = BatchNormalization(name='firstCNN_output_BN')(last_dropout)
##
print(firstCNN_output._keras_shape) # I got here (None, 512, 13, 13)
input_img2 = Input(shape=inputs, name='depth_input')
conv12 = Conv2D(96, (7, 7), activation="relu", name="conv12", strides=(2, 2), padding="same", kernel_initializer="glorot_uniform")(input_img2)
#....
#....
#...
## BN
SecondCNN_output = BatchNormalization(name='SecondCNN_output_BN2')(last_dropout2)
print(SecondCNN_output._keras_shape) # I got here (None, 512, 13, 13)
#
merge2CNN = merge([firstCNN_output, SecondCNN_output], mode='concat', concat_axis=1)
### got here (None, 1024, 13, 13)
conv10 = Conv2D(
nb_classes, (1, 1),init='glorot_uniform',
border_mode='valid', name='conv10')(merge2CNN)
avgpool10 = AveragePooling2D((13, 13), name='avgpool10')(conv10)
flatten = Flatten(name='flatten')(avgpool10)
softmax = Activation("softmax", name='softmax')(flatten)
return Model(inputs=[input_img,input_img2], output=softmax)
`
Training
sn = fusionNet(nb_classes=24)
sgd = SGD(lr=0.001, decay=0.0002, momentum=0.9, nesterov=True)
sn.compile( optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy')
train_datagen = ImageDataGenerator(
rescale=1./255,
zoom_range=0.01,
rotation_range=2.,
featurewise_center=True,
featurewise_std_normalization=True,
)
#train_datagen = ImageDataGenerator(rescale=1./255)
test_datagen = ImageDataGenerator(rescale=1./255)
def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
genX1 = generator.flow_from_directory(dir1,
target_size = (img_height,img_width),
class_mode = 'categorical',
batch_size = batch_size,
shuffle=False)
genX2 = generator.flow_from_directory(dir2,
target_size = (img_height, img_width),
class_mode = 'categorical',
batch_size = batch_size)
while True:
X1i = genX1.next()
X2i = genX2.next()
yield [X1i[0], X2i[0]], X2i[1] #Yield both images and their mutual label
train_generator=generate_generator_multiple(generator=train_datagen,
dir1=train_data_dir,
dir2=train_data_dir_colored,
batch_size=32,
img_height=size,
img_width=size)
validation_generator=generate_generator_multiple(test_datagen,
dir1=validation_data_dir,
dir2=validation_data_dir_colored,
batch_size=batch_size,
img_height=size,
img_width=size)
tensorboardcb = keras.callbacks.TensorBoard(log_dir='/workspace/sqn_224/fusion', histogram_freq=0, batch_size=32, write_graph=True)
sn.fit_generator(
train_generator,
steps_per_epoch=nb_train_samples//32,
nb_epoch=nb_epoch,
validation_data=validation_generator,
validation_steps = nb_validation_samples//32,
callbacks=[tensorboardcb])
scstu
on 16 Aug 2018
@scstu I would like to merge the input data first, which might be a better idea.
wkecn
on 4 Sep 2018
@scstu that can be done in different ways. but keep in mind the number of learned parameters differ depending on the way you merge the input images. so if youre merging in a multispectral-like shape (for 2xRGB images you can merge in two ways as far as i know. so it would be something like: HxWx6 presuming that your concatenating the images, or it could be HxWx3 if youre finding a transformation of the 2 RGB images such as multiplication, add, sub etc..) in either case be careful with the input size for your model and remember that the number of the learned parameters will differ. to know whats better you can only try and compare the results. it also will affect the extracted feature by the convolution differently. i worked on a similar problem and i used multiple branch CNN and 2 input images where each images passes a branch from the CNN and then the features extracted by the two CNN branches are merged in a merging layer before passing it to the top layers.
MjdMahasneh
on 4 Sep 2018
@aendrs thank you very much on your algorithm. He is fantastic.
gledsonmelotti
on 9 Oct 2018
@aendrs it was really good work to generate multi-streams input. thanks
aamirk5
on 6 Nov 2018
@aendrs could you please share your portion of code for concatenation and inputs manipulation
aamirk5
on 24 Nov 2018
I have a doubt. I will train input sets on the same network, for example, model1 receives input X1 (three folders containing classes and each class has the training, validation, and test data) and model2 receives input X2 (three folders containing three classes and each class has the training, validation and test data). Then I will concatenate a convolution of model X1 with one of model X2. So I have two input data, two validation data and two test data. My question is about the following command: steps_per_epoch = nb_train_samples // batchsize. I would like to know if my nb_train_samples is the sum of only the training_class1_X1 + training_class2_X1 + training_class3_X1 or if nb_train_samples is the sum of (training_class1_X1 + training_class2_X1 + training_class3_X1) + (training_class1_X2 + training_class2_X2 + training_class3_X2).
gledsonmelotti
on 29 Nov 2018
@scstu that can be done in different ways. but keep in mind the number of learned parameters differ depending on the way you merge the input images. so if youre merging in a multispectral-like shape (for 2xRGB images you can merge in two ways as far as i know. so it would be something like: HxWx6 presuming that your concatenating the images, or it could be HxWx3 if youre finding a transformation of the 2 RGB images such as multiplication, add, sub etc..) in either case be careful with the input size for your model and remember that the number of the learned parameters will differ. to know whats better you can only try and compare the results. it also will affect the extracted feature by the convolution differently. i worked on a similar problem and i used multiple branch CNN and 2 input images where each images passes a branch from the CNN and then the features extracted by the two CNN branches are merged in a merging layer before passing it to the top layers.
@MjdMahasneh I believe your comment is interesting. Could you help me with a doubt? I will train input sets on the same network, for example, model1 receives input X1 (three folders containing classes and each class has the training, validation, and test data) and model2 receives input X2 (three folders containing three classes and each class has the training, validation and test data). Then I will concatenate a convolution of model X1 with one of model X2. So I have two input data, two validation data and two test data. My question is about the following command: steps_per_epoch = nb_train_samples // batchsize. I would like to know if my nb_train_samples is the sum of only the training_class1_X1 + training_class2_X1 + training_class3_X1 or if nb_train_samples is the sum of (training_class1_X1 + training_class2_X1 + training_class3_X1) + (training_class1_X2 + training_class2_X2 + training_class3_X2).
gledsonmelotti
on 29 Nov 2018
I'm trying to train a two streams CNN architecture, but unfortunately I got an unexpected result whitch is almost the same as one of the two streams (I have already trained every one separately):
here is my code, please tell me what I'm doing wrong :
@aendrs @sdw95927 @wangkechn
`Arch
def fusionNet(nb_classes, inputs=(3, 224, 224)): input_img = Input(shape=inputs, name='RGB_input') conv1 = Conv2D(96, (7, 7), activation="relu", name="conv1", strides=(2, 2), padding="same", ## .... ## .... ## .... firstCNN_output = BatchNormalization(name='firstCNN_output_BN')(last_dropout) ## print(firstCNN_output._keras_shape) # I got here (None, 512, 13, 13) input_img2 = Input(shape=inputs, name='depth_input') conv12 = Conv2D(96, (7, 7), activation="relu", name="conv12", strides=(2, 2), padding="same", kernel_initializer="glorot_uniform")(input_img2) #.... #.... #... ## BN SecondCNN_output = BatchNormalization(name='SecondCNN_output_BN2')(last_dropout2) print(SecondCNN_output._keras_shape) # I got here (None, 512, 13, 13) # merge2CNN = merge([firstCNN_output, SecondCNN_output], mode='concat', concat_axis=1) ### got here (None, 1024, 13, 13) conv10 = Conv2D( nb_classes, (1, 1),init='glorot_uniform', border_mode='valid', name='conv10')(merge2CNN) avgpool10 = AveragePooling2D((13, 13), name='avgpool10')(conv10) flatten = Flatten(name='flatten')(avgpool10) softmax = Activation("softmax", name='softmax')(flatten) return Model(inputs=[input_img,input_img2], output=softmax)`
Training
sn = fusionNet(nb_classes=24) sgd = SGD(lr=0.001, decay=0.0002, momentum=0.9, nesterov=True) sn.compile( optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy') train_datagen = ImageDataGenerator( rescale=1./255, zoom_range=0.01, rotation_range=2., featurewise_center=True, featurewise_std_normalization=True, ) #train_datagen = ImageDataGenerator(rescale=1./255) test_datagen = ImageDataGenerator(rescale=1./255) def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width): genX1 = generator.flow_from_directory(dir1, target_size = (img_height,img_width), class_mode = 'categorical', batch_size = batch_size, shuffle=False) genX2 = generator.flow_from_directory(dir2, target_size = (img_height, img_width), class_mode = 'categorical', batch_size = batch_size) while True: X1i = genX1.next() X2i = genX2.next() yield [X1i[0], X2i[0]], X2i[1] #Yield both images and their mutual label train_generator=generate_generator_multiple(generator=train_datagen, dir1=train_data_dir, dir2=train_data_dir_colored, batch_size=32, img_height=size, img_width=size) validation_generator=generate_generator_multiple(test_datagen, dir1=validation_data_dir, dir2=validation_data_dir_colored, batch_size=batch_size, img_height=size, img_width=size) tensorboardcb = keras.callbacks.TensorBoard(log_dir='/workspace/sqn_224/fusion', histogram_freq=0, batch_size=32, write_graph=True) sn.fit_generator( train_generator, steps_per_epoch=nb_train_samples//32, nb_epoch=nb_epoch, validation_data=validation_generator, validation_steps = nb_validation_samples//32, callbacks=[tensorboardcb])
Could you help me with a doubt? I will train input sets on the same network, for example, model1 receives input X1 (three folders containing classes and each class has the training, validation, and test data) and model2 receives input X2 (three folders containing three classes and each class has the training, validation and test data). Then I will concatenate a convolution of model X1 with one of model X2. So I have two input data, two validation data and two test data. My question is about the following command: steps_per_epoch = nb_train_samples // batchsize. I would like to know if my nb_train_samples is the sum of only the training_class1_X1 + training_class2_X1 + training_class3_X1 or if nb_train_samples is the sum of (training_class1_X1 + training_class2_X1 + training_class3_X1) + (training_class1_X2 + training_class2_X2 + training_class3_X2).
gledsonmelotti
on 29 Nov 2018
I'm trying to train a two streams CNN architecture, but unfortunately I got an unexpected result whitch is almost the same as one of the two streams (I have already trained every one separately):
here is my code, please tell me what I'm doing wrong :
@aendrs @sdw95927 @wangkechn
`Arch
def fusionNet(nb_classes, inputs=(3, 224, 224)): input_img = Input(shape=inputs, name='RGB_input') conv1 = Conv2D(96, (7, 7), activation="relu", name="conv1", strides=(2, 2), padding="same", ## .... ## .... ## .... firstCNN_output = BatchNormalization(name='firstCNN_output_BN')(last_dropout) ## print(firstCNN_output._keras_shape) # I got here (None, 512, 13, 13) input_img2 = Input(shape=inputs, name='depth_input') conv12 = Conv2D(96, (7, 7), activation="relu", name="conv12", strides=(2, 2), padding="same", kernel_initializer="glorot_uniform")(input_img2) #.... #.... #... ## BN SecondCNN_output = BatchNormalization(name='SecondCNN_output_BN2')(last_dropout2) print(SecondCNN_output._keras_shape) # I got here (None, 512, 13, 13) # merge2CNN = merge([firstCNN_output, SecondCNN_output], mode='concat', concat_axis=1) ### got here (None, 1024, 13, 13) conv10 = Conv2D( nb_classes, (1, 1),init='glorot_uniform', border_mode='valid', name='conv10')(merge2CNN) avgpool10 = AveragePooling2D((13, 13), name='avgpool10')(conv10) flatten = Flatten(name='flatten')(avgpool10) softmax = Activation("softmax", name='softmax')(flatten) return Model(inputs=[input_img,input_img2], output=softmax)`
Training
sn = fusionNet(nb_classes=24) sgd = SGD(lr=0.001, decay=0.0002, momentum=0.9, nesterov=True) sn.compile( optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy') train_datagen = ImageDataGenerator( rescale=1./255, zoom_range=0.01, rotation_range=2., featurewise_center=True, featurewise_std_normalization=True, ) #train_datagen = ImageDataGenerator(rescale=1./255) test_datagen = ImageDataGenerator(rescale=1./255) def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width): genX1 = generator.flow_from_directory(dir1, target_size = (img_height,img_width), class_mode = 'categorical', batch_size = batch_size, shuffle=False) genX2 = generator.flow_from_directory(dir2, target_size = (img_height, img_width), class_mode = 'categorical', batch_size = batch_size) while True: X1i = genX1.next() X2i = genX2.next() yield [X1i[0], X2i[0]], X2i[1] #Yield both images and their mutual label train_generator=generate_generator_multiple(generator=train_datagen, dir1=train_data_dir, dir2=train_data_dir_colored, batch_size=32, img_height=size, img_width=size) validation_generator=generate_generator_multiple(test_datagen, dir1=validation_data_dir, dir2=validation_data_dir_colored, batch_size=batch_size, img_height=size, img_width=size) tensorboardcb = keras.callbacks.TensorBoard(log_dir='/workspace/sqn_224/fusion', histogram_freq=0, batch_size=32, write_graph=True) sn.fit_generator( train_generator, steps_per_epoch=nb_train_samples//32, nb_epoch=nb_epoch, validation_data=validation_generator, validation_steps = nb_validation_samples//32, callbacks=[tensorboardcb])Could you help me with a doubt? I will train input sets on the same network, for example, model1 receives input X1 (three folders containing classes and each class has the training, validation, and test data) and model2 receives input X2 (three folders containing three classes and each class has the training, validation and test data). Then I will concatenate a convolution of model X1 with one of model X2. So I have two input data, two validation data and two test data. My question is about the following command: steps_per_epoch = nb_train_samples // batchsize. I would like to know if my nb_train_samples is the sum of only the training_class1_X1 + training_class2_X1 + training_class3_X1 or if nb_train_samples is the sum of (training_class1_X1 + training_class2_X1 + training_class3_X1) + (training_class1_X2 + training_class2_X2 + training_class3_X2).
Hello,
Did you find an answer / solution to your question? I have the same problem / doubt.
AndreAbade
on 22 Jan 2019
I'm trying to train a two streams CNN architecture, but unfortunately I got an unexpected result whitch is almost the same as one of the two streams (I have already trained every one separately):
here is my code, please tell me what I'm doing wrong :
@aendrs @sdw95927 @wangkechn
`Arch
def fusionNet(nb_classes, inputs=(3, 224, 224)): input_img = Input(shape=inputs, name='RGB_input') conv1 = Conv2D(96, (7, 7), activation="relu", name="conv1", strides=(2, 2), padding="same", ## .... ## .... ## .... firstCNN_output = BatchNormalization(name='firstCNN_output_BN')(last_dropout) ## print(firstCNN_output._keras_shape) # I got here (None, 512, 13, 13) input_img2 = Input(shape=inputs, name='depth_input') conv12 = Conv2D(96, (7, 7), activation="relu", name="conv12", strides=(2, 2), padding="same", kernel_initializer="glorot_uniform")(input_img2) #.... #.... #... ## BN SecondCNN_output = BatchNormalization(name='SecondCNN_output_BN2')(last_dropout2) print(SecondCNN_output._keras_shape) # I got here (None, 512, 13, 13) # merge2CNN = merge([firstCNN_output, SecondCNN_output], mode='concat', concat_axis=1) ### got here (None, 1024, 13, 13) conv10 = Conv2D( nb_classes, (1, 1),init='glorot_uniform', border_mode='valid', name='conv10')(merge2CNN) avgpool10 = AveragePooling2D((13, 13), name='avgpool10')(conv10) flatten = Flatten(name='flatten')(avgpool10) softmax = Activation("softmax", name='softmax')(flatten) return Model(inputs=[input_img,input_img2], output=softmax)`
Training
sn = fusionNet(nb_classes=24) sgd = SGD(lr=0.001, decay=0.0002, momentum=0.9, nesterov=True) sn.compile( optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy') train_datagen = ImageDataGenerator( rescale=1./255, zoom_range=0.01, rotation_range=2., featurewise_center=True, featurewise_std_normalization=True, ) #train_datagen = ImageDataGenerator(rescale=1./255) test_datagen = ImageDataGenerator(rescale=1./255) def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width): genX1 = generator.flow_from_directory(dir1, target_size = (img_height,img_width), class_mode = 'categorical', batch_size = batch_size, shuffle=False) genX2 = generator.flow_from_directory(dir2, target_size = (img_height, img_width), class_mode = 'categorical', batch_size = batch_size) while True: X1i = genX1.next() X2i = genX2.next() yield [X1i[0], X2i[0]], X2i[1] #Yield both images and their mutual label train_generator=generate_generator_multiple(generator=train_datagen, dir1=train_data_dir, dir2=train_data_dir_colored, batch_size=32, img_height=size, img_width=size) validation_generator=generate_generator_multiple(test_datagen, dir1=validation_data_dir, dir2=validation_data_dir_colored, batch_size=batch_size, img_height=size, img_width=size) tensorboardcb = keras.callbacks.TensorBoard(log_dir='/workspace/sqn_224/fusion', histogram_freq=0, batch_size=32, write_graph=True) sn.fit_generator( train_generator, steps_per_epoch=nb_train_samples//32, nb_epoch=nb_epoch, validation_data=validation_generator, validation_steps = nb_validation_samples//32, callbacks=[tensorboardcb])Could you help me with a doubt? I will train input sets on the same network, for example, model1 receives input X1 (three folders containing classes and each class has the training, validation, and test data) and model2 receives input X2 (three folders containing three classes and each class has the training, validation and test data). Then I will concatenate a convolution of model X1 with one of model X2. So I have two input data, two validation data and two test data. My question is about the following command: steps_per_epoch = nb_train_samples // batchsize. I would like to know if my nb_train_samples is the sum of only the training_class1_X1 + training_class2_X1 + training_class3_X1 or if nb_train_samples is the sum of (training_class1_X1 + training_class2_X1 + training_class3_X1) + (training_class1_X2 + training_class2_X2 + training_class3_X2).
Hello,
Did you find an answer / solution to your question? I have the same problem / doubt.
I think the training size is the later one, sum of all training samples. The question is equal with:
A model has three sets of parameters A, B and C. We use X1 data to train A & C with B fixed, and use X2 to train B & C with A fixed.
I think now the answer is clear.
sdw95927
on 22 Jan 2019
I am trying similar things, but I am getting stuck with the input of the model.
`
def generate_generator_multiple(generator,dir1, dir2,dir3, batch_size, img_size):
genX1 = generator.flow_from_directory(dir1,
target_size = (img_size,img_size),
class_mode = 'categorical',
batch_size = batch_size,
shuffle=False,
seed=7,subset='training')
genX2 = generator.flow_from_directory(dir2,
target_size = (img_size,img_size),
class_mode = 'categorical',
batch_size = batch_size,
shuffle=False,
seed=7,subset='training')
genX3 = generator.flow_from_directory(dir3,
target_size = (img_size,img_size),
class_mode = 'categorical',
batch_size = batch_size,
shuffle=False,
seed=7,subset='training')
while True:
X1i = genX1.next()
X2i = genX2.next()
X3i = genX3.next()
yield [X1i[0], X2i[0],X3i[0]], X3i[1] #Yield both images and their mutual label
train_datagen = ImageDataGenerator(validation_split=VAL_SPLIT,rescale=1./255)
inputgenerator=generate_generator_multiple(generator=train_datagen,
dir1='mergedata',
dir2='mergedata',
dir3='mergedata',
batch_size=BATCH_SIZE,
img_size=IMG_SIZE)
modelA = load_model('/content/8-base_model_1.h5')
modelA.load_weights('/content/8-weights.002-0.991-0.406.hdf5')
modelB = load_model('/content/30-base_model_1.h5')
modelB.load_weights('/content/30-weights.010-0.499-0.374.hdf5')
modelC = load_model('/content/80-base_model_1.h5')
modelC.load_weights('/content/80-weights.002-1.064-0.305.hdf5')
for layer in modelA.layers:
layer.trainable = False
layer.name = layer.name + str("_1")
for layer in modelB.layers:
layer.trainable = False
layer.name = layer.name + str("_2")
for layer in modelC.layers:
layer.trainable = False
layer.name = layer.name + str("_3")
mod_1 = concatenate([modelA.layers[-1].output, modelB.layers[-1].output,modelC.layers[-1].output])
mod_1 = Dropout(0.2)(mod_1)
mod_1 = Dense(15, activation='relu')(mod_1)
predictions = Dense(9, activation='softmax', name='pred_age')(mod_1)
top_model = Model(inputs=[modelA.input, modelB.input, modelC.input], outputs=predictions)
opt = get_optimizer('adam', lr_rate)
top_model.compile(loss=['categorical_crossentropy'],
optimizer=opt,
metrics=['accuracy','mae'])
model.fit_generator(inputgenerator,
epochs=EPOCHS,
steps_per_epoch = STEPS_PER_EPOCH,
validation_data=validgenerator,
validation_steps = VALIDATION_STEPS,
verbose=1,
callbacks=callbacks)`
I first trained 3 different models:
- model 1 = 3 labels (8,9,10)
- model 2 = 2 labels (30,31)
- model 3 = 4 labels (80,81,82,83)
I would like to combine these 3 models into a final model with 1 output containing 9 labels (8,9,10,30,31,80,81,82,83). The final input needs to get 1 input image instead of 3 images. But I still get stuck. I am building a kind of hierarchy here to improve the accuracy.
aezco
on 25 May 2019
Thanks. Here is how I solved it following some ideas from _issue 3386_, maybe someone might find it useful
input_imgen = ImageDataGenerator(rescale = 1./255, shear_range = 0.2, zoom_range = 0.2, rotation_range=5., horizontal_flip = True) test_imgen = ImageDataGenerator(rescale = 1./255) def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width): genX1 = generator.flow_from_directory(dir1, target_size = (img_height,img_width), class_mode = 'categorical', batch_size = batch_size, shuffle=False, seed=7) genX2 = generator.flow_from_directory(dir2, target_size = (img_height,img_width), class_mode = 'categorical', batch_size = batch_size, shuffle=False, seed=7) while True: X1i = genX1.next() X2i = genX2.next() yield [X1i[0], X2i[0]], X2i[1] #Yield both images and their mutual label inputgenerator=generate_generator_multiple(generator=input_imgen, dir1=train_dir_1, dir2=train_dir_2, batch_size=batch_size, img_height=img_height, img_width=img_height) testgenerator=generate_generator_multiple(test_imgen, dir1=train_dir_1, dir2=train_dir_2, batch_size=batch_size, img_height=img_height, img_width=img_height) history=model.fit_generator(inputgenerator, steps_per_epoch=trainsetsize/batch_size, epochs = epochs, validation_data = testgenerator, validation_steps = testsetsize/batch_size, use_multiprocessing=True, shuffle=False)
How is that that the test generator and train generator(input generator) have the same directories??
Shouldn't it be different??
(dir1 and dir2 should be different)
Please correct me if I'm wrong.
enviz
on 20 Dec 2019
def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
genX1 = generator.flow_from_directory(dir1,
target_size = (img_height,img_width),
class_mode = 'categorical',
batch_size = batch_size,
shuffle=False,
seed=7)
genX2 = generator.flow_from_directory(dir2,
target_size = (img_height,img_width),
class_mode = 'categorical',
batch_size = batch_size,
shuffle=False,
seed=7)
while True:
X1i = genX1.next()
X2i = genX2.next()
yield [X1i[0], X2i[0]], X2i[1] #Yield both images and their mutual label
In this method shown,
I have a situation where I am returning two ground truth masks, and one image using a custom data_generator as shown above.
However, how do I incorporate it into a custom loss function.
def custom_loss(y_true1,ytrue2, y_pred):
#compute loss here
return loss
saahilwho
on 20 Aug 2020
Hi, I am using the train generator for two inputs as @aendrs wrote.
The problem is that the two generators have two different orders, and this is not intended because we have mutual labels. Is there a way to use order of the first generator in the second as well? Thank you!
apres91
on 3 Sep 2020
To reply to my comment, I made my own class of generator that takes as input 2 images and allows multiple labels:
import numpy as np
import keras
import imageio
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs1, list_IDs2, labels, batch_size=32, dim=(32,32,32), n_channels=1,
n_classes=10, shuffle=True):
'Initialization'
self.batch_size = batch_size
self.labels = labels
self.list_IDs1 = list_IDs1
self.list_IDs2 = list_IDs2
self.n_channels = n_channels
self.dim=dim
self.n_classes = n_classes
self.shuffle = shuffle
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs1) / self.batch_size))
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs1))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[indexself.batch_size:(index+1)self.batch_size]
# Find list of IDs
list_IDs_temp1 = [self.list_IDs1[k] for k in indexes]
list_IDs_temp2 = [self.list_IDs2[k] for k in indexes]
# Generate data
[X1,X2], y = self.__data_generation(list_IDs_temp1,list_IDs_temp2,indexes)
return [X1,X2], y
def __data_generation(self, list_IDs_temp1,list_IDs_temp2,indexes):
'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
# Initialization
X1 = np.empty((self.batch_size, *self.dim, self.n_channels))
X2 = np.empty((self.batch_size, *self.dim, self.n_channels))
y = [np.empty(self.batch_size,dtype=np.float32) for _ in range(len(self.labels))]
# Generate data
for i, ID in enumerate(list_IDs_temp1):
# Store sample
X1[i,] = np.stack((imageio.imread(ID),) * 3, axis=-1)
# Store class
for list_id in range(len(self.labels)):
y[list_id][i] = self.labels[list_id][indexes[i]]
for i, ID in enumerate(list_IDs_temp2):
X2[i,] = np.stack((imageio.imread(ID),) * 3, axis=-1)
return [X1,X2], y
This way the order of the two batches of data is the same.
apres91
on 4 Sep 2020
Related issues
rantsandruse
路
3Comments
somewacko
路
3Comments
kylemcdonald
路
3Comments
farizrahman4u
路
3Comments
harishkrishnav
路
3Comments
Most helpful comment
Thanks. Here is how I solved it following some ideas from issue 3386, maybe someone might find it useful
```python
input_imgen = ImageDataGenerator(rescale = 1./255,
shear_range = 0.2,
zoom_range = 0.2,
rotation_range=5.,
horizontal_flip = True)
test_imgen = ImageDataGenerator(rescale = 1./255)
def generate_generator_multiple(generator,dir1, dir2, batch_size, img_height,img_width):
genX1 = generator.flow_from_directory(dir1,
target_size = (img_height,img_width),
class_mode = 'categorical',
batch_size = batch_size,
shuffle=False,
seed=7)
inputgenerator=generate_generator_multiple(generator=input_imgen,
dir1=train_dir_1,
dir2=train_dir_2,
batch_size=batch_size,
img_height=img_height,
img_width=img_height)
testgenerator=generate_generator_multiple(test_imgen,
dir1=train_dir_1,
dir2=train_dir_2,
batch_size=batch_size,
img_height=img_height,
img_width=img_height)
history=model.fit_generator(inputgenerator,
steps_per_epoch=trainsetsize/batch_size,
epochs = epochs,
validation_data = testgenerator,
validation_steps = testsetsize/batch_size,
use_multiprocessing=True,
shuffle=False)