Keras: tensorflow.python.framework.errors_impl.FailedPreconditionError: Error while reading resource variable _AnonymousVar51 from Container

[figoliu]

Please make sure that this is a Bug or a Feature Request and provide all applicable information asked by the template.
If your issue is an implementation question, please ask your question on StackOverflow or on the Keras Slack channel instead of opening a GitHub issue.

System information

• Have I written custom code (as opposed to using example directory): No
• OS Platform and Distribution (e.g., Linux Ubuntu 16.04): Windosw7
• TensorFlow backend (yes / no): yes
• TensorFlow version: 2.0.0, gpu
• Keras version: 2.3.2
• Python version: 3.6.6
• CUDA/cuDNN version: 10.0/7.3
• GPU model and memory: 6G

You can obtain the TensorFlow version with:
python -c "import tensorflow as tf; print(tf.GIT_VERSION, tf.VERSION)"
You can obtain the Keras version with:
python -c 'import keras as k; print(k.__version__)'

Describe the current behavior
When I run "python acgan.py", get the tracback like this:

Function call stack:
keras_scratch_graph
Describe the expected behavior
It should begin to train the gan.

Code to reproduce the issue
Provide a reproducible test case that is the bare minimum necessary to generate the problem.
from __future__ import print_function, division

from keras.datasets import mnist
from keras.layers import Input, Dense, Reshape, Flatten, Dropout, multiply, GaussianNoise
from keras.layers import BatchNormalization, Activation, Embedding, ZeroPadding2D
from keras.layers import MaxPooling2D, concatenate
from keras.layers.advanced_activations import LeakyReLU
from keras.layers.convolutional import UpSampling2D, Conv2D
from keras.models import Sequential, Model
from keras.optimizers import Adam
from keras import losses
from keras.utils import to_categorical
import keras.backend as K
import matplotlib.pyplot as plt
import numpy as np

class CUSGAN():
def __init__(self):
self.img_rows = 28
self.img_cols = 28
self.channels = 1
self.img_shape = (self.img_rows, self.img_cols, self.channels)
self.latent_dim = 100

    optimizer = Adam(0.0002, 0.5)

    # Build and compile the discriminator
    self.discriminator = self.build_discriminator()
    self.discriminator.compile(loss=['binary_crossentropy'],
        optimizer=optimizer,
        metrics=['accuracy'])

    # Build the generator
    self.generator = self.build_generator()

    # Build the encoder
    self.encoder = self.build_encoder()

    # The part of the bigan that trains the discriminator and encoder
    self.discriminator.trainable = False

    # Generate image from sampled noise
    z = Input(shape=(self.latent_dim, ))
    img_ = self.generator(z)

    # Encode image
    img = Input(shape=self.img_shape)
    z_ = self.encoder(img)

    # Latent -> img is fake, and img -> latent is valid
    fake = self.discriminator([z, img_])
    valid = self.discriminator([z_, img])

    # Set up and compile the combined model
    # Trains generator to fool the discriminator
    self.bigan_generator = Model([z, img], [fake, valid])
    self.bigan_generator.compile(loss=['binary_crossentropy', 'binary_crossentropy'],
        optimizer=optimizer)

def train(self, epochs, batch_size=128, sample_interval=50):

    # Load the dataset
    (X_train, _), (_, _) = mnist.load_data()

    # Rescale -1 to 1
    X_train = (X_train.astype(np.float32) - 127.5) / 127.5
    X_train = np.expand_dims(X_train, axis=3)

    # Adversarial ground truths
    valid = np.ones((batch_size, 1))
    fake = np.zeros((batch_size, 1))

    for epoch in range(epochs):


        # ---------------------
        #  Train Discriminator
        # ---------------------

        # Sample noise and generate img
        z = np.random.normal(size=(batch_size, self.latent_dim))
        imgs_ = self.generator.predict(z)

        # Select a random batch of images and encode
        idx = np.random.randint(0, X_train.shape[0], batch_size)
        imgs = X_train[idx]
        z_ = self.encoder.predict(imgs)

        # Train the discriminator (img -> z is valid, z -> img is fake)
        d_loss_real = self.discriminator.train_on_batch([z_, imgs], valid)
        d_loss_fake = self.discriminator.train_on_batch([z, imgs_], fake)
        d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)

        # ---------------------
        #  Train Generator
        # ---------------------

        # Train the generator (z -> img is valid and img -> z is is invalid)
        g_loss = self.bigan_generator.train_on_batch([z, imgs], [valid, fake])

        # Plot the progress
        print ("%d [D loss: %f, acc: %.2f%%] [G loss: %f]" % (epoch, d_loss[0], 100*d_loss[1], g_loss[0]))

        # If at save interval => save generated image samples
        if epoch % sample_interval == 0:
            self.sample_interval(epoch)

......
if __name__ == '__main__':
cusgan = CUSGAN()
cusgan.train(epochs=40000, batch_size=32, sample_interval=2000)
Other info / logs
Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached.

019-11-13 15:59:14.725865: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cublas64_100.dll
C:\Users\admin\AppData\Local\Programs\Python\Python36\lib\site-packages\keras\engine\training.py:297: UserWarning: Discrepancy between trainable weights and collected trainable weights, did you set model.trainable without calling model.compile after ?
'Discrepancy between trainable weights and collected trainable'
0 [D loss: 0.990420, acc: 31.25%] [G loss: 5.191110]
C:\Users\admin\AppData\Local\Programs\Python\Python36\lib\site-packages\keras\engine\training.py:297: UserWarning: Discrepancy between trainable weights and collected trainable weights, did you set model.trainable without calling model.compile after ?
'Discrepancy between trainable weights and collected trainable'
2019-11-13 15:59:19.545140: W tensorflow/core/common_runtime/base_collective_executor.cc:216] BaseCollectiveExecutor::StartAbort Failed precondition: Error while reading resource variable _AnonymousVar51 from Container: localhost. This could mean that the variable was uninitialized. Not found: Resource localhost/_AnonymousVar51/class tensorflow::Var does not exist.
[[{{node mul_21/ReadVariableOp}}]]
Traceback (most recent call last):
File "bigan.py", line 188, in
bigan.train(epochs=40000, batch_size=32, sample_interval=2000)
File "bigan.py", line 150, in train
d_loss_real = self.discriminator.train_on_batch([z_, imgs], valid)
File "C:\Users\admin\AppData\Local\Programs\Python\Python36\lib\site-packages\keras\engine\training.py", line 1514, in train_on_batch
outputs = self.train_function(ins)
File "C:\Users\admin\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow_core\python\keras\backend.py", line 3740, in __call__
outputs = self._graph_fn(*converted_inputs)
File "C:\Users\admin\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow_core\python\eager\function.py", line 1081, in __call__
return self._call_impl(args, kwargs)
File "C:\Users\admin\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow_core\python\eager\function.py", line 1121, in _call_impl
return self._call_flat(args, self.captured_inputs, cancellation_manager)
File "C:\Users\admin\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow_core\python\eager\function.py", line 1224, in _call_flat
ctx, args, cancellation_manager=cancellation_manager)
File "C:\Users\admin\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow_core\python\eager\function.py", line 511, in call
ctx=ctx)
File "C:\Users\admin\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow_core\python\eager\execute.py", line 67, in quick_execute
six.raise_from(core._status_to_exception(e.code, message), None)
File "", line 3, in raise_from
tensorflow.python.framework.errors_impl.FailedPreconditionError: Error while reading resource variable _AnonymousVar51 from Container: localhost. This could mean that the variable was uninitialized. Not found: Resource localhost/_AnonymousVar51/class tensorflow::Var does not exist.
[[node mul_21/ReadVariableOp (defined at C:\Users\admin\AppData\Local\Programs\Python\Python36\lib\site-packages\tensorflow_core\python\framework\ops.py:1751) ]] [Op:__inference_keras_scratch_graph_3478]

[figoliu]

I changed this line

from keras.optimizers import Adam

like this

from tensorflow.keras.optimizers import Adam

this problem dispeared.
Anyone have any idea about this problem? Thanks.

[qq885566]

from tensorflow.keras.optimizers import Adam

this problem dispeared.
Anyone have any idea about this problem? Thanks.

Solve my problem too
Thankful
and ashamed
I have no idea either

[tha23rd]

I am having the same issue running a model in the same repo (wgan) but with the following imports:

from __future__ import print_function, division

from keras.initializers import RandomNormal
from keras.layers.merge import _Merge
from keras.layers import Input, Dense, Reshape, Flatten, Dropout
from keras.layers import BatchNormalization, Activation, ZeroPadding2D
from keras.layers.advanced_activations import LeakyReLU
from keras.models import Sequential, Model
from keras.optimizers import RMSprop
from functools import partial

import keras.backend as K

import matplotlib.pyplot as plt

from tensorflow.keras.optimizers does not exist for me

[oleksandrstriuk]

I changed this line

from keras.optimizers import Adam

like this

from tensorflow.keras.optimizers import Adam

this problem dispeared.
Anyone have any idea about this problem? Thanks.

Solved my problem as well! You're a lifesaver. Thanks!

[SonalSavaliya]

I am having the same issue running a model in the same repo (wgan) but with the following imports:

from __future__ import print_function, division

from keras.initializers import RandomNormal
from keras.layers.merge import _Merge
from keras.layers import Input, Dense, Reshape, Flatten, Dropout
from keras.layers import BatchNormalization, Activation, ZeroPadding2D
from keras.layers.advanced_activations import LeakyReLU
from keras.models import Sequential, Model
from keras.optimizers import RMSprop
from functools import partial

import keras.backend as K

import matplotlib.pyplot as plt

from tensorflow.keras.optimizers does not exist for me

I am having the same problem. Did you get the solution?

[mxahan]

I changed this line

from keras.optimizers import Adam

like this

from tensorflow.keras.optimizers import Adam

this problem dispeared.
Anyone have any idea about this problem? Thanks.

It worked for me.

[kevinn888]

I changed this line

from keras.optimizers import Adam

like this

from tensorflow.keras.optimizers import Adam

this problem dispeared.
Anyone have any idea about this problem? Thanks.

I have same issue but it didn't worked for me
i have a feeling something uninitialized in tf.image.crop_to_bounding_box but i can't find the problem

this my log
tensorflow.python.framework.errors_impl.FailedPreconditionError: 2 root error(s) found. (0) Failed precondition: Error while reading resource variable _AnonymousVar359 from Container: localhost. This could mean that the variable was uninitialized. Not found: Resource localhost/_AnonymousVar359/class tensorflow::Var does not exist. [[node regr_vote/crop_to_bounding_box_6/ReadVariableOp_2 (defined at C:\Users\kevin\anaconda3\envs\tf-gpu\lib\site-packages\keras\backend\tensorflow_backend.py:3009) ]] [[input_image/_8]] (1) Failed precondition: Error while reading resource variable _AnonymousVar359 from Container: localhost. This could mean that the variable was uninitialized. Not found: Resource localhost/_AnonymousVar359/class tensorflow::Var does not exist. [[node regr_vote/crop_to_bounding_box_6/ReadVariableOp_2 (defined at C:\Users\kevin\anaconda3\envs\tf-gpu\lib\site-packages\keras\backend\tensorflow_backend.py:3009) ]]

[mostafa610]

I changed this line

from keras.optimizers import Adam

like this

from tensorflow.keras.optimizers import Adam

this problem dispeared.
Anyone have any idea about this problem? Thanks.

I have same issue but it didn't worked for me
i have a feeling something uninitialized in tf.image.crop_to_bounding_box but i can't find the problem

this my log

  (0) Failed precondition:  Error while reading resource variable _AnonymousVar359 from Container: localhost. This could mean that the variable was uninitialized. Not found: Resource localhost/_AnonymousVar359/class tensorflow::Var does not exist.
         [[node regr_vote/crop_to_bounding_box_6/ReadVariableOp_2 (defined at C:\Users\kevin\anaconda3\envs\tf-gpu\lib\site-packages\keras\backend\tensorflow_backend.py:3009) ]]
         [[input_image/_8]]
  (1) Failed precondition:  Error while reading resource variable _AnonymousVar359 from Container: localhost. This could mean that the variable was uninitialized. Not found: Resource localhost/_AnonymousVar359/class tensorflow::Var does not exist.
         [[node regr_vote/crop_to_bounding_box_6/ReadVariableOp_2 (defined at C:\Users\kevin\anaconda3\envs\tf-gpu\lib\site-packages\keras\backend\tensorflow_backend.py:3009) ]]```

i have the same issue

[kevinn888]

I changed this line

from keras.optimizers import Adam

like this

from tensorflow.keras.optimizers import Adam

this problem dispeared.
Anyone have any idea about this problem? Thanks.

I have same issue but it didn't worked for me
i have a feeling something uninitialized in tf.image.crop_to_bounding_box but i can't find the problem

this my log

  (0) Failed precondition:  Error while reading resource variable _AnonymousVar359 from Container: localhost. This could mean that the variable was uninitialized. Not found: Resource localhost/_AnonymousVar359/class tensorflow::Var does not exist.
         [[node regr_vote/crop_to_bounding_box_6/ReadVariableOp_2 (defined at C:\Users\kevin\anaconda3\envs\tf-gpu\lib\site-packages\keras\backend\tensorflow_backend.py:3009) ]]
         [[input_image/_8]]
  (1) Failed precondition:  Error while reading resource variable _AnonymousVar359 from Container: localhost. This could mean that the variable was uninitialized. Not found: Resource localhost/_AnonymousVar359/class tensorflow::Var does not exist.
         [[node regr_vote/crop_to_bounding_box_6/ReadVariableOp_2 (defined at C:\Users\kevin\anaconda3\envs\tf-gpu\lib\site-packages\keras\backend\tensorflow_backend.py:3009) ]]```

I tried to trace the error by running function separately,
and I got error width must be >= target + offset

is it possible that Error while reading resource variable connected to this error..??

[VeereshShringari]

I am also facing same issue with my model :

FailedPreconditionError:  Error while reading resource variable _AnonymousVar1297 from Container: localhost. This could mean that the variable was uninitialized. Not found: Resource localhost/_AnonymousVar1297/class tensorflow::Var does not exist.
     [[node metrics_1/weighted_loss/ExpandDims/ReadVariableOp (defined at C:\Users\Veeresh\Anaconda3\lib\site-packages\tensorflow_core\python\framework\ops.py:1751) ]] [Op:__inference_keras_scratch_graph_109288]

Function call stack:
keras_scratch_graph 

And the fix ;

from tensorflow.keras.optimizers import Adam

hasn't worked for me. Any solution on this error may help me too

[rajatkhanna1999]

I changed this line

from keras.optimizers import Adam

like this

from tensorflow.keras.optimizers import Adam

this problem dispeared.
Anyone have any idea about this problem? Thanks.

I have same issue but it didn't worked for me
i have a feeling something uninitialized in tf.image.crop_to_bounding_box but i can't find the problem

this my log

  (0) Failed precondition:  Error while reading resource variable _AnonymousVar359 from Container: localhost. This could mean that the variable was uninitialized. Not found: Resource localhost/_AnonymousVar359/class tensorflow::Var does not exist.
         [[node regr_vote/crop_to_bounding_box_6/ReadVariableOp_2 (defined at C:\Users\kevin\anaconda3\envs\tf-gpu\lib\site-packages\keras\backend\tensorflow_backend.py:3009) ]]
         [[input_image/_8]]
  (1) Failed precondition:  Error while reading resource variable _AnonymousVar359 from Container: localhost. This could mean that the variable was uninitialized. Not found: Resource localhost/_AnonymousVar359/class tensorflow::Var does not exist.
         [[node regr_vote/crop_to_bounding_box_6/ReadVariableOp_2 (defined at C:\Users\kevin\anaconda3\envs\tf-gpu\lib\site-packages\keras\backend\tensorflow_backend.py:3009) ]]```

Have u found the fix to this problem?

[leiparov]

I changed this line

from keras.optimizers import Adam

like this

from tensorflow.keras.optimizers import Adam

this problem dispeared.
Anyone have any idea about this problem? Thanks.

It worked! I don't know why but I worked!

[sowshan]

getting the same error even after i change it
trainer.trainModel()
Training on: ['high_traffic']
Training with Batch Size: 4
Number of Experiments: 100
Training with transfer learning from pretrained Model
Epoch 1/100
Traceback (most recent call last):

File "", line 1, in
trainer.trainModel()

File "C:\Users\Sowmya\Anaconda3\lib\site-packages\imageai\Detection\Custom__init__.py", line 291, in trainModel
max_queue_size=8

File "C:\Users\Sowmya\Anaconda3\lib\site-packages\keras\legacy\interfaces.py", line 91, in wrapper
return func(args, *kwargs)

File "C:\Users\Sowmya\Anaconda3\lib\site-packages\keras\engine\training.py", line 1732, in fit_generator
initial_epoch=initial_epoch)

File "C:\Users\Sowmya\Anaconda3\lib\site-packages\keras\engine\training_generator.py", line 220, in fit_generator
reset_metrics=False)

File "C:\Users\Sowmya\Anaconda3\lib\site-packages\keras\engine\training.py", line 1514, in train_on_batch
outputs = self.train_function(ins)

File "C:\Users\Sowmya\Anaconda3\lib\site-packages\tensorflow\python\keras\backend.py", line 3792, in __call__
outputs = self._graph_fn(*converted_inputs)

File "C:\Users\Sowmya\Anaconda3\lib\site-packages\tensorflow\python\eager\function.py", line 1605, in __call__
return self._call_impl(args, kwargs)

File "C:\Users\Sowmya\Anaconda3\lib\site-packages\tensorflow\python\eager\function.py", line 1645, in _call_impl
return self._call_flat(args, self.captured_inputs, cancellation_manager)

File "C:\Users\Sowmya\Anaconda3\lib\site-packages\tensorflow\python\eager\function.py", line 1746, in _call_flat
ctx, args, cancellation_manager=cancellation_manager))

File "C:\Users\Sowmya\Anaconda3\lib\site-packages\tensorflow\python\eager\function.py", line 598, in call
ctx=ctx)

File "C:\Users\Sowmya\Anaconda3\lib\site-packages\tensorflow\python\eager\execute.py", line 60, in quick_execute
inputs, attrs, num_outputs)

NotFoundError: Resource localhost/_AnonymousVar3555/class tensorflow::Var does not exist.
[[node model_6_1/yolo_layer_12/AssignAddVariableOp (defined at C:\Users\Sowmya\Anaconda3\lib\site-packages\keras\backend\tensorflow_backend.py:3009) ]] [Op:__inference_keras_scratch_graph_266403]

Function call stack:
keras_scratch_graph

[annekqiu]

I get the same problem, and I will try this way .and someone knows why is OK?