Autokeras: StructuredDataClassifier's exported model can't predict or evaluate

[exiarepairii]

Bug Description

I use autokeras to train a classifier, I can predict or evaluate with the . But the model exported by the export_model function can't predict or evaluate.
When I try model.evaluate(x_valid,y_valid)it raise the error below,
and model.evaluate(x_valid,y_valid) raise the same error. No matter which dataset I use, the iteration ends at 32.

 32/2000 [..............................] - ETA: 48s
---------------------------------------------------------------------------
UnimplementedError                        Traceback (most recent call last)
<ipython-input-20-73f41c3caa9d> in <module>
----> 1 model.evaluate(x_valid,y_valid)

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\keras\engine\training.py in evaluate(self, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing)
    928         max_queue_size=max_queue_size,
    929         workers=workers,
--> 930         use_multiprocessing=use_multiprocessing)
    931 
    932   def predict(self,

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\keras\engine\training_v2.py in evaluate(self, model, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, **kwargs)
    488         sample_weight=sample_weight, steps=steps, callbacks=callbacks,
    489         max_queue_size=max_queue_size, workers=workers,
--> 490         use_multiprocessing=use_multiprocessing, **kwargs)
    491 
    492   def predict(self, model, x, batch_size=None, verbose=0, steps=None,

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\keras\engine\training_v2.py in _model_iteration(self, model, mode, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, **kwargs)
    473               mode=mode,
    474               training_context=training_context,
--> 475               total_epochs=1)
    476           cbks.make_logs(model, epoch_logs, result, mode)
    477 

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\keras\engine\training_v2.py in run_one_epoch(model, iterator, execution_function, dataset_size, batch_size, strategy, steps_per_epoch, num_samples, mode, training_context, total_epochs)
    126         step=step, mode=mode, size=current_batch_size) as batch_logs:
    127       try:
--> 128         batch_outs = execution_function(iterator)
    129       except (StopIteration, errors.OutOfRangeError):
    130         # TODO(kaftan): File bug about tf function and errors.OutOfRangeError?

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\keras\engine\training_v2_utils.py in execution_function(input_fn)
     96     # `numpy` translates Tensors to values in Eager mode.
     97     return nest.map_structure(_non_none_constant_value,
---> 98                               distributed_function(input_fn))
     99 
    100   return execution_function

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\eager\def_function.py in __call__(self, *args, **kwds)
    566         xla_context.Exit()
    567     else:
--> 568       result = self._call(*args, **kwds)
    569 
    570     if tracing_count == self._get_tracing_count():

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\eager\def_function.py in _call(self, *args, **kwds)
    636               *args, **kwds)
    637       # If we did not create any variables the trace we have is good enough.
--> 638       return self._concrete_stateful_fn._filtered_call(canon_args, canon_kwds)  # pylint: disable=protected-access
    639 
    640     def fn_with_cond(*inner_args, **inner_kwds):

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\eager\function.py in _filtered_call(self, args, kwargs)
   1609          if isinstance(t, (ops.Tensor,
   1610                            resource_variable_ops.BaseResourceVariable))),
-> 1611         self.captured_inputs)
   1612 
   1613   def _call_flat(self, args, captured_inputs, cancellation_manager=None):

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\eager\function.py in _call_flat(self, args, captured_inputs, cancellation_manager)
   1690       # No tape is watching; skip to running the function.
   1691       return self._build_call_outputs(self._inference_function.call(
-> 1692           ctx, args, cancellation_manager=cancellation_manager))
   1693     forward_backward = self._select_forward_and_backward_functions(
   1694         args,

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\eager\function.py in call(self, ctx, args, cancellation_manager)
    543               inputs=args,
    544               attrs=("executor_type", executor_type, "config_proto", config),
--> 545               ctx=ctx)
    546         else:
    547           outputs = execute.execute_with_cancellation(

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\tensorflow_core\python\eager\execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
     65     else:
     66       message = e.message
---> 67     six.raise_from(core._status_to_exception(e.code, message), None)
     68   except TypeError as e:
     69     keras_symbolic_tensors = [

c:\users\exia\appdata\local\programs\python\python36\lib\site-packages\six.py in raise_from(value, from_value)

UnimplementedError:  Cast double to string is not supported
     [[node Cast (defined at <ipython-input-20-73f41c3caa9d>:1) ]] [Op:__inference_distributed_function_1731]

Function call stack:
distributed_function

Reproducing Steps

import autokeras as ak clf = ak.StructuredDataClassifier(max_trials=10) clf.fit(x_train, y_train,validation_data=(x_valid, y_valid)) model = clf.export_model() model.evaluate(x_valid,y_valid)
where x_train is a (5000, 4) numpy.ndarray, y_train is a (5000, 1) numpy.ndarray
x_valid is a (2000, 4) numpy.ndarray, y_valid is a (2000, 1) numpy.ndarray
no matter I use model.evaluate(x_valid,y_valid) or model.evaluate(x_train,y_train) it raise the same error.

Setup Details

Include the details about the versions of:

• OS type and version: Win10
• Python: 3.6.5
• autokeras: 1.0.1
• scikit-learn:0.22
• numpy:1.18.0
• scipy:1.4.1
• tensorflow:2.1.0

[jbrownlee]

I get the same fault with models fit via StructuredDataRegressor and StructuredDataClassifier

[adriangb]

Same issue:

import autokeras as ak
import numpy as np
from sklearn.model_selection import train_test_split


n_points = 100
n_features = 6
n_classes = 10
X = np.random.rand(n_points, n_features)
print(X.shape, X.dtype)# random (100, 6) shaped array
y = np.random.randint(low=0, high=n_classes, size=n_points)
print(y.shape, y.dtype)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)

clf = ak.StructuredDataClassifier(max_trials=1)
clf.fit(X_train, y_train, epochs=2)
predicted_y = clf.predict(X_test)
clf.evaluate(X_test, y_test)

model = clf.export_model()
predicted_y_2 = model.predict(X_test)

Could this be related to #929? It seems to me like one of the pre-processors isn't being exported correctly. @haifeng-jin mentioned that a "new" version, I'm not sure which version he was referring to, I tried pulling from the master and no different.

[haifeng-jin]

We had a new release, autokeras 1.0.2. If you use it with the latest tf-nightly. This issue should be resolved.

[adriangb]

The above code snippet is still broken for me. I tested on Ubuntu and Mac OS.

Package versions:

autokeras==1.0.2
keras-tuner==1.0.1
tf-estimator-nightly==2.1.0.dev2020022109
tf-nightly==2.2.0.dev20200218

Output/error:

WARNING:tensorflow:A checkpoint was restored (e.g. tf.train.Checkpoint.restore or tf.keras.Model.load_weights) but not all checkpointed values were used. See above for specific issues. Use expect_partial() on the load status object, e.g. tf.train.Checkpoint.restore(...).expect_partial(), to silence these warnings, or use assert_consumed() to make the check explicit. See https://www.tensorflow.org/guide/checkpoint#loading_mechanics for details.
WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.iter
WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.beta_1
WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.beta_2
WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.decay
WARNING:tensorflow:Unresolved object in checkpoint: (root).optimizer.learning_rate
WARNING:tensorflow:A checkpoint was restored (e.g. tf.train.Checkpoint.restore or tf.keras.Model.load_weights) but not all checkpointed values were used. See above for specific issues. Use expect_partial() on the load status object, e.g. tf.train.Checkpoint.restore(...).expect_partial(), to silence these warnings, or use assert_consumed() to make the check explicit. See https://www.tensorflow.org/guide/checkpoint#loading_mechanics for details.
2020-02-21 18:37:51.015891: W tensorflow/core/framework/op_kernel.cc:1729] OP_REQUIRES failed at cast_op.cc:123 : Unimplemented: Cast double to string is not supported
Traceback (most recent call last):
  File ".../scratch_2.py", line 24, in <module>
    predicted_y_2 = model.predict(X_test)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py", line 919, in predict
    use_multiprocessing=use_multiprocessing)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_v2.py", line 496, in predict
    workers=workers, use_multiprocessing=use_multiprocessing, **kwargs)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_v2.py", line 473, in _model_iteration
    total_epochs=1)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_v2.py", line 128, in run_one_epoch
    batch_outs = execution_function(iterator)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/training_v2_utils.py", line 98, in execution_function
    distributed_function(input_fn))
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py", line 576, in __call__
    result = self._call(*args, **kwds)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py", line 646, in _call
    return self._concrete_stateful_fn._filtered_call(canon_args, canon_kwds)  # pylint: disable=protected-access
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 1660, in _filtered_call
    self.captured_inputs)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 1741, in _call_flat
    ctx, args, cancellation_manager=cancellation_manager))
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/eager/function.py", line 598, in call
    ctx=ctx)
  File ".../venv/lib/python3.7/site-packages/tensorflow/python/eager/execute.py", line 60, in quick_execute
    inputs, attrs, num_outputs)
tensorflow.python.framework.errors_impl.UnimplementedError:  Cast double to string is not supported
     [[node Cast (defined at .../scratch_2.py:24) ]] [Op:__inference_distributed_function_3792]

Function call stack:
distributed_function

@haifeng-jin, could you test that code snippet with the above package versions and let me know if I am doing something wrong.

[haifeng-jin]

We are trying to convert all the tutorials to colab. Making sure all the functions have a working example.

[adriangb]

We are trying to convert all the tutorials to colab. Making sure all the functions have a working example.

I am not sure I understand what this means in relation to this issue. Would you like the above example/bug to be posted on colab for testing? It's still not clear to me if I am doing something wrong or if this is a bug.

[exiarepairii]

We had a new release, autokeras 1.0.2. If you use it with the latest tf-nightly. This issue should be resolved.

I have upgraded the autokeras, but still have the same problem.

[haifeng-jin]

I mean this may still be a bug. We will test it out during we write the colab notebooks. And fix it.

I guess the bug may be related to the custom preprocessing layer we implemented for coverting the numerical features to categorical ones. That custom layer will be removed. We will use the official layers provided in tf2.2 to do the trick, which are not released yet.

[exiarepairii]

I mean this may still be a bug. We will test it out during we write the colab notebooks. And fix it.

I guess the bug may be related to the custom preprocessing layer we implemented for coverting the numerical features to categorical ones. That custom layer will be removed. We will use the official layers provided in tf2.2 to do the trick, which are not released yet.

thanks for your efforts, I look forward to the progress of autokeras

[adriangb]

I mean this may still be a bug. We will test it out during we write the colab notebooks. And fix it.

I guess the bug may be related to the custom preprocessing layer we implemented for coverting the numerical features to categorical ones. That custom layer will be removed. We will use the official layers provided in tf2.2 to do the trick, which are not released yet.

Awesome sounds great! It'll be amazing once that is implemented. Thank you very much.

[stale[bot]]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.

[njan-creative]

I mean this may still be a bug. We will test it out during we write the colab notebooks. And fix it.

I guess the bug may be related to the custom preprocessing layer we implemented for coverting the numerical features to categorical ones. That custom layer will be removed. We will use the official layers provided in tf2.2 to do the trick, which are not released yet.

This is fixed now ?

[Karim-53]

not fixed yet (v1.0.3)

[haifeng-jin]

@Karim-53 Really? I thought I fixed it.
With autokeras==1.0.3 and keras-tuner=1.0.2rc0, there should not be this bug any more.
What error message are you getting?

Thanks.

[Karim-53]

Yes, the bug is still there. We tried with 2 different computers.

To reproduce the error:
run a very basic example: https://autokeras.com/tutorial/structured_data_regression/

The error might happen (or not). If not then

run these lines once again:

predicted_y = reg.predict(...)
print(reg.evaluate(...))

in the second run, the error always happen:
Unresolved object in checkpoint

As you can see, many people are already complaining about that (all same issue):

[haifeng-jin]

@Karim-53 Thank you for the reply.
I just tried.
The solution is a little bit tricky.
This code runs fine. You can use it as a reference.

import autokeras as ak
import numpy as np
from sklearn.model_selection import train_test_split


n_points = 100
n_features = 6
n_classes = 10
X = np.random.rand(n_points, n_features)
print(X.shape, X.dtype)# random (100, 6) shaped array
y = np.random.randint(low=0, high=n_classes, size=n_points)
print(y.shape, y.dtype)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)

clf = ak.StructuredDataClassifier(max_trials=1)
clf.fit(X_train, y_train, epochs=2)
predicted_y = clf.predict(X_test)
clf.evaluate(X_test, y_test)

model = clf.export_model()
import tensorflow as tf

X_test = tf.data.Dataset.from_tensor_slices(X_test.astype(np.unicode)).batch(32)
predicted_y_2 = model.predict(X_test)
print(predicted_y_2)

[TianyuanCai]

@Karim-53 Thank you for the reply.
I just tried.
The solution is a little bit tricky.
This code runs fine. You can use it as a reference.

import autokeras as ak
import numpy as np
from sklearn.model_selection import train_test_split


n_points = 100
n_features = 6
n_classes = 10
X = np.random.rand(n_points, n_features)
print(X.shape, X.dtype)# random (100, 6) shaped array
y = np.random.randint(low=0, high=n_classes, size=n_points)
print(y.shape, y.dtype)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)

clf = ak.StructuredDataClassifier(max_trials=1)
clf.fit(X_train, y_train, epochs=2)
predicted_y = clf.predict(X_test)
clf.evaluate(X_test, y_test)

model = clf.export_model()
import tensorflow as tf

X_test = tf.data.Dataset.from_tensor_slices(X_test.astype(np.unicode)).batch(32)
predicted_y_2 = model.predict(X_test)
print(predicted_y_2)

Running into the same issue – so seems like the solution to wrap the data into a tfds object?

[haifeng-jin]

Yes. The main issue is the exported model needs the data to be in strings format.