Incubator-mxnet: mxnet.base.MXNetError: [12:37:13] src/executor/../common/exec_utils.h:391: InferShape pass cannot decide shapes for the following arguments

Created on 9 Jan 2020 · 1Comment · Source: apache/incubator-mxnet

Description

I find that MXNET behaves differently from Tensorflow when I try to load one simple lstm model using the following code. I get the exception under the MXNET backend. But the code runs correctly under Tensorflow.

Error Message

Traceback (most recent call last):
File "/root/anaconda3/lib/python3.6/site-packages/mxnet/symbol/symbol.py", line 1776, in simple_bind
 ctypes.byref(exe_handle)))
File "/root/anaconda3/lib/python3.6/site-packages/mxnet/base.py", line 278, in check_call
 raise MXNetError(py_str(_LIB.MXGetLastError()))
mxnet.base.MXNetError: [12:29:57] src/executor/../common/exec_utils.h:391: InferShape pass cannot decide shapes for the following arguments (-1 means unknown dimensions). Please consider providing them as inputs:
/lstm_1_input2: [-1,50,25],
Stack trace:
[bt] (0) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x6ccefb) [0x7f0743234efb]
[bt] (1) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x392c8c3) [0x7f07464948c3]
[bt] (2) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(mxnet::exec::GraphExecutor::Init(nnvm::Symbol, mxnet::Context const&, std::map<std::string, mxnet::Context, std::less<std::string>, std::allocator<std::pair<std::string const, mxnet::Context> > > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::unordered_map<std::string, mxnet::TShape, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::TShape> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::unordered_set<std::string, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::string> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::unordered_map<std::string, mxnet::NDArray, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::NDArray> > >*, mxnet::Executor*, std::unordered_map<nnvm::NodeEntry, mxnet::NDArray, nnvm::NodeEntryHash, nnvm::NodeEntryEqual, std::allocator<std::pair<nnvm::NodeEntry const, mxnet::NDArray> > > const&)+0x40f) [0x7f07464c161f]
[bt] (3) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(mxnet::Executor::SimpleBind(nnvm::Symbol, mxnet::Context const&, std::map<std::string, mxnet::Context, std::less<std::string>, std::allocator<std::pair<std::string const, mxnet::Context> > > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::unordered_map<std::string, mxnet::TShape, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::TShape> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::unordered_set<std::string, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::string> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::unordered_map<std::string, mxnet::NDArray, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::NDArray> > >*, mxnet::Executor*)+0x2a8) [0x7f07464c2198]
[bt] (4) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x3887d83) [0x7f07463efd83]
[bt] (5) /root/anaconda3/lib/python3.6/lib-dynload/../../libffi.so.6(ffi_call_unix64+0x4c) [0x7f078434aec0]
[bt] (6) /root/anaconda3/lib/python3.6/lib-dynload/../../libffi.so.6(ffi_call+0x22d) [0x7f078434a87d]
[bt] (7) /root/anaconda3/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so(_ctypes_callproc+0x2ce) [0x7f0784560ede]
[bt] (8) /root/anaconda3/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so(+0x13915) [0x7f0784561915]



During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "crash_checker.py", line 45, in <module>
 model = keras.models.load_model(file_path,custom_objects=custom_objects())
File "/root/anaconda3/lib/python3.6/site-packages/keras/engine/saving.py", line 496, in load_model
 model = _deserialize_model(f, custom_objects, compile)
File "/root/anaconda3/lib/python3.6/site-packages/keras/engine/saving.py", line 302, in _deserialize_model
 model = model_from_config(model_config, custom_objects=custom_objects)
File "/root/anaconda3/lib/python3.6/site-packages/keras/engine/saving.py", line 535, in model_from_config
 return deserialize(config, custom_objects=custom_objects)
File "/root/anaconda3/lib/python3.6/site-packages/keras/layers/__init__.py", line 55, in deserialize
 printable_module_name='layer')
File "/root/anaconda3/lib/python3.6/site-packages/keras/utils/generic_utils.py", line 145, in deserialize_keras_object
 list(custom_objects.items())))
File "/root/anaconda3/lib/python3.6/site-packages/keras/engine/sequential.py", line 301, in from_config
 model.add(layer)
File "/root/anaconda3/lib/python3.6/site-packages/keras/engine/sequential.py", line 181, in add
 output_tensor = layer(self.outputs[0])
File "/root/anaconda3/lib/python3.6/site-packages/keras/engine/base_layer.py", line 480, in __call__
 x = K.identity(x)
File "/root/anaconda3/lib/python3.6/site-packages/keras/backend/mxnet_backend.py", line 780, in identity
 x_value = eval(x)
File "/root/anaconda3/lib/python3.6/site-packages/keras/backend/mxnet_backend.py", line 595, in eval
 ret = _forward_pass(x)[0].asnumpy()
File "/root/anaconda3/lib/python3.6/site-packages/keras/backend/mxnet_backend.py", line 4640, in _forward_pass
 executor = x.symbol.simple_bind(mx.cpu(), grad_req='null')
File "/root/anaconda3/lib/python3.6/site-packages/mxnet/symbol/symbol.py", line 1782, in simple_bind
 raise RuntimeError(error_msg)
RuntimeError: simple_bind error. Arguments:
[12:29:57] src/executor/../common/exec_utils.h:391: InferShape pass cannot decide shapes for the following arguments (-1 means unknown dimensions). Please consider providing them as inputs:
/lstm_1_input2: [-1,50,25],
Stack trace:
[bt] (0) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x6ccefb) [0x7f0743234efb]
[bt] (1) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x392c8c3) [0x7f07464948c3]
[bt] (2) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(mxnet::exec::GraphExecutor::Init(nnvm::Symbol, mxnet::Context const&, std::map<std::string, mxnet::Context, std::less<std::string>, std::allocator<std::pair<std::string const, mxnet::Context> > > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::unordered_map<std::string, mxnet::TShape, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::TShape> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::unordered_set<std::string, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::string> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::unordered_map<std::string, mxnet::NDArray, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::NDArray> > >*, mxnet::Executor*, std::unordered_map<nnvm::NodeEntry, mxnet::NDArray, nnvm::NodeEntryHash, nnvm::NodeEntryEqual, std::allocator<std::pair<nnvm::NodeEntry const, mxnet::NDArray> > > const&)+0x40f) [0x7f07464c161f]
[bt] (3) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(mxnet::Executor::SimpleBind(nnvm::Symbol, mxnet::Context const&, std::map<std::string, mxnet::Context, std::less<std::string>, std::allocator<std::pair<std::string const, mxnet::Context> > > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::vector<mxnet::Context, std::allocator<mxnet::Context> > const&, std::unordered_map<std::string, mxnet::TShape, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::TShape> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::unordered_map<std::string, int, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, int> > > const&, std::vector<mxnet::OpReqType, std::allocator<mxnet::OpReqType> > const&, std::unordered_set<std::string, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::string> > const&, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::vector<mxnet::NDArray, std::allocator<mxnet::NDArray> >*, std::unordered_map<std::string, mxnet::NDArray, std::hash<std::string>, std::equal_to<std::string>, std::allocator<std::pair<std::string const, mxnet::NDArray> > >*, mxnet::Executor*)+0x2a8) [0x7f07464c2198]
[bt] (4) /root/anaconda3/lib/python3.6/site-packages/mxnet/libmxnet.so(+0x3887d83) [0x7f07463efd83]
[bt] (5) /root/anaconda3/lib/python3.6/lib-dynload/../../libffi.so.6(ffi_call_unix64+0x4c) [0x7f078434aec0]
[bt] (6) /root/anaconda3/lib/python3.6/lib-dynload/../../libffi.so.6(ffi_call+0x22d) [0x7f078434a87d]
[bt] (7) /root/anaconda3/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so(_ctypes_callproc+0x2ce) [0x7f0784560ede]
[bt] (8) /root/anaconda3/lib/python3.6/lib-dynload/_ctypes.cpython-36m-x86_64-linux-gnu.so(+0x13915) [0x7f0784561915]

To Reproduce

script.py

import os
import sys
import argparse

parse = argparse.ArgumentParser()
parse.add_argument("--backend", type=str, help="name of backends")

flags, unparsed = parse.parse_known_args(sys.argv[1:])

bk = flags.backend
os.environ['KERAS_BACKEND'] = bk
os.environ["TF_CPP_MIN_LOG_LEVEL"] = '2'
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = ""
import keras


new_model = keras.models.load_model("lstm1-PM-data_origin.h5")
new_model.summary()
print("Done")

The structure of model is shown below