Onnxruntime: Need CentOS support

[jim-meyer]

Describe the bug
I would like to be able to use onnxruntime on CentOS 7.6. I can pip install onnxruntime 0.4.0 on CentOS 7.6 fine. But when I try to "import onnxruntime" using python 3.6.3 it fails:

/opt/app-root/lib/python3.6/site-packages/onnxruntime/capi/_pybind_state.py:12: UserWarning: Cannot load onnxruntime.capi. Error: '/lib64/libstdc++.so.6: version `GLIBCXX_3.4.20' not found (required by /opt/app-root/lib/python3.6/site-packages/onnxruntime/capi/onnxruntime_pybind11_state.so)'
  warnings.warn("Cannot load onnxruntime.capi. Error: '{0}'".format(str(e)))
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/opt/app-root/lib/python3.6/site-packages/onnxruntime/__init__.py", line 21, in <module>
    from onnxruntime.capi._pybind_state import RunOptions, SessionOptions, get_device, NodeArg, ModelMetadata
ImportError: cannot import name 'RunOptions'

This almost certainly fails for other supported (>=3.5) versions of python though I haven't tried any other versions.

The output of strings /usr/lib64/libstdc++.so.6 | grep GLIBCXX is:

GLIBCXX_3.4
GLIBCXX_3.4.1
GLIBCXX_3.4.2
GLIBCXX_3.4.3
GLIBCXX_3.4.4
GLIBCXX_3.4.5
GLIBCXX_3.4.6
GLIBCXX_3.4.7
GLIBCXX_3.4.8
GLIBCXX_3.4.9
GLIBCXX_3.4.10
GLIBCXX_3.4.11
GLIBCXX_3.4.12
GLIBCXX_3.4.13
GLIBCXX_3.4.14
GLIBCXX_3.4.15
GLIBCXX_3.4.16
GLIBCXX_3.4.17
GLIBCXX_3.4.18
GLIBCXX_3.4.19
GLIBCXX_DEBUG_MESSAGE_LENGTH

From https://gcc.gnu.org/onlinedocs/libstdc++/manual/abi.html it appears that onnxruntime is built with gcc 4.9 on Ubuntu but CentOS 7.6 only has gcc 4.8. And this appears to be the root of the problem.

System information

• OS Platform and Distribution: centos/python-36-centos7:1 from DockerHub (7.6)
• ONNX Runtime installed from (source or binary): binary
• ONNX Runtime version: 0.4.0
• Python version: 3.6

Discussion
I wanted to open this as an issue to hopefully spark some discussion and/or see if CentOS support is something that is being worked on by anybody.

Does anybody know if onnxruntime can be built with gcc 4.8? Or if there are components in the onnxruntime build tool chain that absolutely requires gcc 4.9?

[snnn]

but CentOS 7.6 only has gcc 4.8. And this appears to be the root of the problem.

Yes. You are right.

[jywu-msft]

onnxruntime was built on ubuntu 16.04 (gcc 5.4.0)

onnxruntime requires gcc >= 5.0
see https://github.com/microsoft/onnxruntime/blob/master/BUILD.md

[mayeut]

onnxruntime could be built on the manylinux2010 image based on CentOS 6 using gcc-8. This would allow to upload a valid manylinux2010 wheel on PyPI rather than uploading a misleading/non-conforming manylinux1 wheel.
I was able to build at least the CPU version (not tried other flavors) using this image.
c.f. https://github.com/pypa/manylinux

I hacked a bit the CMake files in order to get it to build (not PR ready at all and I'm not working on it, just to show it can build):

[root@37be2db82782 onnxruntime]# git diff
diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
index f5e60805..d6d1ee4c 100644
--- a/cmake/CMakeLists.txt
+++ b/cmake/CMakeLists.txt
@@ -204,7 +204,9 @@ endif()
 if(onnxruntime_BUILD_SHARED_LIB OR onnxruntime_ENABLE_PYTHON)
   if(onnxruntime_ENABLE_PYTHON)
     find_package(PythonInterp 3.5 REQUIRED)
-    find_package(PythonLibs 3.5 REQUIRED)
+    #find_package(PythonLibs 3.4 REQUIRED)
+    set(PYTHON_INCLUDE_DIRS "/opt/python/cp36-cp36m/include/python3.6m/")
+    set(PYTHON_LIBRARIES "")
   else()
     find_package(PythonInterp 3.4 REQUIRED)
     find_package(PythonLibs 3.4 REQUIRED)
diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake
index 24a2e973..6d7edabf 100644
--- a/cmake/onnxruntime_python.cmake
+++ b/cmake/onnxruntime_python.cmake
@@ -60,7 +60,7 @@ onnxruntime_add_include_to_target(onnxruntime_pybind11_state gsl)
 if(APPLE)
   set(ONNXRUNTIME_SO_LINK_FLAG "-Xlinker -exported_symbols_list ${ONNXRUNTIME_ROOT}/python/exported_symbols.lst")
 elseif(UNIX)
-  set(ONNXRUNTIME_SO_LINK_FLAG "-Xlinker --version-script=${ONNXRUNTIME_ROOT}/python/version_script.lds -Xlinker --no-undefined -Xlinker --gc-sections")
+  set(ONNXRUNTIME_SO_LINK_FLAG "-Xlinker --version-script=${ONNXRUNTIME_ROOT}/python/version_script.lds -Xlinker --gc-sections")
 else()
   set(ONNXRUNTIME_SO_LINK_FLAG "-DEF:${ONNXRUNTIME_ROOT}/python/pybind.def")
 endif()
@@ -103,7 +103,7 @@ elseif (APPLE)
     INSTALL_RPATH_USE_LINK_PATH FALSE)
 else()
   target_link_libraries(onnxruntime_pybind11_state PRIVATE ${onnxruntime_pybind11_state_libs} ${PYTHON_LIBRARY}
-          ${ONNXRUNTIME_SO_LINK_FLAG} ${onnxruntime_EXTERNAL_LIBRARIES})
+          ${ONNXRUNTIME_SO_LINK_FLAG} ${onnxruntime_EXTERNAL_LIBRARIES} rt)
   set_target_properties(onnxruntime_pybind11_state PROPERTIES LINK_FLAGS "-Xlinker -rpath=\$ORIGIN")
 endif()

diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake
index 3189b671..5c0480e6 100644
--- a/cmake/onnxruntime_unittests.cmake
+++ b/cmake/onnxruntime_unittests.cmake
@@ -290,7 +290,7 @@ if (SingleUnitTestProject)
   AddTest(
     TARGET onnxruntime_test_all
     SOURCES ${all_tests}
-    LIBS ${onnxruntime_test_providers_libs} ${onnxruntime_test_common_libs}
+    LIBS ${onnxruntime_test_providers_libs} ${onnxruntime_test_common_libs} rt
     DEPENDS ${all_dependencies}
   )

@@ -448,7 +448,7 @@ set(onnx_test_libs
 list(APPEND onnx_test_libs ${onnxruntime_EXTERNAL_LIBRARIES} libprotobuf) # test code uses delimited parsing and hence needs to link with the full protobuf

 add_executable(onnx_test_runner ${onnx_test_runner_src_dir}/main.cc)
-target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs})
+target_link_libraries(onnx_test_runner PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs} rt)
 onnxruntime_add_include_to_target(onnx_test_runner gsl)
 target_include_directories(onnx_test_runner PRIVATE ${ONNXRUNTIME_ROOT})
 set_target_properties(onnx_test_runner PROPERTIES FOLDER "ONNXRuntimeTest")
@@ -523,6 +523,7 @@ if (onnxruntime_BUILD_SHARED_LIB)
 else()
   target_link_libraries(onnxruntime_perf_test PRIVATE onnx_test_runner_common ${GETOPT_LIB_WIDE} ${onnx_test_libs})
 endif()
+target_link_libraries(onnxruntime_perf_test PRIVATE rt)
 set_target_properties(onnxruntime_perf_test PROPERTIES FOLDER "ONNXRuntimeTest")

 # shared lib
diff --git a/setup.py b/setup.py
index 92384379..a6442787 100644
--- a/setup.py
+++ b/setup.py
@@ -3,8 +3,10 @@
 # Licensed under the MIT License.
 #--------------------------------------------------------------------------

-from setuptools import setup, find_packages
+from setuptools import setup, find_packages, Extension
+from distutils.command.build_ext import build_ext
 from os import path, getcwd
+from shutil import copyfile
 import platform
 import sys
 import datetime
@@ -43,6 +45,16 @@ try:
 except ImportError:
     bdist_wheel = None

+onnxruntime_ext = Extension(
+    "onnxruntime.capi.onnxruntime_pybind11_state",
+    ["onnxruntime/capi/onnxruntime_pybind11_state.so"],
+)
+
+class onnxruntime_build_ext(build_ext):
+    def build_extension(self, ext):
+        copyfile(ext.sources[0], self.get_ext_fullpath(ext.name))
+
+
 # Additional binaries
 if platform.system() == 'Linux':
   libs = ['onnxruntime_pybind11_state.so', 'libmkldnn.so.0', 'libmklml_intel.so', 'libiomp5.so']
@@ -53,7 +65,7 @@ elif platform.system() == "Darwin":
 else:
   libs = ['onnxruntime_pybind11_state.pyd', 'mkldnn.dll', 'mklml.dll', 'libiomp5md.dll']

-data = [path.join('capi', x) for x in libs if path.isfile(path.join('onnxruntime', 'capi', x))]
+data = []  #[path.join('capi', x) for x in libs if path.isfile(path.join('onnxruntime', 'capi', x))]

 # Additional examples
 examples_names = ["mul_1.pb", "logreg_iris.onnx", "sigmoid.onnx"]
@@ -88,7 +100,7 @@ setup(
     long_description=long_description,
     author='Microsoft Corporation',
     author_email='[email protected]',
-    cmdclass={'bdist_wheel': bdist_wheel},
+    cmdclass={'bdist_wheel': bdist_wheel, 'build_ext':onnxruntime_build_ext},
     license="MIT License",
     packages=['onnxruntime',
               'onnxruntime.backend',
@@ -96,6 +108,7 @@ setup(
               'onnxruntime.datasets',
               'onnxruntime.tools',
               ],
+    ext_modules=[onnxruntime_ext],
     package_data={
         'onnxruntime': data + examples + extra,
     },

[izeigerman]

I second this. @mayeut thank you so much for sharing your snippet. Right now we have to maintain our own onnxruntime build in order to use it on CentOS 7 systems. The CentOS 7 and its RedHat counterpart are highly adopted OS in enterprise environments and to have onnxruntime support out of the box would be just great. Especially if it's not too hard to achieve. Are there any plans to fix this?

[asimonov]

I see this issue (Cannot load onnxruntime.capi. Error: '/usr/lib/x86_64-linux-gnu/libstdc++.so.6: versionGLIBCXX_3.4.20' not found`) on our Ubuntu14 (dont ask...) system.
What is the best way for me to make this work?
I installed onnxruntime via pip3 install.
Python 3.6.8

[snnn]

Hi @asimonov

Currently we don't officially support Ubuntu 14.04. We are working on that. If you could upgrade your compiler(gcc or clang) to a newer version that has full C++14 support, then it should work.

[asimonov]

Well, i would love to upgrade but we are on ubuntu14 because we use ROS indigo which needs ubuntu14 :(

On 23 Aug 2019, at 18:42, Changming Sun [e-mail] wrote:

Hi @asimonov

Currently we don't officially support Ubuntu 14.04. We are working on that. If you could upgrade your compiler(gcc or clang) to a newer version that has full C++14 support, then it should work.

—
You are receiving this because you were mentioned.
Reply to this email directly, view it on GitHub, or mute the thread.

[snnn]

@pranavsharma @faxu FYI.

[snnn]

@asimonov, what's your onnxruntime python package version? 0.4.0 or 0.5.0 ?

[asimonov]

i get these versions installing from pip3:

Collecting onnx
  Downloading https://files.pythonhosted.org/packages/88/50/e4a5a869093f35884d1fd95b46b24705ab27adb7e562a2a307523c043be3/onnx-1.5.0-cp36-cp36m-manylinux1_x86_64.whl (7.0MB)
    100% |████████████████████████████████| 7.1MB 4.3MB/s 
Collecting onnxruntime
  Downloading https://files.pythonhosted.org/packages/fa/e1/1548ef61a5c3b583c7ce777c9f38e30b7fdeda309ddcc886b5883cba5771/onnxruntime-0.4.0-cp36-cp36m-manylinux1_x86_64.whl (3.1MB)
    100% |████████████████████████████████| 3.1MB 4.5MB/s 

[jim-meyer]

@asimonov You need to do pip install --upgrade pip. onnxruntime==0.5.0 is built as a universal wheel. Only newer versions of pip support universal wheels. Hence your version of pip isn't seeing onnxruntime==0.5.0..

[asimonov]

thanks! that worked

(I forgot we frozen pip version at some point when there was global pip issue few months back)

[snnn]

Now it is supported in the latest master branch. Please try it again(you need to build onnxruntime from source, or wait for the next release.)

[asimonov-hitachi]

@snnn , thank you for working on it!

I still cannot build the master today.
Get this error:

[  2%] Building CXX object external/protobuf/cmake/CMakeFiles/libprotobuf.dir/__/src/google/protobuf/arena.cc.o
/home/nissan/hitGit/onnxruntime/onnxruntime/core/mlas/lib/x86_64/QgemmU8X8KernelAvx512Common.h: Assembler messages:
/home/nissan/hitGit/onnxruntime/onnxruntime/core/mlas/lib/x86_64/QgemmU8X8KernelAvx512Common.h:326: Error: operand type mismatch for `vpbroadcastw'
/home/nissan/hitGit/onnxruntime/onnxruntime/core/mlas/lib/x86_64/QgemmU8X8KernelAvx512Common.h:326: Error: operand type mismatch for `vpmaddubsw'
/home/nissan/hitGit/onnxruntime/onnxruntime/core/mlas/lib/x86_64/QgemmU8X8KernelAvx512Common.h:326: Error: operand type mismatch for `vpmaddwd'
...

attaching log file as well where it says it cannot understand the option for avx512

CMakeError.log

[snnn]

Hi @asimonov-hitachi

Thank you for helping me testing it. We need to know your operating system version and binutils version. Could you please run the following command and give me the output?

cat /etc/redhat-release 
which as
as --version

[asimonov]

Changming,

It is actually Ubuntu 14, not Redhat. Sorry for confusion. Can open new
issue if its better.

Here are the details of my system:

root@poise:~/hitGit/onnxruntime# ll /etc/*release
-rw-r--r-- 1 root root 105 Mar  1  2019 /etc/lsb-release
-rw-r--r-- 1 root root 249 Mar  1  2019 /etc/os-release
root@poise:~/hitGit/onnxruntime#
root@poise:~/hitGit/onnxruntime#
root@poise:~/hitGit/onnxruntime#
root@poise:~/hitGit/onnxruntime# cat /etc/lsb-release
DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=14.04
DISTRIB_CODENAME=trusty
DISTRIB_DESCRIPTION="Ubuntu 14.04.6 LTS"
root@poise:~/hitGit/onnxruntime# which as
/usr/bin/as
root@poise:~/hitGit/onnxruntime# as --version
GNU assembler (GNU Binutils for Ubuntu) 2.24
Copyright 2013 Free Software Foundation, Inc.
This program is free software; you may redistribute it under the terms of
the GNU General Public License version 3 or later.
This program has absolutely no warranty.
This assembler was configured for a target of `x86_64-linux-gnu'.

[snnn]

Ok. please. And @yuslepukhin will follow up and solve the avx512 issue.