Darling: It cannot run ClickHouse Darwin build

[alexey-milovidov]

Expected Result
It should successfully run clickhouse binary.
See https://github.com/ClickHouse/ClickHouse/issues/15609

Actual Result
Segmentation fault at startup.

Steps To Reproduce

$ wget https://builds.clickhouse.tech/master/macos/clickhouse
$ chmod a+x clickhouse
$ darling shell
Darling [/Volumes/SystemRoot/home/milovidov/work/darwin_build]$ ./clickhouse 
Segmentation fault: 11

System Information

| Software | Version |
| --- | --- |
| Linux Kernel | 5.3.0-64-generic |
| Darling | de92485e |

[bugaevc]

(lldb) bt
* thread #1, queue = 'com.apple.main-thread', stop reason = EXC_BAD_ACCESS (code=1, address=0x0)
  * frame #0: 0x0000000000000000
    frame #1: 0x00007fffe9d96c89 libc++.1.dylib`std::__1::codecvt<char, char, __mbstate_t>::encoding(this=0x00007ffff20bb5b0) const at __locale:939
    frame #2: 0x00007fffe9d969c2 libc++.1.dylib`std::__1::__stdinbuf<char>::imbue(this=0x00007fffe9e00a00, __loc=0x00007fffffdfcee0) at __std_stream:84
    frame #3: 0x00007fffe9d968f7 libc++.1.dylib`std::__1::__stdinbuf<char>::__stdinbuf(this=0x00007fffe9e00a00, __fp=0x00007fffea6953c0, __st=0x00007fffe9e00a68) at __std_stream:76
    frame #4: 0x00007fffe9d965b5 libc++.1.dylib`std::__1::__stdinbuf<char>::__stdinbuf(this=0x00007fffe9e00a00, __fp=0x00007fffea6953c0, __st=0x00007fffe9e00a68) at __std_stream:75
    frame #5: 0x00007fffe9d96301 libc++.1.dylib`std::__1::DoIOSInit::DoIOSInit(this=0x00007fffe9e00f50) at iostream.cpp:111
    frame #6: 0x00007fffe9d966f5 libc++.1.dylib`std::__1::DoIOSInit::DoIOSInit(this=0x00007fffe9e00f50) at iostream.cpp:107
    frame #7: 0x00007fffe9d967ea libc++.1.dylib`std::__1::ios_base::Init::Init(this=0x00007fffe9dfff08) at iostream.cpp:152
    frame #8: 0x00007fffe9d962a5 libc++.1.dylib`std::__1::ios_base::Init::Init(this=0x00007fffe9dfff08) at iostream.cpp:151
    frame #9: 0x00007fffe9d98790 libc++.1.dylib`::__cxx_global_var_init() at iostream.cpp:80
    frame #10: 0x00007fffe9d987b9 libc++.1.dylib`_GLOBAL__sub_I_iostream.cpp at iostream.cpp:0
    frame #11: 0x00007ffff8051d90 dyld`ImageLoaderMachO::doModInitFunctions(ImageLoader::LinkContext const&) + 944
    frame #12: 0x00007ffff80524a5 dyld`ImageLoaderMachO::doInitialization(ImageLoader::LinkContext const&) + 69
    frame #13: 0x00007ffff8036d50 dyld`ImageLoader::recursiveInitialization(ImageLoader::LinkContext const&, unsigned int, char const*, ImageLoader::InitializerTimingList&, ImageLoader::UninitedUpwards&) + 1008
    frame #14: 0x00007ffff8036c4f dyld`ImageLoader::recursiveInitialization(ImageLoader::LinkContext const&, unsigned int, char const*, ImageLoader::InitializerTimingList&, ImageLoader::UninitedUpwards&) + 751
    frame #15: 0x00007ffff803315e dyld`ImageLoader::processInitializers(ImageLoader::LinkContext const&, unsigned int, ImageLoader::InitializerTimingList&, ImageLoader::UninitedUpwards&) + 350
    frame #16: 0x00007ffff8033197 dyld`ImageLoader::processInitializers(ImageLoader::LinkContext const&, unsigned int, ImageLoader::InitializerTimingList&, ImageLoader::UninitedUpwards&) + 407
    frame #17: 0x00007ffff8033288 dyld`ImageLoader::runInitializers(ImageLoader::LinkContext const&, ImageLoader::InitializerTimingList&) + 152
    frame #18: 0x00007ffff800300a dyld`dyld::initializeMainExecutable() + 218
    frame #19: 0x00007ffff800ae2c dyld`dyld::_main(macho_header const*, unsigned long, int, char const**, char const**, char const**, unsigned long*) + 5708
    frame #20: 0x00007ffff802efab dyld`dyldbootstrap::start(dyld3::MachOLoaded const*, int, char const**, dyld3::MachOLoaded const*, unsigned long*) + 315
    frame #21: 0x00007ffff8055725 dyld`_dyld_start + 37

[alexey-milovidov]

It should not use libc++ dylib, we link it statically...

[alexey-milovidov]

How to enumerate dylib dependencies of the Mach-O binary? (similar to ldd on Linux)

[bugaevc]

otool -L

In your case, it appears to statically link to everything but libSystem:

Darling [~]$ otool -L Downloads/clickhouse 
Downloads/clickhouse:
    /usr/lib/libSystem.B.dylib (compatibility version 1.0.0, current version 1252.200.5)

[alexey-milovidov]

Ok. As expected.
But why it loads libc++.1.dylib?

[bugaevc]

But why it loads libc++.1.dylib?

As a transitive dependency — libSystem reexports libdispatch, libdispatch depends on libobjc, and libobjc depends on libc++.

I wonder if the crash could be caused by symbol name clashes, assuming your executable exports libc++ symbols.

[alexey-milovidov]

Yes, it's possible.

As I understand, if my binary has some symbol and it's exported, and it loads some dynamic libraries that need this symbol, the symbol from my binary should be preferred for dynamic linking.

[bugaevc]

Yes; but that's also where Mach-O two-level namespace comes into play (or not). Could you please figure out if you export libc++ symbols? :slightly_smiling_face:

[alexey-milovidov]

I'm almost sure we do (for Linux build we export everything, for Darwin cross-compile build it should be similar).

[alexey-milovidov]

My main point is that it works fine on real Mac...

[bugaevc]

Sure, I'm not saying you should change anything. If it works on macOS and doesn't work on Darling, it's our bug. We just need to figure out what's going on and fix it, hence me asking 🙂

[bugaevc]

Yes, symbol conflicts is what's going on:

dyld: weak bind: libc++.1.dylib:0x7FFFE9DFB820 = clickhouse:__ZTv0_n24_NSt3__113basic_ostreamIcNS_11char_traitsIcEEED0Ev, *0x7FFFE9DFB820 = 0x7FFFF0DFF2B0
dyld: weak bind: libc++.1.dylib:0x7FFFE9DFBB80 = clickhouse:__ZTv0_n24_NSt3__113basic_ostreamIcNS_11char_traitsIcEEED0Ev, *0x7FFFE9DFBB80 = 0x7FFFF0DFF2B0
dyld: weak bind: libc++.1.dylib:0x7FFFE9DFF068 = clickhouse:__ZTv0_n24_NSt3__113basic_ostreamIcNS_11char_traitsIcEEED0Ev, *0x7FFFE9DFF068 = 0x7FFFF0DFF2B0
dyld: weak bind: libc++.1.dylib:0x7FFFE9DFF198 = clickhouse:__ZTv0_n24_NSt3__113basic_ostreamIcNS_11char_traitsIcEEED0Ev, *0x7FFFE9DFF198 = 0x7FFFF0DFF2B0

Now the next question is, why. Are we not in two-level namespace mode? Are we building libc++ wrong?

[bugaevc]

Are we building libc++ wrong?

Looking at __ZTCNSt3__114basic_iostreamIcNS_11char_traitsIcEEEE0_NS_13basic_istreamIcS2_EE in particular [0], on Darling it is "weak external", but in Apple's build it's just "external". That explains it.

[0] it's construction vtable for std::__1::basic_istream<char, std::__1::char_traits<char> >-in-std::__1::basic_iostream<char, std::__1::char_traits<char> >, according to c++filt

[bugaevc]

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 505017261..c54aef8eb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,6 +1,6 @@
 project(libcxx)

-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostdinc -std=c++11 -mmacosx-version-min=10.10")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -nostdinc -std=c++11 -mmacosx-version-min=10.10 -fvisibility=hidden -fvisibility-inlines-hidden")

 include_directories(
        BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/include
@@ -64,8 +64,11 @@ add_darling_library(cxx SHARED $<TARGET_OBJECTS:cxx_obj>)
 make_fat(cxx)
 target_link_libraries(cxx PRIVATE system cxxabi)
 set_target_properties(cxx PROPERTIES OUTPUT_NAME "c++.1")
-set_property(TARGET cxx APPEND_STRING PROPERTY LINK_FLAGS " -Wl,-reexport_library,${CMAKE_BINARY_DIR}/src/external/libcxxabi/libc++abi.dylib \
-      -Wl,-unexported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/lib/libc++unexp.exp ")
+set_property(TARGET cxx APPEND_STRING PROPERTY LINK_FLAGS " \
+      -Wl,-reexport_library,${CMAKE_BINARY_DIR}/src/external/libcxxabi/libc++abi.dylib \
+      -Wl,-unexported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/lib/libc++unexp.exp \
+      -Wl,-force_symbols_not_weak_list,${CMAKE_CURRENT_SOURCE_DIR}/lib/notweak.exp \
+      -Wl,-force_symbols_weak_list,${CMAKE_CURRENT_SOURCE_DIR}/lib/weak.exp ")

 install(TARGETS cxx DESTINATION libexec/darling/usr/lib)
 InstallSymlink("libc++.1.dylib" "${CMAKE_INSTALL_PREFIX}/libexec/darling/usr/lib/libc++.dylib")

gets me

Darling [~]$ Downloads/clickhouse 
Use one of the following commands:
clickhouse local [args] 
clickhouse client [args] 
clickhouse benchmark [args] 
clickhouse server [args] 
clickhouse extract-from-config [args] 
clickhouse compressor [args] 
clickhouse format [args] 
clickhouse copier [args] 
clickhouse obfuscator [args] 
clickhouse install [args] 
clickhouse start [args] 
clickhouse stop [args] 
clickhouse status [args] 
clickhouse restart [args] 
Darling [~]$ 

[bugaevc]

There are still similar differences though; here's libdyld using flat namespace:

Darling [~]$ nm -m /usr/lib/system/libdyld.dylib | grep malloc
                 (undefined) external _malloc
                 (undefined) external _malloc_size

macOS ~ % nm -m /usr/lib/system/libdyld.dylib| grep malloc
                 (undefined) external _calloc (from libsystem_malloc)
                 (undefined) external _free (from libsystem_malloc)
                 (undefined) external _malloc (from libsystem_malloc)
                 (undefined) external _malloc_size (from libsystem_malloc)