Uvloop: call sock_accept(or sock_recv) and close sock before next uv loop will cause entire program core dump

[chenfengyuan]

• uvloop version: git 490c410(current master)
• Python version: Python 3.6.1
• Platform: Linux 4.10.0-26-generic #30-Ubuntu SMP Tue Jun 27 09:30:12 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux
• Can you reproduce the bug with PYTHONASYNCIODEBUG in env?: yes

Hi all,

call sock_accept and close sock before next uv loop will cause entire program core dump. Because epoll will return EBADF if libuv try register closed socket, then libuv will call abort. https://github.com/libuv/libuv/blob/7452ef4e06a4f99ee26b694c65476401534f2725/src/unix/linux-core.c#L225-L227

Run any code below directly can reproduce core dump. Core dump may cause programs hard to debug and make programmer take more time to find the root bug source.
If I don't use uvloop(comment asyncio.set_event_loop(uvloop.new_event_loop())), all code below will be fine.

sock_accept

# coding=utf-8
import uvloop
import asyncio
from signal import (
    signal,
    SIGINT,
)
import socket
from contextlib import (
    closing,
)
import logging
logging.basicConfig(format='%(asctime)s [%(process)d] [%(levelname)s] [%(filename)s:%(lineno)d] : %(message)s')


async def a():
    loop = asyncio.get_event_loop()
    sock = socket.socket()
    sock.bind(('127.0.0.1', 0))
    sock.listen(10)
    sock.setblocking(0)
    loop.sock_accept(sock)
    sock.close()

async def b():
    await asyncio.sleep(1)


def main():
    asyncio.set_event_loop(uvloop.new_event_loop())
    loop = asyncio.get_event_loop()
    loop.set_debug(True)
    signal(SIGINT, lambda s, f: loop.stop())
    loop.create_task(a())
    # noinspection PyBroadException
    try:
        loop.run_until_complete(loop.create_task(b()))
        # loop.run_forever()
    except:
        loop.stop()


if __name__ == '__main__':
    main()

sock_recv

# coding=utf-8
import uvloop
import asyncio
from signal import (
    signal,
    SIGINT,
)
import socket
from contextlib import (
    closing,
)
import logging
logging.basicConfig(format='%(asctime)s [%(process)d] [%(levelname)s] [%(filename)s:%(lineno)d] : %(message)s')


logger = logging.getLogger(__name__)


async def a():
    loop = asyncio.get_event_loop()
    sock = socket.socket()
    sock.connect(('github.com', 80))
    sock.setblocking(0)
    loop.sock_recv(sock, 10)
    sock.close()

async def b():
    await asyncio.sleep(1)


def main():
    asyncio.set_event_loop(uvloop.new_event_loop())
    loop = asyncio.get_event_loop()
    loop.set_debug(True)
    signal(SIGINT, lambda s, f: loop.stop())
    loop.create_task(a())
    # noinspection PyBroadException
    try:
        loop.run_until_complete(loop.create_task(b()))
        # loop.run_forever()
    except:
        loop.stop()


if __name__ == '__main__':
    main()

gdb backtrace

#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:58
#1  0x00007ffff711937a in __GI_abort () at abort.c:89
#2  0x00007ffff304bb6e in uv__io_poll (loop=loop@entry=0x555555c7b670, timeout=1000) at src/unix/linux-core.c:227
#3  0x00007ffff3041394 in uv_run (loop=0x555555c7b670, mode=mode@entry=UV_RUN_DEFAULT) at src/unix/core.c:352
#4  0x00007ffff2edb826 in __pyx_f_6uvloop_4loop_4Loop___run (__pyx_v_self=0x7ffff2c18028, 
    __pyx_v_mode=UV_RUN_DEFAULT) at uvloop/loop.c:11736
#5  0x00007ffff2fed6e8 in __pyx_f_6uvloop_4loop_4Loop__run (__pyx_v_self=0x7ffff2c18028, 
    __pyx_v_mode=UV_RUN_DEFAULT) at uvloop/loop.c:12205
#6  0x00007ffff2effbcd in __pyx_pf_6uvloop_4loop_4Loop_24run_forever (__pyx_v_self=0x7ffff2c18028)
    at uvloop/loop.c:25171
#7  __pyx_pw_6uvloop_4loop_4Loop_25run_forever (__pyx_v_self=0x7ffff2c18028, unused=<optimized out>)
    at uvloop/loop.c:24957
#8  0x00007ffff2eac36b in __Pyx_PyObject_CallMethO (arg=0x0, func=0x7ffff32b4990) at uvloop/loop.c:137637
#9  __Pyx_PyObject_CallNoArg (func=0x7ffff32b4990) at uvloop/loop.c:6634
#10 0x00007ffff30281c8 in __pyx_pf_6uvloop_4loop_4Loop_44run_until_complete (__pyx_v_future=0x7ffff32c5158, 
    __pyx_v_self=<optimized out>) at uvloop/loop.c:26507
#11 __pyx_pw_6uvloop_4loop_4Loop_45run_until_complete (__pyx_v_self=<optimized out>, 
    __pyx_v_future=<optimized out>) at uvloop/loop.c:26195
#12 0x0000555555647e64 in _PyCFunction_FastCallDict (kwargs=0x0, nargs=1, args=0x7ffff39d55a0, 
    func_obj=0x7ffff32b41f8) at Objects/methodobject.c:209
#13 _PyCFunction_FastCallKeywords (func=func@entry=0x7ffff32b41f8, stack=stack@entry=0x7ffff39d55a0, nargs=1, 
    kwnames=kwnames@entry=0x0) at Objects/methodobject.c:295
#14 0x00005555556da6fe in call_function (pp_stack=pp_stack@entry=0x7fffffffd0a8, oparg=oparg@entry=1, 
    kwnames=kwnames@entry=0x0) at Python/ceval.c:4798
#15 0x00005555556dfea3 in _PyEval_EvalFrameDefault (f=<optimized out>, throwflag=<optimized out>)
    at Python/ceval.c:3284
#16 0x00005555556da34d in PyEval_EvalFrameEx (throwflag=0, f=0x7ffff39d5418) at Python/ceval.c:718
#17 _PyEval_EvalCodeWithName (_co=0x7ffff7ed3db0, globals=globals@entry=0x7ffff7f58240, locals=locals@entry=0x0, 
    args=<optimized out>, argcount=0, kwnames=kwnames@entry=0x0, kwargs=0x7ffff7f86ba0, kwcount=0, kwstep=1, 
    defs=0x0, defcount=0, kwdefs=0x0, closure=0x0, name=0x7ffff7e9eb90, qualname=0x7ffff7e9eb90)
    at Python/ceval.c:4128
#18 0x00005555556da614 in fast_function (kwnames=0x0, nargs=<optimized out>, stack=<optimized out>, 
    func=0x7ffff32bfd08) at Python/ceval.c:4939
#19 call_function (pp_stack=pp_stack@entry=0x7fffffffd338, oparg=oparg@entry=0, kwnames=kwnames@entry=0x0)
    at Python/ceval.c:4819
#20 0x00005555556dfea3 in _PyEval_EvalFrameDefault (f=<optimized out>, throwflag=<optimized out>)
    at Python/ceval.c:3284
#21 0x00005555556da34d in PyEval_EvalFrameEx (throwflag=0, f=0x7ffff7f86a20) at Python/ceval.c:718
#22 _PyEval_EvalCodeWithName (_co=_co@entry=0x7ffff7ed3e40, globals=globals@entry=0x7ffff7f6f150, 
    locals=locals@entry=0x7ffff7ed3e40, args=args@entry=0x0, argcount=argcount@entry=0, kwnames=kwnames@entry=0x0, 
    kwargs=0x8, kwcount=0, kwstep=2, defs=0x0, defcount=0, kwdefs=0x0, closure=0x0, name=0x0, qualname=0x0)
    at Python/ceval.c:4128
#23 0x00005555556db1c3 in PyEval_EvalCodeEx (closure=0x0, kwdefs=0x0, defcount=0, defs=0x0, kwcount=0, kws=0x0, 
    argcount=0, args=0x0, locals=locals@entry=0x7ffff7ed3e40, globals=globals@entry=0x7ffff7f6f150, 
    _co=_co@entry=0x7ffff7ed3e40) at Python/ceval.c:4149
#24 PyEval_EvalCode (co=co@entry=0x7ffff7ed3e40, globals=globals@entry=0x7ffff7f58240, 
    locals=locals@entry=0x7ffff7f58240) at Python/ceval.c:695
#25 0x00005555555b2eb9 in run_mod (arena=0x7ffff7f6f150, flags=0x7fffffffd63c, locals=0x7ffff7f58240,

[chenfengyuan]

Maybe we could use a socket wrapper that makes close and other methods async, to avoid this problem entirely.I suppose asyncio without uvloop also need an async version of close( in this gist asyncio will wait forever on sock_recv). Or can we use a modified version of libuv? There is already a libuv compatible hack

[ly0]

it's very helpful to me!

[1st1]

There's not much we can do here. uvloop can't detect/intercept socket.close(), and libuv/epoll have no mechanisms to behave correctly when such an unexpected thing happens.

I filed an issue to add new API to CPython https://bugs.python.org/issue32038 to intercept socket.close() calls.

[1st1]

@saghul Do you think it's possible to fix epoll code in libuv like this?

diff --git a/src/unix/linux-core.c b/src/unix/linux-core.c
index 4d480ce1..29ca3fe2 100644
--- a/src/unix/linux-core.c
+++ b/src/unix/linux-core.c
@@ -241,6 +241,10 @@ void uv__io_poll(uv_loop_t* loop, int timeout) {
      * events, skip the syscall and squelch the events after epoll_wait().
      */
     if (uv__epoll_ctl(loop->backend_fd, op, w->fd, &e)) {
+      if (errno == EBADF) {
+        uv__io_stop(loop, w, POLLIN | POLLOUT | UV__POLLRDHUP | UV__POLLPRI);
+        continue;
+      }
       if (errno != EEXIST)
         abort();

[chenfengyuan]

@1st1 I filed an issue months ago (https://bugs.python.org/issue30996).
I suppose callback or loop.sock_close can solve the problem to some degree. Either way is better than do nothing.

[1st1]

Good news! It appears there's undocumented API for preventing socket objects from closing. uvloop will no longer crash. The fix will be in the next release (soon).

[1st1]

Please try uvloop v0.9.0.