Websocket: fatal error: runtime: out of memory

[antiWalker]

runtime stack:
runtime.throw(0xb5ca45, 0x16)
/usr/local/go/src/runtime/panic.go:608 +0x72
runtime.sysMap(0xc070000000, 0x4000000, 0x1120f38)
/usr/local/go/src/runtime/mem_linux.go:156 +0xc7
runtime.(mheap).sysAlloc(0x1107a60, 0x4000000, 0x1107a88, 0x7f31c9538d18)
/usr/local/go/src/runtime/malloc.go:619 +0x1c7
runtime.(mheap).grow(0x1107a60, 0x3, 0x0)
/usr/local/go/src/runtime/mheap.go:920 +0x42
runtime.(mheap).allocSpanLocked(0x1107a60, 0x3, 0x1120f48, 0xc00)
/usr/local/go/src/runtime/mheap.go:848 +0x337
runtime.(mheap).alloc_m(0x1107a60, 0x3, 0x59, 0x7f31c9876fff)
/usr/local/go/src/runtime/mheap.go:692 +0x119
runtime.(mheap).alloc.func1()
/usr/local/go/src/runtime/mheap.go:759 +0x4c
runtime.(mheap).alloc(0x1107a60, 0x3, 0x7f31c9010059, 0x7f31c9538c80)
/usr/local/go/src/runtime/mheap.go:758 +0x8a
runtime.(mcentral).grow(0x110a3d8, 0x0)
/usr/local/go/src/runtime/mcentral.go:232 +0x94
runtime.(mcentral).cacheSpan(0x110a3d8, 0x7f31c9538c80)
/usr/local/go/src/runtime/mcentral.go:106 +0x2f8
runtime.(mcache).refill(0x7f31d10cd000, 0x45d259)
/usr/local/go/src/runtime/mcache.go:122 +0x95
runtime.(mcache).nextFree.func1()
/usr/local/go/src/runtime/malloc.go:749 +0x32
runtime.systemstack(0x0)
/usr/local/go/src/runtime/asm_amd64.s:351 +0x66
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1229

goroutine 192605 [running]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_amd64.s:311 fp=0xc06ff8cab0 sp=0xc06ff8caa8 pc=0x4598e0
runtime.(mcache).nextFree(0x7f31d10cd000, 0x1059, 0xc06fff6000, 0x7f31c9538c80, 0xc0002c8001)
/usr/local/go/src/runtime/malloc.go:748 +0xb6 fp=0xc06ff8cb08 sp=0xc06ff8cab0 pc=0x40b666
runtime.mallocgc(0x1300, 0x0, 0x1, 0xc06fffac00)
/usr/local/go/src/runtime/malloc.go:903 +0x793 fp=0xc06ff8cba8 sp=0xc06ff8cb08 pc=0x40bfb3
runtime.makechan(0xa52f80, 0x200, 0xc06fffac00)
/usr/local/go/src/runtime/chan.go:99 +0x14c fp=0xc06ff8cbe8 sp=0xc06ff8cba8 pc=0x404b2c
main.NewClient(...)
/Users/wang/go/src/imgo/worker/client.go:37
main.serveWs(0xc000322000, 0xc0c340, 0xc06fc33dc0, 0xc06ffe2100)
/Users/wang/go/src/imgo/worker/websocket.go:59 +0x35b fp=0xc06ff8ccb0 sp=0xc06ff8cbe8 pc=0x9f324b
main.InitWebsocket.func1(0xc0c340, 0xc06fc33dc0, 0xc06ffe2100)
/Users/wang/go/src/imgo/worker/websocket.go:34 +0x4b fp=0xc06ff8cce0 sp=0xc06ff8ccb0 pc=0x9f440b
net/http.HandlerFunc.ServeHTTP(0xb78100, 0xc0c340, 0xc06fc33dc0, 0xc06ffe2100)
/usr/local/go/src/net/http/server.go:1964 +0x44 fp=0xc06ff8cd08 sp=0xc06ff8cce0 pc=0x691914
net/http.(ServeMux).ServeHTTP(0x1101060, 0xc0c340, 0xc06fc33dc0, 0xc06ffe2100)
/usr/local/go/src/net/http/server.go:2361 +0x127 fp=0xc06ff8cd48 sp=0xc06ff8cd08 pc=0x6935c7
net/http.serverHandler.ServeHTTP(0xc000324000, 0xc0c340, 0xc06fc33dc0, 0xc06ffe2100)
/usr/local/go/src/net/http/server.go:2741 +0xab fp=0xc06ff8cd78 sp=0xc06ff8cd48 pc=0x69468b
net/http.(conn).serve(0xc06fdae8c0, 0xc0cd00, 0xc04a051bc0)
/usr/local/go/src/net/http/server.go:1847 +0x646 fp=0xc06ff8cfc8 sp=0xc06ff8cd78 pc=0x690976
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc06ff8cfd0 sp=0xc06ff8cfc8 pc=0x45b841
created by net/http.(Server).Serve
/usr/local/go/src/net/http/server.go:2851 +0x2f5

goroutine 1 [IO wait]:
internal/poll.runtime_pollWait(0x7f31cee61d60, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:173 +0x66
internal/poll.(pollDesc).wait(0xc000326018, 0x72, 0xc00005e200, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9a
internal/poll.(pollDesc).waitRead(0xc000326018, 0xffffffffffffff00, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(FD).Accept(0xc000326000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1a0
net.(netFD).accept(0xc000326000, 0x40bb0f, 0xc06fdae8c0, 0xa0)
/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(TCPListener).accept(0xc000122020, 0xc000567d70, 0xc01d36a, 0x91001d2bc6ad1941)
/usr/local/go/src/net/tcpsock_posix.go:139 +0x2e
net.(TCPListener).AcceptTCP(0xc000122020, 0xc000567d98, 0x494fd6, 0x5cbd509d)
/usr/local/go/src/net/tcpsock.go:247 +0x47
net/http.tcpKeepAliveListener.Accept(0xc000122020, 0xc000567de8, 0x18, 0xc000000300, 0x694bc5)
/usr/local/go/src/net/http/server.go:3232 +0x2f
net/http.(Server).Serve(0xc000324000, 0xc0c8c0, 0xc000122020, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2826 +0x22f
net/http.(Server).ListenAndServe(0xc000324000, 0xc000324000, 0xb78100)
/usr/local/go/src/net/http/server.go:2764 +0xb6
net/http.ListenAndServe(0xc00010ecc0, 0xc, 0x0, 0x0, 0x0, 0xc000109860)
/usr/local/go/src/net/http/server.go:3004 +0x74
main.InitWebsocket(0xc00010ecc0, 0xc, 0x1, 0x0)
/Users/wang/go/src/imgo/worker/websocket.go:36 +0x62
main.main()
/Users/wang/go/src/imgo/worker/main.go:40 +0x1c3

goroutine 17 [syscall, 14 minutes]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:139 +0x9c
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:29 +0x41

goroutine 18 [IO wait, 14 minutes]:
internal/poll.runtime_pollWait(0x7f31cee61f00, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:173 +0x66
internal/poll.(pollDesc).wait(0xc000128098, 0x72, 0xc00005e200, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9a
internal/poll.(pollDesc).waitRead(0xc000128098, 0xffffffffffffff00, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(FD).Accept(0xc000128080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1a0
net.(netFD).accept(0xc000128080, 0xc000042700, 0x30, 0x30)
/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(TCPListener).accept(0xc00000c040, 0x50, 0x7f31d10cd000, 0x0)
/usr/local/go/src/net/tcpsock_posix.go:139 +0x2e
net.(TCPListener).AcceptTCP(0xc00000c040, 0x40c398, 0x30, 0xaf8c60)
/usr/local/go/src/net/tcpsock.go:247 +0x47
net/http.tcpKeepAliveListener.Accept(0xc00000c040, 0xaf8c60, 0xc00001d4a0, 0xa8b9e0, 0x10ee760)
/usr/local/go/src/net/http/server.go:3232 +0x2f
net/http.(Server).Serve(0xc000087110, 0xc0c8c0, 0xc00000c040, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2826 +0x22f
net/http.(Server).ListenAndServe(0xc000087110, 0xc000087110, 0x0)
/usr/local/go/src/net/http/server.go:2764 +0xb6
net/http.ListenAndServe(0xc00010ea20, 0xc, 0xc06180, 0xc0002915c0, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:3004 +0x74
imgo/libs.StartPprof.func1(0xc00028f800, 0xc0002915c0)
/Users/wang/go/src/imgo/libs/pprof.go:17 +0x51
created by imgo/libs.StartPprof
/Users/wang/go/src/imgo/libs/pprof.go:16 +0x184

[antiWalker]

when i stress test my websocket server.when i send the number of conns reach to 30 thousands。the server pannic "fatal error: runtime: out of memory".
the server is 4 core 4G memory.

[antiWalker]

show that i just use 1.9G memory.

[antiWalker]

no body can answer me ?
so disappoint。

[ghost]

There's insufficient information to diagnose the problem you are having with this package. Please provide information describing the actual problem with this package.

no body can answer me ? so disappoint。

This package is maintained by volunteers. They are not at your beck and call, particularly in the wee hours of the morning.

[IngCr3at1on]

This is interesting; I haven't run into anything like this in the past myself. I wonder if we could replicate this behavior in a simple 'proof' (removing any outside factors to ensure this is absolutely related to the websocket implementation).

[antiWalker]

There's insufficient information to diagnose the problem you are having with this package. Please provide information describing the actual problem with this package.

no body can answer me ? so disappoint。

This package is maintained by volunteers. They are not at your beck and call, particularly in the wee hours of the morning.

when i stress it,and the memory is reach 70%. it will be appear。

[ghost]

The only thing we know about what you are testing is that it uses this package. You need to provide more information.

[IngCr3at1on]

note: https://github.com/gorilla/websocket/issues/72 we know gorilla is (or at least was unless something has broken) capable of managing the high connection load you're talking about. Please provide more information or preferably if possible attempt to prove your issue in a small application you can submit here so that one of us may run it.

just to add I hit the echo example server with a modified client on my desktop here with 1000 connections and there's nothing noticeable; if I go much over that I encounter host restrictions not related to the memory (unable to dial the server ;) )

[antiWalker]

note: #72 we know gorilla is (or at least was unless something has broken) capable of managing the high connection load you're talking about. Please provide more information or preferably if possible attempt to prove your issue in a small application you can submit here so that one of us may run it.

just to add I hit the echo example server with a modified client on my desktop here with 1000 connections and there's nothing noticeable; if I go much over that I encounter host restrictions not related to the memory (unable to dial the server ;) )

thank you for your replay, information for you:
just now ,i got a new matchine,its config is 4 core and 16 G memory.
Core logic is 3 goroutines per connection (the heartbeat, the reader and the writer)
ulimit -n 1000000
when i send 100000 connections to my websocket server named "worker".
my server load is :

5.3G/100000=53k ,so 53k per connection
when i send 200000 connections to my websocket server named "worker".
my server load is :

11.5G/200000=57.5k ,so 57.5k per connection
and i continue to stress my websocket server,when the connection number reach to 220 thousand.
memory percentage is 78%，
-bash: fork: Cannot allocate memory
and the error is happend:
fatal error: runtime: out of memory.
so you meet this situation? thank you~

[niaow]

Hi. We cannot really help without any data on memory usage. Please consider sending a heap profile. If you are unsure how to do this, this post may help:
https://jvns.ca/blog/2017/09/24/profiling-go-with-pprof/

[niaow]

Also, as previously stated, a code snippet would help

[ghost]

Please show relevant code in /Users/wang/go/src/imgo/worker/client.go and /Users/wang/go/src/imgo/worker/websocket.go.

The application panics when allocating a channel with a capacity of 512. How large are the elements in that channel?

[antiWalker]

Please show relevant code in /Users/wang/go/src/imgo/worker/client.go and /Users/wang/go/src/imgo/worker/websocket.go.

The application panics when allocating a channel with a capacity of 512. How large are the elements in that channel?
thank you reply me。
some code is：
c.cancel = make(chan int, svr)
c.receiveHeart = make(chan int, svr)
c.startHeart = make(chan int, svr)
c.writeClose = make(chan int, svr)
c.receiveAck = make(chan int, svr)
c.retansClose = make(chan int, svr)
i set svr = 512. why 512 will panic error?
so i should change svr to 256?

[ghost]

The channel allocations are probably not the problem.

I don't know how to help without seeing a memory profile as suggested by jaddr2dude or seeing more of the code in /Users/wang/go/src/imgo/worker/client.go and /Users/wang/go/src/imgo/worker/websocket.go.

[elithrar]

Timing this out - please re-open if you can provide more information (a memory profile, number of available file descriptors on the system, etc.)