Reactor-netty: LEAK: ByteBuf.release() was not called before it's garbage-collected
Expected behavior
No leak
Actual behavior
LEAK: ByteBuf.release() was not called before it's garbage-collected
LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records:
#1:
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:273)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:646)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:581)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#2:
io.netty.buffer.AdvancedLeakAwareByteBuf.forEachByte(AdvancedLeakAwareByteBuf.java:670)
io.netty.handler.codec.http.HttpObjectDecoder$HeaderParser.parse(HttpObjectDecoder.java:801)
io.netty.handler.codec.http.HttpObjectDecoder.readHeaders(HttpObjectDecoder.java:601)
io.netty.handler.codec.http.HttpObjectDecoder.decode(HttpObjectDecoder.java:227)
io.netty.handler.codec.http.HttpServerCodec$HttpServerRequestDecoder.decode(HttpServerCodec.java:101)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:646)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:581)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#3:
io.netty.buffer.AdvancedLeakAwareByteBuf.forEachByte(AdvancedLeakAwareByteBuf.java:670)
io.netty.handler.codec.http.HttpObjectDecoder$HeaderParser.parse(HttpObjectDecoder.java:801)
io.netty.handler.codec.http.HttpObjectDecoder.readHeaders(HttpObjectDecoder.java:581)
io.netty.handler.codec.http.HttpObjectDecoder.decode(HttpObjectDecoder.java:227)
io.netty.handler.codec.http.HttpServerCodec$HttpServerRequestDecoder.decode(HttpServerCodec.java:101)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:646)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:581)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#4:
io.netty.buffer.AdvancedLeakAwareByteBuf.getUnsignedByte(AdvancedLeakAwareByteBuf.java:160)
io.netty.handler.codec.http.HttpObjectDecoder.skipControlCharacters(HttpObjectDecoder.java:566)
io.netty.handler.codec.http.HttpObjectDecoder.decode(HttpObjectDecoder.java:202)
io.netty.handler.codec.http.HttpServerCodec$HttpServerRequestDecoder.decode(HttpServerCodec.java:101)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:646)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:581)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#5: Hint: 'reactor.left.httpCodec' will handle the message from this point. io.netty.channel.DefaultChannelPipeline.touch(DefaultChannelPipeline.java:116)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:646)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:581)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#6: Hint: 'DefaultChannelPipeline$HeadContext#0' will handle the message from this point. io.netty.channel.DefaultChannelPipeline.touch(DefaultChannelPipeline.java:116)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:646)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:581)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#7: io.netty.buffer.AdvancedLeakAwareByteBuf.writeBytes(AdvancedLeakAwareByteBuf.java:634)
io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:345)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:148)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:646)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:581)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
Created at: io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:331)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:185)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:176)
io.netty.buffer.AbstractByteBufAllocator.ioBuffer(AbstractByteBufAllocator.java:137)
io.netty.channel.DefaultMaxMessagesRecvByteBufAllocator$MaxMessageHandle.allocate(DefaultMaxMessagesRecvByteBufAllocator.java:114)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:147)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:646)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:581)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:498)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:460)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
: 24 leak records were discarded because they were duplicates : 24 leak records were discarded because the leak record count is targeted to 4. Use system property io.netty.leakDetection.targetRecords to increase the limit.
Steps to reproduce
Unfortunately I have no idea how to reproduce it as this error happens randomly in our service in production.
Reactor Netty version
0.7.8
JVM version (e.g. java -version)
Java 8u181
typbug
madgnome
👍11
All 71 comments
That's pretty interesting is this a particular service, using reactor directly or via something else (spring ?)
smaldini
on 31 Aug 2018
🤦♂️ oops forgot to mention that.
It's a service using Spring Boot 2.0.4 with webflux
madgnome
on 31 Aug 2018
@madgnome did you run with -Dio.netty.leakDetection.level=paranoid? Also enable logging logging.level.reactor.ipc.netty=DEBUG, thus we will have more information in the stack traces with the leaked memory.
violetagg
on 31 Aug 2018
@madgnome Are you using ReactiveAuthenticationManagerAdapter or Scheduling on elastic?
dave-fl
on 31 Aug 2018
@violetagg haven't tried in paranoid mode this it happens in production.
@dave-fl yes we're using ReactiveAuthenticationManagerAdapter
madgnome
on 1 Sep 2018
We began doing a more end to end style test and it failed after 11 hours. So although we got a longer (by removing elastic) test run, things still failed. Here are some of the recent leak logs.
This is on the latest snapshot and Netty 4.1.29
2018-09-01T14:38:07.83-0400 [APP/PROC/WEB/0] OUT 2018-09-01 18:38:07.831 ERROR 19 --- [server-epoll-13] o.s.w.s.adapter.HttpWebHandlerAdapter : Unhandled failure: Connection has been closed, response already set (status=200)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:315)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT java.lang.Thread.run(Thread.java:748)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT #3:
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe$1.run(AbstractEpollChannel.java:382)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.access$300(SslHandler.java:165)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1297)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.buffer.AdvancedLeakAwareByteBuf.nioBufferCount(AdvancedLeakAwareByteBuf.java:706)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:404)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT java.lang.Thread.run(Thread.java:748)
2018-09-01T14:38:08.71-0400 [APP/PROC/WEB/0] OUT : 4 leak records were discarded because the leak record count is targeted to 4. Use system property io.netty.leakDetection.targetRecords to increase the limit.
2018-09-01T14:38:10.34-0400 [APP/PROC/WEB/0] OUT 2018-09-01 18:38:10.342 ERROR 19 --- [server-epoll-16] o.s.w.s.adapter.HttpWebHandlerAdapter : Unhandled failure: Connection has been closed, response already set (status=200)
2018-09-01T14:59:52.17-0400 [APP/PROC/WEB/0] OUT 2018-09-01 18:59:52.179 ERROR 19 --- [client-epoll-22] io.netty.util.ResourceLeakDetector : LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
2018-09-01T14:59:52.17-0400 [APP/PROC/WEB/0] OUT Recent access records:
2018-09-01T14:59:52.17-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:273)
2018-09-01T14:59:52.17-0400 [APP/PROC/WEB/0] OUT io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
2018-09-01T14:59:52.17-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.http.HttpObjectDecoder$LineParser.parse(HttpObjectDecoder.java:850)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.http.HttpObjectDecoder$HeaderParser.parse(HttpObjectDecoder.java:801)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.buffer.AdvancedLeakAwareByteBuf.getUnsignedByte(AdvancedLeakAwareByteBuf.java:160)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.http.HttpObjectDecoder.skipControlCharacters(HttpObjectDecoder.java:566)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.http.HttpClientCodec$Decoder.decode(HttpClientCodec.java:202)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT java.lang.Thread.run(Thread.java:748)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT #6:
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.DefaultChannelPipeline.touch(DefaultChannelPipeline.java:116)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1297)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT java.lang.Thread.run(Thread.java:748)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.allocate(SslHandler.java:1909)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
2018-09-01T14:59:52.18-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
2018-09-01T14:59:52.75-0400 [APP/PROC/WEB/0] OUT 2018-09-01 18:59:52.753 ERROR 19 --- [server-epoll-12] o.s.w.s.adapter.HttpWebHandlerAdapter : Unhandled failure: Connection has been closed, response already set (status=200)
2018-09-01T15:04:16.17-0400 [APP/PROC/WEB/0] OUT 2018-09-01 19:04:16.170 WARN 19 --- [server-epoll-14] o.s.h.s.r.ReactorHttpHandlerAdapter : Handling completed with error: Connection has been closed
2018-09-01T15:04:17.65-0400 [APP/PROC/WEB/0] OUT at reactor.core.publisher.FluxConcatMap$ConcatMapImmediate.drain(FluxConcatMap.java:349) ~[reactor-core-3.1.9.BUILD-SNAPSHOT.jar!/:3.1.9.BUILD-SNAPSHOT]
2018-09-01T15:04:17.65-0400 [APP/PROC/WEB/0] OUT at reactor.core.publisher.FluxRetryWhen$RetryWhenMainSubscriber.onComplete(FluxRetryWhen.java:164) ~[reactor-core-3.1.9.BUILD-SNAPSHOT.jar!/:3.1.9.BUILD-SNAPSHOT]
2018-09-01T15:04:17.65-0400 [APP/PROC/WEB/0] OUT at reactor.core.publisher.MonoPeekTerminal$MonoTerminalPeekSubscriber.onComplete(MonoPeekTerminal.java:318) ~[reactor-core-3.1.9.BUILD-SNAPSHOT.jar!/:3.1.9.BUILD-SNAPSHOT]
2018-09-01T15:04:17.65-0400 [APP/PROC/WEB/0] OUT at reactor.core.publisher.FluxPeek$PeekSubscriber.onComplete(FluxPeek.java:245) ~[reactor-core-3.1.9.BUILD-SNAPSHOT.jar!/:3.1.9.BUILD-SNAPSHOT]
2018-09-01T15:04:17.65-0400 [APP/PROC/WEB/0] OUT at reactor.core.publisher.FluxMap$MapSubscriber.onComplete(FluxMap.java:130) ~[reactor-core-3.1.9.BUILD-SNAPSHOT.jar!/:3.1.9.BUILD-SNAPSHOT]
2018-09-01T15:04:17.65-0400 [APP/PROC/WEB/0] OUT at reactor.core.publisher.FluxRetryPredicate$RetryPredicateSubscriber.onComplete(FluxRetryPredicate.java:107) ~[reactor-core-3.1.9.BUILD-SNAPSHOT.jar!/:3.1.9.BUILD-SNAPSHOT]
2018-09-01T15:04:17.65-0400 [APP/PROC/WEB/0] OUT at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[netty-transport-4.1.29.Final.jar!/:4.1.29.Final]
2018-09-01T15:04:17.65-0400 [APP/PROC/WEB/0] OUT at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310) ~[netty-codec-4.1.29.Final.jar!/:4.1.29.Final]
2018-09-01T15:04:17.65-0400 [APP/PROC/WEB/0] OUT at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253) ~[netty-transport-4.1.29.Final.jar!/:4.1.29.Final]
2018-09-01T15:04:17.65-0400 [APP/PROC/WEB/0] OUT at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) ~[netty-transport-4.1.29.Final.jar!/:4.1.29.Final]
2018-09-01T15:04:17.65-0400 [APP/PROC/WEB/0] OUT at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428) ~[netty-codec-4.1.29.Final.jar!/:4.1.29.Final]
2018-09-01T15:04:18.70-0400 [APP/PROC/WEB/0] OUT 2018-09-01 19:04:18.709 ERROR 19 --- [server-epoll-10] o.s.w.s.adapter.HttpWebHandlerAdapter : Unhandled failure: Connection has been closed, response already set (status=200)
2018-09-01T15:04:19.52-0400 [APP/PROC/WEB/0] OUT 2018-09-01 19:04:19.520 ERROR 19 --- [server-epoll-11] o.s.w.s.adapter.HttpWebHandlerAdapter : Unhandled failure: Connection has been closed, response already set (status=200)
2018-09-01T15:04:20.04-0400 [APP/PROC/WEB/0] OUT 2018-09-01 19:04:20.049 WARN 19 --- [server-epoll-16] o.s.h.s.r.ReactorHttpHandlerAdapter : Handling completed with error: Connection has been closed
2018-09-01T15:04:21.63-0400 [APP/PROC/WEB/0] OUT 2018-09-01 19:04:21.636 ERROR 19 --- [server-epoll-13] o.s.w.s.adapter.HttpWebHandlerAdapter : Unhandled failure: Connection has been closed, response already set (status=200)
2018-09-01T15:04:21.63-0400 [APP/PROC/WEB/0] OUT 2018-09-01 19:04:21.636 WARN 19 --- [server-epoll-13] o.s.h.s.r.ReactorHttpHandlerAdapter : Handling completed with error: Connection has been closed
2018-09-01T15:04:22.57-0400 [APP/PROC/WEB/0] OUT 2018-09-01 19:04:22.574 WARN 19 --- [server-epoll-12] o.s.h.s.r.ReactorHttpHandlerAdapter : Handling completed with error: Connection has been closed
2018-09-01T15:04:25.01-0400 [APP/PROC/WEB/0] OUT 2018-09-01 19:04:25.011 ERROR 19 --- [server-epoll-16] o.s.w.s.adapter.HttpWebHandlerAdapter : Unhandled failure: Connection has been closed, response already set (status=200)
2018-09-01T15:04:26.49-0400 [APP/PROC/WEB/0] OUT 2018-09-01 19:04:26.490 WARN 19 --- [server-epoll-16] o.s.h.s.r.ReactorHttpHandlerAdapter : Handling completed with error: Connection has been closed
2018-09-01T15:04:26.70-0400 [APP/PROC/WEB/0] OUT 2018-09-01 19:04:26.706 WARN 19 --- [server-epoll-11] o.s.h.s.r.ReactorHttpHandlerAdapter : Handling completed with error: Connection has been closed
2018-09-01T15:04:26.71-0400 [APP/PROC/WEB/0] OUT 2018-09-01 19:04:26.713 WARN 19 --- [server-epoll-14] o.s.h.s.r.ReactorHttpHandlerAdapter : Handling completed with error: Connection has been closed
2018-09-01T15:04:27.28-0400 [APP/PROC/WEB/1] OUT 2018-09-01 19:04:27.284 ERROR 15 --- [server-epoll-12] o.s.w.s.adapter.HttpWebHandlerAdapter : Unhandled failure: Connection has been closed, response already set (status=200)
2018-09-01T15:04:27.44-0400 [APP/PROC/WEB/0] OUT 2018-09-01 19:04:27.440 WARN 19 --- [server-epoll-16] o.s.h.s.r.ReactorHttpHandlerAdapter : Handling completed with error: Connection has been closed
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT 2018-09-01 19:04:27.741 ERROR 19 --- [client-epoll-17] io.netty.util.ResourceLeakDetector : LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:273)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1315)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT #3:
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT java.lang.Thread.run(Thread.java:748)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.buffer.AdvancedLeakAwareByteBuf.internalNioBuffer(AdvancedLeakAwareByteBuf.java:736)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.toByteBuffer(SslHandler.java:1439)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1297)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe$1.run(AbstractEpollChannel.java:382)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:404)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT java.lang.Thread.run(Thread.java:748)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT #5:
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.buffer.AdvancedLeakAwareByteBuf.nioBufferCount(AdvancedLeakAwareByteBuf.java:706)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler$SslEngineType$1.unwrap(SslHandler.java:217)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:315)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:176)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
2018-09-01T15:04:27.74-0400 [APP/PROC/WEB/0] OUT io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163)
2018-09-01T15:04:27.79-0400 [APP/PROC/WEB/0] OUT 2018-09-01 19:04:27.795 ERROR 19 --- [server-epoll-10] o.s.w.s.adapter.HttpWebHandlerAdapter : Unhandled failure: Connection has been closed, response already set (status=200)
@dave-fl I cannot see in these stack traces reactor netty classes.
The recommendation here will be really helpful as we add additional hints to the Netty's leak detection mechanism (if it is not production of course). These hints will help a lot.
Also recently I fixed several possible memory leaks when the request/response fails, so if you can try Reactor Netty 0.7.9.BUILD-SNAPSHOT.
Regards,
Violeta
violetagg
on 1 Sep 2018
I will try but it might blow up as we are running this with production style load and if we do that, we might not be able to simulate the timeouts being observed.
Note that the Cloud Gateway code is using Mono.timeout on http client, not sure if something odd could be going on here.
dave-fl
on 1 Sep 2018
@dave-fl then instead of all reactor netty logging, enable just this location reactor.ipc.netty.channel.FluxReceive
violetagg
on 1 Sep 2018
@dave-fl I need the stack traces
violetagg
on 1 Sep 2018
In an effort to try to simplify this. I'm reproducing with a single user. I can trigger things pretty easily, there is no particular unique action that is being performed that causes this to happen consistently. It's just random e.g. if I do the same action 100 times, it will happen a few times.
When it does take place there are no errors, i.e. all requests are coming back correctly from the source systems.
This may or may not be relevant but the logs are littered with this (even though no errors are taking place anywhere i.e. all requests are being properly sent and returned from source and destination systems).
2018-09-02 12:29:52.080 DEBUG 36264 --- [ctor-http-nio-1] r.i.n.http.server.HttpServerOperations : [id: 0xcfe94cb6, L:/127.0.0.1:8090 - R:/127.0.0.1:24901] Decreasing pending responses, now 0
2018-09-02 12:29:52.080 DEBUG 36264 --- [ctor-http-nio-1] r.ipc.netty.http.server.HttpServer : [id: 0xcfe94cb6, L:/127.0.0.1:8090 - R:/127.0.0.1:24901] FLUSH
2018-09-02 12:29:52.081 DEBUG 36264 --- [ctor-http-nio-1] r.ipc.netty.channel.ChannelOperations : [id: 0xcfe94cb6, L:/127.0.0.1:8090 - R:/127.0.0.1:24901] Discarding inbound content
2018-09-02 12:29:52.081 DEBUG 36264 --- [ctor-http-nio-1] r.ipc.netty.http.server.HttpServer : [id: 0xcfe94cb6, L:/127.0.0.1:8090 - R:/127.0.0.1:24901] USER_EVENT: [Handler Terminated]
2018-09-02 12:29:52.081 DEBUG 36264 --- [ctor-http-nio-1] r.ipc.netty.http.server.HttpServer : [id: 0xcfe94cb6, L:/127.0.0.1:8090 - R:/127.0.0.1:24901] READ COMPLETE
2018-09-02 12:29:52.081 DEBUG 36264 --- [ctor-http-nio-1] r.ipc.netty.http.server.HttpServer : [id: 0xcfe94cb6, L:/127.0.0.1:8090 - R:/127.0.0.1:24901] EXCEPTION: java.io.IOException: An established connection was aborted by the software in your host machine
java.io.IOException: An established connection was aborted by the software in your host machine
at sun.nio.ch.SocketDispatcher.read0(Native Method) ~[na:1.8.0_144]
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:43) ~[na:1.8.0_144]
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[na:1.8.0_144]
at sun.nio.ch.IOUtil.read(IOUtil.java:192) ~[na:1.8.0_144]
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) ~[na:1.8.0_144]
at io.netty.buffer.PooledUnsafeDirectByteBuf.setBytes(PooledUnsafeDirectByteBuf.java:288) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1108) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
at io.netty.buffer.WrappedByteBuf.writeBytes(WrappedByteBuf.java:813) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
at io.netty.buffer.AdvancedLeakAwareByteBuf.writeBytes(AdvancedLeakAwareByteBuf.java:635) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:347) ~[netty-transport-4.1.29.Final.jar:4.1.29.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:148) ~[netty-transport-4.1.29.Final.jar:4.1.29.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628) [netty-transport-4.1.29.Final.jar:4.1.29.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563) [netty-transport-4.1.29.Final.jar:4.1.29.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480) [netty-transport-4.1.29.Final.jar:4.1.29.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442) [netty-transport-4.1.29.Final.jar:4.1.29.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884) [netty-common-4.1.29.Final.jar:4.1.29.Final]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_144]
Hopefully the below stack is useful, if you want to add more logging I can test with that as well.
2018-09-02 12:52:01.442 DEBUG 7772 --- [reactor-http-nio-1] r.ipc.netty.http.client.HttpClient : [id: 0x6dfc389e, L:- R:somehost:443] FLUSH
2018-09-02 12:52:01.408 DEBUG 7772 --- [reactor-http-nio-3] r.ipc.netty.http.client.HttpClient : [id: 0x5405c3bc, L:- R:somehost:443] READ: 14346B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
APP RESPONSE
+--------+-------------------------------------------------+----------------+
2018-09-02 12:52:01.444 DEBUG 7772 --- [reactor-http-nio-1] r.i.n.channel.ChannelOperationsHandler : [id: 0x6dfc389e, L:/10. - R:somehost:443] Writing object
2018-09-02 12:52:01.445 DEBUG 7772 --- [reactor-http-nio-1] r.ipc.netty.http.server.HttpServer : [id: 0x2a919b00, L:/0:0:0:0:0:0:0:0:8090] READ: [id: 0x608758dd, L:/127.0.0.1:8090 - R:/127.0.0.1:29111]
2018-09-02 12:52:01.445 DEBUG 7772 --- [reactor-http-nio-2] reactor.ipc.netty.channel.FluxReceive : [id: 0x9e96cbfd, L:/127.0.0.1:8090 - R:/127.0.0.1:29110] Subscribing inbound receiver [pending: 0, cancelled:false, inboundDone: true]
2018-09-02 12:52:01.445 DEBUG 7772 --- [reactor-http-nio-1] r.ipc.netty.http.server.HttpServer : [id: 0x2a919b00, L:/0:0:0:0:0:0:0:0:8090] READ COMPLETE
2018-09-02 12:52:01.445 DEBUG 7772 --- [reactor-http-nio-1] r.i.n.channel.ChannelOperationsHandler : [id: 0x6dfc389e, L:/10. - R:somehost:443] Writing object EmptyLastHttpContent
2018-09-02 12:52:01.445 DEBUG 7772 --- [reactor-http-nio-1] r.ipc.netty.http.client.HttpClient : [id: 0x6dfc389e, L:/10. - R:somehost:443] WRITE: 0B
2018-09-02 12:52:01.445 DEBUG 7772 --- [reactor-http-nio-1] r.ipc.netty.http.client.HttpClient : [id: 0x6dfc389e, L:/10. - R:somehost:443] FLUSH
2018-09-02 12:52:01.544 ERROR 7772 --- [reactor-http-nio-1] io.netty.util.ResourceLeakDetector : LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records:
#1:
Hint: Buffered ByteBufHolder in Inbound Flux Queue
io.netty.handler.codec.http.DefaultHttpContent.touch(DefaultHttpContent.java:88)
io.netty.handler.codec.http.DefaultLastHttpContent.touch(DefaultLastHttpContent.java:88)
io.netty.handler.codec.http.DefaultLastHttpContent.touch(DefaultLastHttpContent.java:28)
reactor.ipc.netty.channel.FluxReceive.onInboundNext(FluxReceive.java:324)
reactor.ipc.netty.channel.ChannelOperations.onInboundNext(ChannelOperations.java:311)
reactor.ipc.netty.http.client.HttpClientOperations.onInboundNext(HttpClientOperations.java:605)
reactor.ipc.netty.channel.ChannelOperationsHandler.channelRead(ChannelOperationsHandler.java:138)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#2:
Hint: 'reactor.right.reactiveBridge' will handle the message from this point.
io.netty.handler.codec.http.DefaultHttpContent.touch(DefaultHttpContent.java:88)
io.netty.handler.codec.http.DefaultLastHttpContent.touch(DefaultLastHttpContent.java:88)
io.netty.handler.codec.http.DefaultLastHttpContent.touch(DefaultLastHttpContent.java:28)
io.netty.channel.DefaultChannelPipeline.touch(DefaultChannelPipeline.java:116)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
Created at:
io.netty.buffer.SimpleLeakAwareByteBuf.unwrappedDerived(SimpleLeakAwareByteBuf.java:143)
io.netty.buffer.SimpleLeakAwareByteBuf.readRetainedSlice(SimpleLeakAwareByteBuf.java:67)
io.netty.buffer.AdvancedLeakAwareByteBuf.readRetainedSlice(AdvancedLeakAwareByteBuf.java:107)
io.netty.handler.codec.http.HttpObjectDecoder.decode(HttpObjectDecoder.java:305)
io.netty.handler.codec.http.HttpClientCodec$Decoder.decode(HttpClientCodec.java:202)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
2018-09-02 12:52:01.557 DEBUG 7772 --- [reactor-http-nio-3] r.ipc.netty.http.server.HttpServer : [id: 0x61a7dab8, L:/127.0.0.1:8090 - R:/127.0.0.1:29107] WRITE: 924B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
BUNCH OF STUFF FROM APP
+--------+-------------------------------------------------+----------------+
2018-09-02 12:52:01.801 ERROR 7772 --- [reactor-http-nio-1] io.netty.util.ResourceLeakDetector : LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records:
#1:
Hint: Buffered ByteBufHolder in Inbound Flux Queue
io.netty.handler.codec.http.DefaultHttpContent.touch(DefaultHttpContent.java:88)
io.netty.handler.codec.http.DefaultHttpContent.touch(DefaultHttpContent.java:24)
reactor.ipc.netty.channel.FluxReceive.onInboundNext(FluxReceive.java:324)
reactor.ipc.netty.channel.ChannelOperations.onInboundNext(ChannelOperations.java:311)
reactor.ipc.netty.http.client.HttpClientOperations.onInboundNext(HttpClientOperations.java:628)
reactor.ipc.netty.channel.ChannelOperationsHandler.channelRead(ChannelOperationsHandler.java:138)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#2:
Hint: 'reactor.right.reactiveBridge' will handle the message from this point.
io.netty.handler.codec.http.DefaultHttpContent.touch(DefaultHttpContent.java:88)
io.netty.handler.codec.http.DefaultHttpContent.touch(DefaultHttpContent.java:24)
io.netty.channel.DefaultChannelPipeline.touch(DefaultChannelPipeline.java:116)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#3:
io.netty.handler.codec.http.DefaultHttpContent.release(DefaultHttpContent.java:94)
io.netty.util.ReferenceCountUtil.release(ReferenceCountUtil.java:88)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:90)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#4:
io.netty.handler.codec.http.DefaultHttpContent.retain(DefaultHttpContent.java:70)
io.netty.handler.codec.http.HttpContentDecoder.decode(HttpContentDecoder.java:145)
io.netty.handler.codec.http.HttpContentDecoder.decode(HttpContentDecoder.java:46)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:88)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#5:
Hint: 'reactor.left.decompressor' will handle the message from this point.
io.netty.handler.codec.http.DefaultHttpContent.touch(DefaultHttpContent.java:88)
io.netty.handler.codec.http.DefaultHttpContent.touch(DefaultHttpContent.java:24)
io.netty.channel.DefaultChannelPipeline.touch(DefaultChannelPipeline.java:116)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:297)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:413)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
Created at:
io.netty.buffer.SimpleLeakAwareByteBuf.unwrappedDerived(SimpleLeakAwareByteBuf.java:143)
io.netty.buffer.SimpleLeakAwareByteBuf.readRetainedSlice(SimpleLeakAwareByteBuf.java:67)
io.netty.buffer.AdvancedLeakAwareByteBuf.readRetainedSlice(AdvancedLeakAwareByteBuf.java:107)
io.netty.handler.codec.http.HttpObjectDecoder.decode(HttpObjectDecoder.java:345)
io.netty.handler.codec.http.HttpClientCodec$Decoder.decode(HttpClientCodec.java:202)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
Sorry last one. I really hope these are useful. This is basically the end to end of a single click to a leak. I've removed things where I have to.
2018-09-02 14:06:39.597 DEBUG 32212 --- [ctor-http-nio-1] r.ipc.netty.http.server.HttpServer : [id: 0xd8555528, L:/0:0:0:0:0:0:0:0:8090] READ: [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784]
2018-09-02 14:06:39.597 DEBUG 32212 --- [ctor-http-nio-1] r.ipc.netty.http.server.HttpServer : [id: 0xd8555528, L:/0:0:0:0:0:0:0:0:8090] READ COMPLETE
2018-09-02 14:06:39.597 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.http.server.HttpServerOperations : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] New http connection, requesting read
2018-09-02 14:06:39.597 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.channel.ContextHandler : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] After pipeline DefaultChannelPipeline{(reactor.left.loggingHandler = io.netty.handler.logging.LoggingHandler), (ServerContextHandler#0 = reactor.ipc.netty.channel.ServerContextHandler), (reactor.left.httpCodec = io.netty.handler.codec.http.HttpServerCodec), (reactor.left.httpServerHandler = reactor.ipc.netty.http.server.HttpServerHandler), (reactor.right.reactiveBridge = reactor.ipc.netty.channel.ChannelOperationsHandler)}
2018-09-02 14:06:39.597 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] REGISTERED
2018-09-02 14:06:39.597 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] ACTIVE
2018-09-02 14:06:39.603 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] READ: 1024B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:39.610 DEBUG 32212 --- [ctor-http-nio-1] r.ipc.netty.http.server.HttpServer : [id: 0xd8555528, L:/0:0:0:0:0:0:0:0:8090] READ: [id: 0x16feef32, L:/127.0.0.1:8090 - R:/127.0.0.1:41787]
2018-09-02 14:06:39.634 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] READ COMPLETE
2018-09-02 14:06:39.634 DEBUG 32212 --- [ctor-http-nio-1] r.ipc.netty.http.server.HttpServer : [id: 0xd8555528, L:/0:0:0:0:0:0:0:0:8090] READ COMPLETE
2018-09-02 14:06:39.634 DEBUG 32212 --- [ctor-http-nio-3] r.i.n.http.server.HttpServerOperations : [id: 0x16feef32, L:/127.0.0.1:8090 - R:/127.0.0.1:41787] New http connection, requesting read
2018-09-02 14:06:39.634 DEBUG 32212 --- [ctor-http-nio-3] r.ipc.netty.channel.ContextHandler : [id: 0x16feef32, L:/127.0.0.1:8090 - R:/127.0.0.1:41787] After pipeline DefaultChannelPipeline{(reactor.left.loggingHandler = io.netty.handler.logging.LoggingHandler), (ServerContextHandler#0 = reactor.ipc.netty.channel.ServerContextHandler), (reactor.left.httpCodec = io.netty.handler.codec.http.HttpServerCodec), (reactor.left.httpServerHandler = reactor.ipc.netty.http.server.HttpServerHandler), (reactor.right.reactiveBridge = reactor.ipc.netty.channel.ChannelOperationsHandler)}
2018-09-02 14:06:39.634 DEBUG 32212 --- [ctor-http-nio-3] r.ipc.netty.http.server.HttpServer : [id: 0x16feef32, L:/127.0.0.1:8090 - R:/127.0.0.1:41787] REGISTERED
2018-09-02 14:06:39.634 DEBUG 32212 --- [ctor-http-nio-3] r.ipc.netty.http.server.HttpServer : [id: 0x16feef32, L:/127.0.0.1:8090 - R:/127.0.0.1:41787] ACTIVE
2018-09-02 14:06:39.634 DEBUG 32212 --- [ctor-http-nio-1] r.ipc.netty.http.server.HttpServer : [id: 0xd8555528, L:/0:0:0:0:0:0:0:0:8090] READ: [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788]
2018-09-02 14:06:39.634 DEBUG 32212 --- [ctor-http-nio-1] r.ipc.netty.http.server.HttpServer : [id: 0xd8555528, L:/0:0:0:0:0:0:0:0:8090] READ COMPLETE
2018-09-02 14:06:39.634 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.http.server.HttpServerOperations : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] New http connection, requesting read
2018-09-02 14:06:39.634 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.channel.ContextHandler : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] After pipeline DefaultChannelPipeline{(reactor.left.loggingHandler = io.netty.handler.logging.LoggingHandler), (ServerContextHandler#0 = reactor.ipc.netty.channel.ServerContextHandler), (reactor.left.httpCodec = io.netty.handler.codec.http.HttpServerCodec), (reactor.left.httpServerHandler = reactor.ipc.netty.http.server.HttpServerHandler), (reactor.right.reactiveBridge = reactor.ipc.netty.channel.ChannelOperationsHandler)}
2018-09-02 14:06:39.634 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] REGISTERED
2018-09-02 14:06:39.634 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] ACTIVE
2018-09-02 14:06:39.638 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] READ: 727B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:39.640 DEBUG 32212 --- [ctor-http-nio-3] r.ipc.netty.http.server.HttpServer : [id: 0x16feef32, L:/127.0.0.1:8090 - R:/127.0.0.1:41787] READ: 1024B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:39.642 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.http.server.HttpServerOperations : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] Increasing pending responses, now 1
2018-09-02 14:06:39.642 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] READ COMPLETE
2018-09-02 14:06:39.642 DEBUG 32212 --- [ctor-http-nio-3] r.ipc.netty.http.server.HttpServer : [id: 0x16feef32, L:/127.0.0.1:8090 - R:/127.0.0.1:41787] READ COMPLETE
2018-09-02 14:06:39.642 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.channel.ChannelOperations : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] [HttpServer] Handler is being applied: org.springframework.http.server.reactive.ReactorHttpHandlerAdapter@33b3763
2018-09-02 14:06:39.645 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.resources.DefaultPoolResources : [id: 0x5aa42ae4, L: - R:80] Channel acquired, now 1 active connections and 0 inactive connections
2018-09-02 14:06:39.645 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.c.PooledClientContextHandler : Acquiring existing channel from pool: DefaultPromise@6e1a98c1(success: [id: 0x5aa42ae4, L: - R:80]) SimpleChannelPool{activeConnections=1, inactiveConnections=0}
2018-09-02 14:06:39.645 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.c.PooledClientContextHandler : [id: 0x5aa42ae4, L: - R:80] Acquired active channel
2018-09-02 14:06:39.645 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.channel.ChannelOperations : [id: 0x5aa42ae4, L: - R:80] [HttpClient] Handler is being applied: HttpClientHandler{startURI=http://someendpoint, method=GET, handler=reactor.ipc.netty.http.client.HttpClient$$Lambda$548/634255648@26e38e80}
2018-09-02 14:06:39.645 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.channel.ChannelOperationsHandler : [id: 0x5aa42ae4, L: - R:80] Writing object DefaultHttpRequest(decodeResult: success, version: HTTP/1.1)
GET /someendpoint HTTP/1.1
user-agent: ReactorNetty/0.7.9.BUILD-SNAPSHOT
host: somehost
accept: */*
accept-encoding: gzip
Authorization:
2018-09-02 14:06:39.648 DEBUG 32212 --- [ctor-http-nio-3] r.ipc.netty.http.server.HttpServer : [id: 0x16feef32, L:/127.0.0.1:8090 - R:/127.0.0.1:41787] READ: 739B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:39.648 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] READ: 1024B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:39.653 DEBUG 32212 --- [ctor-http-nio-3] r.i.n.http.server.HttpServerOperations : [id: 0x16feef32, L:/127.0.0.1:8090 - R:/127.0.0.1:41787] Increasing pending responses, now 1
2018-09-02 14:06:39.653 DEBUG 32212 --- [ctor-http-nio-3] r.ipc.netty.http.server.HttpServer : [id: 0x16feef32, L:/127.0.0.1:8090 - R:/127.0.0.1:41787] READ COMPLETE
2018-09-02 14:06:39.653 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] READ COMPLETE
2018-09-02 14:06:39.653 DEBUG 32212 --- [ctor-http-nio-3] r.ipc.netty.channel.ChannelOperations : [id: 0x16feef32, L:/127.0.0.1:8090 - R:/127.0.0.1:41787] [HttpServer] Handler is being applied: org.springframework.http.server.reactive.ReactorHttpHandlerAdapter@33b3763
2018-09-02 14:06:39.658 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] READ: 712B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:39.658 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.http.server.HttpServerOperations : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] Increasing pending responses, now 1
2018-09-02 14:06:39.659 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] READ COMPLETE
2018-09-02 14:06:39.659 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.channel.ChannelOperations : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] [HttpServer] Handler is being applied: org.springframework.http.server.reactive.ReactorHttpHandlerAdapter@33b3763
2018-09-02 14:06:39.666 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x5aa42ae4, L: - R:80] WRITE: 1303B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:39.668 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x5aa42ae4, L: - R:80] FLUSH
2018-09-02 14:06:39.668 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.channel.ChannelOperationsHandler : [id: 0x5aa42ae4, L: - R:80] Writing object EmptyLastHttpContent
2018-09-02 14:06:39.668 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x5aa42ae4, L: - R:80] WRITE: 0B
2018-09-02 14:06:39.668 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x5aa42ae4, L: - R:80] FLUSH
2018-09-02 14:06:40.127 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x5aa42ae4, L: - R:80] READ: 481B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.128 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.http.client.HttpClientOperations : [id: 0x5aa42ae4, L: - R:80] Received response (auto-read:false) :
2018-09-02 14:06:40.128 DEBUG 32212 --- [ctor-http-nio-2] reactor.ipc.netty.channel.FluxReceive : [id: 0x5aa42ae4, L: - R:80] Subscribing inbound receiver [pending: 0, cancelled:false, inboundDone: false]
2018-09-02 14:06:40.128 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.http.client.HttpClientOperations : [id: 0x5aa42ae4, L: - R:80] Received last HTTP packet
2018-09-02 14:06:40.131 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.c.PooledClientContextHandler : Acquiring existing channel from pool: DefaultPromise@16595f89(incomplete) SimpleChannelPool{activeConnections=0, inactiveConnections=1}
2018-09-02 14:06:40.131 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.resources.DefaultPoolResources : [id: 0x364f29b5, L: - R:443] Channel acquired, now 1 active connections and 0 inactive connections
2018-09-02 14:06:40.131 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x5aa42ae4, L: - R:80] USER_EVENT: [Handler Terminated]
2018-09-02 14:06:40.131 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.channel.ChannelOperationsHandler : [id: 0x5aa42ae4, L: - R:80] Disposing context reactor.ipc.netty.channel.PooledClientContextHandler@4068bbb
2018-09-02 14:06:40.131 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.c.PooledClientContextHandler : [id: 0x364f29b5, L: - R::443] Acquired active channel
2018-09-02 14:06:40.131 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.c.PooledClientContextHandler : [id: 0x5aa42ae4, L: - R:80] Releasing channel
2018-09-02 14:06:40.131 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.resources.DefaultPoolResources : [id: 0x5aa42ae4, L: - R:80] Channel released, now 0 active connections and 1 inactive connections
2018-09-02 14:06:40.131 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x5aa42ae4, L: - R:80] READ COMPLETE
2018-09-02 14:06:40.131 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.channel.ChannelOperations : [id: 0x364f29b5, L: - R:443] [HttpClient] Handler is being applied: HttpClientHandler{startURI=https://someendpoint, method=GET, handler=reactor.ipc.netty.http.client.HttpClient$$Lambda$548/634255648@53eb959d}
2018-09-02 14:06:40.131 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0x364f29b5, L: - R:443] Writing object DefaultHttpRequest(decodeResult: success, version: HTTP/1.1)
GET /someendpoint
user-agent: ReactorNetty/0.7.9.BUILD-SNAPSHOT
host: somehost
accept: */*
accept-encoding: gzip
2018-09-02 14:06:40.137 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0x364f29b5, L: - R:443] WRITE: 509B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.138 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0x364f29b5, L:L: - R:443] FLUSH
2018-09-02 14:06:40.138 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0x364f29b5, L:L: - R:443] Writing object EmptyLastHttpContent
2018-09-02 14:06:40.138 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0x364f29b5, L:L: - R:443] WRITE: 0B
2018-09-02 14:06:40.138 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0x364f29b5, L:L: - R:443] FLUSH
2018-09-02 14:06:40.242 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0x364f29b5, L:L: - R:443] READ: 1492B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.243 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.http.client.HttpClientOperations : [id: 0x364f29b5, L: - R:443] Received response (auto-read:false) :
2018-09-02 14:06:40.244 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.resources.DefaultPoolResources : [id: 0xb1a140e8, L: - R:443] Channel acquired, now 1 active connections and 2 inactive connections
2018-09-02 14:06:40.244 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.c.PooledClientContextHandler : Acquiring existing channel from pool: DefaultPromise@19ec4edf(success: [id: 0xb1a140e8, L: - R:443]) SimpleChannelPool{activeConnections=1, inactiveConnections=2}
2018-09-02 14:06:40.244 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.c.PooledClientContextHandler : [id: 0xb1a140e8, L: - R:443] Acquired active channel
2018-09-02 14:06:40.245 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.resources.DefaultPoolResources : [id: 0x825b0cdc, L: - R:443] Channel acquired, now 2 active connections and 1 inactive connections
2018-09-02 14:06:40.245 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.c.PooledClientContextHandler : Acquiring existing channel from pool: DefaultPromise@408224d0(success: [id: 0x825b0cdc, L: - R:443]) SimpleChannelPool{activeConnections=2, inactiveConnections=1}
2018-09-02 14:06:40.245 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.c.PooledClientContextHandler : [id: 0x825b0cdc, L: - R:443] Acquired active channel
2018-09-02 14:06:40.246 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.c.PooledClientContextHandler : Acquiring existing channel from pool: DefaultPromise@3ef8010e(incomplete) SimpleChannelPool{activeConnections=2, inactiveConnections=1}
2018-09-02 14:06:40.246 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.resources.DefaultPoolResources : [id: 0x257bc7d0, L: - R:443] Channel acquired, now 3 active connections and 0 inactive connections
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.c.PooledClientContextHandler : [id: 0x257bc7d0, L: - R:443] Acquired active channel
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.channel.ChannelOperations : [id: 0x257bc7d0, L: - R:443] [HttpClient] Handler is being applied: HttpClientHandler{startURI=https://someendpoint, method=GET, handler=org.springframework.cloud.gateway.filter.NettyRoutingFilter$$Lambda$617/757733877@4adf7ec7}
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x257bc7d0, L: - R:443] USER_EVENT: reactor.ipc.netty.NettyPipeline$SendOptionsChangeEvent@7ab47f74
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.http.client.HttpClientOperations : [id: 0x364f29b5, L: - R:443] Received last HTTP packet
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.channel.ChannelOperationsHandler : [id: 0x257bc7d0, L: - R:443] New sending options
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0x364f29b5, L: - R:443] USER_EVENT: [Handler Terminated]
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0x364f29b5, L: - R:443] Disposing context reactor.ipc.netty.channel.PooledClientContextHandler@4c5d8478
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.c.PooledClientContextHandler : [id: 0x364f29b5, L: - R:443] Releasing channel
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.channel.ChannelOperationsHandler : [id: 0x257bc7d0, L: - R:443] Writing object DefaultHttpRequest(decodeResult: success, version: HTTP/1.1)
GET /someendpoint
Bunch of Stuff
host: somehost
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.resources.DefaultPoolResources : [id: 0x364f29b5, L: - R:443] Channel released, now 0 active connections and 1 inactive connections
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0x364f29b5, L: - R:443] READ COMPLETE
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.channel.ChannelOperations : [id: 0xb1a140e8, L: - R:443] [HttpClient] Handler is being applied: HttpClientHandler{startURI=https://someendpoint, method=GET, handler=org.springframework.cloud.gateway.filter.NettyRoutingFilter$$Lambda$617/757733877@42e4d1f3}
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0xb1a140e8, L: - R:443] USER_EVENT: reactor.ipc.netty.NettyPipeline$SendOptionsChangeEvent@b9445b8
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0xb1a140e8, L: - R:443] New sending options
2018-09-02 14:06:40.247 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0xb1a140e8, LL: - R:443] Writing object DefaultHttpRequest(decodeResult: success, version: HTTP/1.1)
GET /someendpoint
Bunch of stuff
host: somehost
2018-09-02 14:06:40.269 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x257bc7d0, L: - R:443] WRITE: 1956B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.271 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0xb1a140e8, L: - R:443] WRITE: 1971B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.271 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x257bc7d0, L: - R:443] FLUSH
2018-09-02 14:06:40.272 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0xb1a140e8, L: - R:443] FLUSH
2018-09-02 14:06:40.272 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0xb1a140e8, L: - R:443] Writing object
2018-09-02 14:06:40.272 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.channel.ChannelOperationsHandler : [id: 0x257bc7d0, L: - R:443] Writing object
2018-09-02 14:06:40.272 DEBUG 32212 --- [ctor-http-nio-2] reactor.ipc.netty.channel.FluxReceive : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] Subscribing inbound receiver [pending: 0, cancelled:false, inboundDone: true]
2018-09-02 14:06:40.272 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.channel.ChannelOperations : [id: 0x825b0cdc, L: - R:443] [HttpClient] Handler is being applied: HttpClientHandler{startURI=https://someendpoint, method=GET, handler=org.springframework.cloud.gateway.filter.NettyRoutingFilter$$Lambda$617/757733877@52e80594}
2018-09-02 14:06:40.273 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0x825b0cdc, L: - R:443] USER_EVENT: reactor.ipc.netty.NettyPipeline$SendOptionsChangeEvent@1dc9499f
2018-09-02 14:06:40.273 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0x825b0cdc, L: - R:443] New sending options
2018-09-02 14:06:40.273 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0x825b0cdc, L: - R:443] Writing object DefaultHttpRequest(decodeResult: success, version: HTTP/1.1)
GET /someendpoint
Bunch of stuff
host: somehost
2018-09-02 14:06:40.294 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0x825b0cdc, L: - R:443] WRITE: 1979B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.297 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0x825b0cdc, L: - R:443] FLUSH
2018-09-02 14:06:40.298 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0x825b0cdc, L: - R:443] Writing object
2018-09-02 14:06:40.298 DEBUG 32212 --- [ctor-http-nio-4] reactor.ipc.netty.channel.FluxReceive : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] Subscribing inbound receiver [pending: 0, cancelled:false, inboundDone: true]
2018-09-02 14:06:40.298 DEBUG 32212 --- [ctor-http-nio-3] reactor.ipc.netty.channel.FluxReceive : [id: 0x16feef32, L:/127.0.0.1:8090 - R:/127.0.0.1:41787] Subscribing inbound receiver [pending: 0, cancelled:false, inboundDone: true]
2018-09-02 14:06:40.298 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0xb1a140e8, L: - R:443] Writing object EmptyLastHttpContent
2018-09-02 14:06:40.298 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0xb1a140e8, L: - R:443] WRITE: 0B
2018-09-02 14:06:40.298 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.channel.ChannelOperationsHandler : [id: 0x257bc7d0, L: - R:443] Writing object EmptyLastHttpContent
2018-09-02 14:06:40.298 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0xb1a140e8, L: - R:443] FLUSH
2018-09-02 14:06:40.298 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x257bc7d0, L: - R:443] WRITE: 0B
2018-09-02 14:06:40.298 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x257bc7d0, L: - R:443] FLUSH
2018-09-02 14:06:40.298 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0x825b0cdc, L: - R:443] Writing object EmptyLastHttpContent
2018-09-02 14:06:40.298 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0x825b0cdc, L: - R:443] WRITE: 0B
2018-09-02 14:06:40.298 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0x825b0cdc, L: - R:443] FLUSH
2018-09-02 14:06:40.327 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x257bc7d0, L: - R:443] READ: 1940B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
|00000000| 48 54 54 50 2f 31 2e 31 20 32 30 30 20 4f 4b 0d |HTTP/1.1 200 OK.|
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.334 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.http.client.HttpClientOperations : [id: 0x257bc7d0, L: - R:443] Received response (auto-read:false) :
2018-09-02 14:06:40.334 DEBUG 32212 --- [ctor-http-nio-2] reactor.ipc.netty.channel.FluxReceive : [id: 0x257bc7d0, L: - R:443] Subscribing inbound receiver [pending: 0, cancelled:false, inboundDone: false]
2018-09-02 14:06:40.334 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.http.client.HttpClientOperations : [id: 0x257bc7d0, L: - R:443] Received last HTTP packet
2018-09-02 14:06:40.335 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x257bc7d0, L: - R:443] USER_EVENT: [Handler Terminated]
2018-09-02 14:06:40.335 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.channel.ChannelOperationsHandler : [id: 0x257bc7d0, L: - R:443] Disposing context reactor.ipc.netty.channel.PooledClientContextHandler@7397a302
2018-09-02 14:06:40.335 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.c.PooledClientContextHandler : [id: 0x257bc7d0, L: - R:443] Releasing channel
2018-09-02 14:06:40.335 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.resources.DefaultPoolResources : [id: 0x257bc7d0, L: - R:443] Channel released, now 2 active connections and 1 inactive connections
2018-09-02 14:06:40.335 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.client.HttpClient : [id: 0x257bc7d0, L: - R:443] READ COMPLETE
2018-09-02 14:06:40.341 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0xb1a140e8, L: - R:443] READ: 1658B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
|00000000| 48 54 54 50 2f 31 2e 31 20 32 30 30 20 4f 4b 0d |HTTP/1.1 200 OK.|
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.341 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.http.client.HttpClientOperations : [id: 0xb1a140e8, L: - R:443] Received response (auto-read:false) :
2018-09-02 14:06:40.341 DEBUG 32212 --- [ctor-http-nio-4] reactor.ipc.netty.channel.FluxReceive : [id: 0xb1a140e8, L: - R:443] Subscribing inbound receiver [pending: 0, cancelled:false, inboundDone: false]
2018-09-02 14:06:40.342 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.http.client.HttpClientOperations : [id: 0xb1a140e8, L: - R:443] Received last HTTP packet
2018-09-02 14:06:40.342 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.channel.ChannelOperationsHandler : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] Writing object DefaultHttpResponse(decodeResult: success, version: HTTP/1.1)
HTTP/1.1 200 OK
transfer-encoding: chunked
Bunch of stuff
2018-09-02 14:06:40.342 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0xb1a140e8, L: - R:443] USER_EVENT: [Handler Terminated]
2018-09-02 14:06:40.342 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0xb1a140e8, L: - R:443] Disposing context reactor.ipc.netty.channel.PooledClientContextHandler@249e5f99
2018-09-02 14:06:40.342 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.c.PooledClientContextHandler : [id: 0xb1a140e8, L: - R:443] Releasing channel
2018-09-02 14:06:40.342 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.resources.DefaultPoolResources : [id: 0xb1a140e8, L: - R:443] Channel released, now 1 active connections and 2 inactive connections
2018-09-02 14:06:40.342 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0xb1a140e8, L: - R:443] READ COMPLETE
2018-09-02 14:06:40.342 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] Writing object DefaultHttpResponse(decodeResult: success, version: HTTP/1.1)
HTTP/1.1 200 OK
transfer-encoding: chunked
Bunch of stuff
2018-09-02 14:06:40.355 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] WRITE: 1238B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.355 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] WRITE: 1238B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
|00000000| 48 54 54 50 2f 31 2e 31 20 32 30 30 20 4f 4b 0d |HTTP/1.1 200 OK.|
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.355 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] FLUSH
2018-09-02 14:06:40.356 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] FLUSH
2018-09-02 14:06:40.356 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.channel.ChannelOperationsHandler : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] Writing object
2018-09-02 14:06:40.356 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] Writing object
2018-09-02 14:06:40.356 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] WRITE: 5B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.356 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] WRITE: 5B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.362 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] WRITE: 454B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.366 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] WRITE: 736B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.366 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] WRITE: 2B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.367 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] FLUSH
2018-09-02 14:06:40.367 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] WRITE: 2B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.367 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] FLUSH
2018-09-02 14:06:40.367 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.http.server.HttpServerOperations : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] Last HTTP response frame
2018-09-02 14:06:40.367 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] Writing object EmptyLastHttpContent
2018-09-02 14:06:40.367 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.http.server.HttpServerOperations : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] Last HTTP response frame
2018-09-02 14:06:40.367 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.channel.ChannelOperationsHandler : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] Writing object EmptyLastHttpContent
2018-09-02 14:06:40.367 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] WRITE: 5B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.367 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.http.server.HttpServerOperations : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] Decreasing pending responses, now 0
2018-09-02 14:06:40.367 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] FLUSH
2018-09-02 14:06:40.367 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] WRITE: 5B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.367 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.http.server.HttpServerOperations : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] Decreasing pending responses, now 0
2018-09-02 14:06:40.367 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] FLUSH
2018-09-02 14:06:40.368 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.channel.ChannelOperations : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] Discarding inbound content
2018-09-02 14:06:40.368 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] USER_EVENT: [Handler Terminated]
2018-09-02 14:06:40.368 DEBUG 32212 --- [ctor-http-nio-2] r.i.n.channel.ChannelOperationsHandler : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] Disposing context reactor.ipc.netty.channel.ServerContextHandler@aa53b55
2018-09-02 14:06:40.368 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] READ COMPLETE
2018-09-02 14:06:40.368 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 - R:/127.0.0.1:41784] EXCEPTION: java.io.IOException: An established connection was aborted by the software in your host machine
java.io.IOException: An established connection was aborted by the software in your host machine
at sun.nio.ch.SocketDispatcher.read0(Native Method) ~[na:1.8.0_144]
at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:43) ~[na:1.8.0_144]
at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) ~[na:1.8.0_144]
at sun.nio.ch.IOUtil.read(IOUtil.java:192) ~[na:1.8.0_144]
at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) ~[na:1.8.0_144]
at io.netty.buffer.PooledUnsafeDirectByteBuf.setBytes(PooledUnsafeDirectByteBuf.java:288) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
at io.netty.buffer.AbstractByteBuf.writeBytes(AbstractByteBuf.java:1108) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
at io.netty.buffer.WrappedByteBuf.writeBytes(WrappedByteBuf.java:813) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
at io.netty.buffer.AdvancedLeakAwareByteBuf.writeBytes(AdvancedLeakAwareByteBuf.java:635) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
at io.netty.channel.socket.nio.NioSocketChannel.doReadBytes(NioSocketChannel.java:347) ~[netty-transport-4.1.29.Final.jar:4.1.29.Final]
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:148) ~[netty-transport-4.1.29.Final.jar:4.1.29.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628) [netty-transport-4.1.29.Final.jar:4.1.29.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563) [netty-transport-4.1.29.Final.jar:4.1.29.Final]
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480) [netty-transport-4.1.29.Final.jar:4.1.29.Final]
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442) [netty-transport-4.1.29.Final.jar:4.1.29.Final]
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884) [netty-common-4.1.29.Final.jar:4.1.29.Final]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_144]
2018-09-02 14:06:40.368 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 ! R:/127.0.0.1:41784] INACTIVE
2018-09-02 14:06:40.369 DEBUG 32212 --- [ctor-http-nio-2] r.ipc.netty.http.server.HttpServer : [id: 0xe389b4e7, L:/127.0.0.1:8090 ! R:/127.0.0.1:41784] UNREGISTERED
2018-09-02 14:06:40.367 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.channel.ChannelOperations : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] Discarding inbound content
2018-09-02 14:06:40.369 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.server.HttpServer : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] USER_EVENT: [Handler Terminated]
2018-09-02 14:06:40.369 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.channel.ChannelOperationsHandler : [id: 0x22da6a18, L:/127.0.0.1:8090 - R:/127.0.0.1:41788] Disposing context reactor.ipc.netty.channel.ServerContextHandler@aa53b55
2018-09-02 14:06:40.505 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0x825b0cdc, L: - R:443] READ: 10136B
+-------------------------------------------------+
| 0 1 2 3 4 5 6 7 8 9 a b c d e f |
+--------+-------------------------------------------------+----------------+
|00000000| 48 54 54 50 2f 31 2e 31 20 32 30 30 20 4f 4b 0d |HTTP/1.1 200 OK.|
+--------+-------------------------------------------------+----------------+
2018-09-02 14:06:40.510 DEBUG 32212 --- [ctor-http-nio-4] r.i.n.http.client.HttpClientOperations : [id: 0x825b0cdc, L: - R:443] Received response (auto-read:false) :
2018-09-02 14:06:40.511 DEBUG 32212 --- [ctor-http-nio-4] reactor.ipc.netty.channel.FluxReceive : [id: 0x825b0cdc, L: - R:443] Subscribing inbound receiver [pending: 0, cancelled:false, inboundDone: false]
2018-09-02 14:06:40.514 ERROR 32212 --- [ctor-http-nio-4] io.netty.util.ResourceLeakDetector : LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records:
#1:
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:273)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#2:
io.netty.buffer.AdvancedLeakAwareByteBuf.getUnsignedByte(AdvancedLeakAwareByteBuf.java:160)
io.netty.buffer.ByteBufUtil$HexUtil.appendPrettyHexDump(ByteBufUtil.java:1092)
io.netty.buffer.ByteBufUtil$HexUtil.access$300(ByteBufUtil.java:946)
io.netty.buffer.ByteBufUtil.appendPrettyHexDump(ByteBufUtil.java:942)
io.netty.buffer.ByteBufUtil.appendPrettyHexDump(ByteBufUtil.java:933)
io.netty.handler.logging.LoggingHandler.formatByteBuf(LoggingHandler.java:335)
io.netty.handler.logging.LoggingHandler.format(LoggingHandler.java:290)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:239)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#3:
io.netty.buffer.AdvancedLeakAwareByteBuf.getUnsignedByte(AdvancedLeakAwareByteBuf.java:160)
io.netty.buffer.ByteBufUtil$HexUtil.appendPrettyHexDump(ByteBufUtil.java:1098)
io.netty.buffer.ByteBufUtil$HexUtil.access$300(ByteBufUtil.java:946)
io.netty.buffer.ByteBufUtil.appendPrettyHexDump(ByteBufUtil.java:942)
io.netty.buffer.ByteBufUtil.appendPrettyHexDump(ByteBufUtil.java:933)
io.netty.handler.logging.LoggingHandler.formatByteBuf(LoggingHandler.java:335)
io.netty.handler.logging.LoggingHandler.format(LoggingHandler.java:290)
io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:239)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#4:
Hint: 'reactor.left.loggingHandler' will handle the message from this point.
io.netty.channel.DefaultChannelPipeline.touch(DefaultChannelPipeline.java:116)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#5:
io.netty.buffer.AdvancedLeakAwareByteBuf.internalNioBuffer(AdvancedLeakAwareByteBuf.java:736)
io.netty.handler.ssl.SslHandler.toByteBuffer(SslHandler.java:1439)
io.netty.handler.ssl.SslHandler.access$300(SslHandler.java:165)
io.netty.handler.ssl.SslHandler$SslEngineType$1.unwrap(SslHandler.java:217)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1297)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#6:
io.netty.buffer.AdvancedLeakAwareByteBuf.nioBufferCount(AdvancedLeakAwareByteBuf.java:706)
io.netty.handler.ssl.SslHandler.toByteBuffer(SslHandler.java:1439)
io.netty.handler.ssl.SslHandler.access$300(SslHandler.java:165)
io.netty.handler.ssl.SslHandler$SslEngineType$1.unwrap(SslHandler.java:217)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1297)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
Created at:
io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:331)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:185)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:176)
io.netty.handler.ssl.SslHandler.allocate(SslHandler.java:1909)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1292)
io.netty.handler.ssl.SslHandler.decodeNonJdkCompatible(SslHandler.java:1211)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1245)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
: 2997 leak records were discarded because they were duplicates
: 2997 leak records were discarded because the leak record count is targeted to 4. Use system property io.netty.leakDetection.targetRecords to increase the limit.
2018-09-02 14:06:40.521 DEBUG 32212 --- [ctor-http-nio-4] r.ipc.netty.http.client.HttpClient : [id: 0x825b0cdc, L: - R:443] READ COMPLETE
I just ran FluxReceiveTest.testByteBufsReleasedWhenTimeout and it shows as passed but the logs show memory being leaked. Not sure if that is to be expected.
dave-fl
on 2 Sep 2018
@dave-fl yep I think we have an issue. I'm on it.
violetagg
on 3 Sep 2018
👍5
Indirectly related, but it might be useful to add some sort of deterministic leak detection to the test cases as an enhancement, this way assertions can be used to check for buffer size rather than counting on the garbage collector to have run. Perhaps already "well known" or maybe even better solutions exist. But definitely makes visibility into the buffer count easier at least for me.
dave-fl
on 4 Sep 2018
@dave-fl Do you specify response timeout when using Spring Gateway?
violetagg
on 11 Sep 2018
@violetagg Yes, we set all our timeouts
dave-fl
on 11 Sep 2018
@violetagg I am not sure that timeouts are the full issue here, e.g. if any sort of IO Exception happens mid-flight the same end result could be the case. We are seeing this in some of our calls. I think it would be worth creating a test case that attempts to simulate an abrupt termination of a connection although whatever has to be done in order to fix the timeout issue might end up fixing this as well.
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT 2018-09-10 22:22:56.025 TRACE 15 --- [server-epoll-13] filter.ErrorFilter : reactor.ipc.netty.channel.AbortedException: Connection has been closed
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT reactor.ipc.netty.channel.AbortedException: Connection has been closed
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at reactor.ipc.netty.channel.ChannelOperationsHandler.discard(ChannelOperationsHandler.java:365) ~[reactor-netty-0.7.8.RELEASE.jar!/:0.7.8.RELEASE]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at reactor.ipc.netty.channel.ChannelOperationsHandler.drain(ChannelOperationsHandler.java:375) ~[reactor-netty-0.7.8.RELEASE.jar!/:0.7.8.RELEASE]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at reactor.ipc.netty.channel.ChannelOperationsHandler.handlerRemoved(ChannelOperationsHandler.java:208) ~[reactor-netty-0.7.8.RELEASE.jar!/:0.7.8.RELEASE]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at io.netty.channel.DefaultChannelPipeline.callHandlerRemoved0(DefaultChannelPipeline.java:670) ~[netty-transport-4.1.27.Final.jar!/:4.1.27.Final]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at io.netty.channel.DefaultChannelPipeline.destroyDown(DefaultChannelPipeline.java:922) ~[netty-transport-4.1.27.Final.jar!/:4.1.27.Final]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at io.netty.channel.DefaultChannelPipeline.destroyUp(DefaultChannelPipeline.java:888) ~[netty-transport-4.1.27.Final.jar!/:4.1.27.Final]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at io.netty.channel.DefaultChannelPipeline.destroy(DefaultChannelPipeline.java:880) ~[netty-transport-4.1.27.Final.jar!/:4.1.27.Final]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at io.netty.channel.DefaultChannelPipeline.access$700(DefaultChannelPipeline.java:46) ~[netty-transport-4.1.27.Final.jar!/:4.1.27.Final]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at io.netty.channel.DefaultChannelPipeline$HeadContext.channelUnregistered(DefaultChannelPipeline.java:1416) ~[netty-transport-4.1.27.Final.jar!/:4.1.27.Final]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:181) ~[netty-transport-4.1.27.Final.jar!/:4.1.27.Final]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:167) ~[netty-transport-4.1.27.Final.jar!/:4.1.27.Final]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at io.netty.channel.DefaultChannelPipeline.fireChannelUnregistered(DefaultChannelPipeline.java:865) ~[netty-transport-4.1.27.Final.jar!/:4.1.27.Final]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:830) ~[netty-transport-4.1.27.Final.jar!/:4.1.27.Final]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163) ~[netty-common-4.1.27.Final.jar!/:4.1.27.Final]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:404) ~[netty-common-4.1.27.Final.jar!/:4.1.27.Final]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:322) ~[netty-transport-native-epoll-4.1.27.Final-linux-x86_64.jar!/:4.1.27.Final]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884) ~[netty-common-4.1.27.Final.jar!/:4.1.27.Final]
2018-09-10T18:22:56.02-0400 [APP/PROC/WEB/1] OUT at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_172]
@dave-fl Do you think you can try that
https://github.com/spring-cloud/spring-cloud-gateway/pull/540
violetagg
on 11 Sep 2018
@violetagg I can try that. I see that you have moved to Netty native timeouts.
Can you please comment on the possibility of a leak with the connections being aborted. I am mentioning this because I still saw leaks when in paranoid mode even when no timeouts were taking place.
dave-fl
on 11 Sep 2018
Since WebFlux was mentioned earlier in this thread, keep in mind the following two fixes SPR-17025 and SPR-17246. There is also SPR-17193 if Jackson is in use.
rstoyanchev
on 11 Sep 2018
@dave-fl With Reactor Netty 0.7.x implementation the timeouts (provided by Flux/Mono) cannot be handled correctly and because of that Spring Gateway implementation will use the Netty's ReadTimeoutHandler.
With Reactor Netty 0.8.x implementation we can handle this use case properly.
Can you please comment on the possibility of a leak with the connections being aborted. I am mentioning this because I still saw leaks when in paranoid mode even when no timeouts were taking place.
If you are able to provide some reproducible scenario? Otherwise I'll try to simulate something on my side and do a code review for possible memory leaks in such situations.
violetagg
on 11 Sep 2018
@violetagg I wish I could, unfortunately I don't have one as this is happening on an end to end test.
I believe there are some proxies out there that can simulate various network conditions. It might be worth running a gateway style test where the client and connection from gateway to the back end server route through one of these proxies that simulate the occasional connection/network error.
I think I saw that 0.8.x is now RC1, are these cases already handled or on the TODO list before GA?
@rstoyanchev Thanks for mentioning those. We've cherry already cherry picked SPR-17025.
dave-fl
on 11 Sep 2018
I think I saw that 0.8.x is now RC1, are these cases already handled or on the TODO list before GA?
they are handled
violetagg
on 12 Sep 2018
+1
Having the same issue popping at random on production deployment. We have Spring Boot 2 service, using WebFlux and reactive WebClient. Can we expect a fix any time soon, guys?
antoniy
on 22 Sep 2018
@antoniy Till now we identified issue when Flux/Mono#timeout is used and recommended Netty's ReadTimeoutHandler/WriteTimeoutHandler usage instead of Flux/Mono#timeout. Is your scenario the same or something else?
violetagg
on 22 Sep 2018
I think majority of issues are something else. My testing shows that 0.8x still leaks. Not like the Titanic but the messages are in the logs.
Also, the fix you made for Spring Cloud Gateway does not seem to work with the 0.8x Branch.
I think they could be related to client disconnects.
dave-fl
on 22 Sep 2018
@dave-fl I'm asking because sometimes it might be an application leak then Reactor Netty. So stating "I see memory leaks reported in the logs" doesn't mean it is the same issue.
@antoniy If you are able to provide a reproducible scenario it will be really helpful.
violetagg
on 22 Sep 2018
@violetagg I'm not aware what's the reason for the leak in our app. We are using pretty standard stuff. Calling third-party services with WebClient with GET/POST requests and serving simple POST to our system via WebFlux. I have no reproducible scenario. It's popping at random. Here is one such log:
LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records:
Created at:
io.netty.buffer.PooledByteBufAllocator.newHeapBuffer(PooledByteBufAllocator.java:314)
io.netty.buffer.AbstractByteBufAllocator.heapBuffer(AbstractByteBufAllocator.java:166)
io.netty.buffer.AbstractByteBufAllocator.heapBuffer(AbstractByteBufAllocator.java:157)
io.netty.handler.codec.compression.JdkZlibDecoder.decode(JdkZlibDecoder.java:180)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.embedded.EmbeddedChannel.writeInbound(EmbeddedChannel.java:325)
io.netty.handler.codec.http.HttpContentDecoder.decode(HttpContentDecoder.java:234)
io.netty.handler.codec.http.HttpContentDecoder.decodeContent(HttpContentDecoder.java:155)
io.netty.handler.codec.http.HttpContentDecoder.decode(HttpContentDecoder.java:147)
io.netty.handler.codec.http.HttpContentDecoder.decode(HttpContentDecoder.java:46)
io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:88)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:417)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:317)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:745)
We have docker OutOfMemory errors though. I think it might be related to netty leaking some native memory and that issue might be the way to fix those. JVM process has no OOMs at this point but docker container in AWS is crashing due to OOMs. Sharing this additional information in case it helps a bit.
antoniy
on 22 Sep 2018
Do you reproduce on WebFlux 5.1.0.RELEASE ?
smaldini
on 27 Sep 2018
When I tested with Greenwhich snapshot a couple of weeks ago I had both leaks and this error (Native timeouts were broken).
https://github.com/spring-cloud/spring-cloud-gateway/issues/549
I can try with their latest snapshot again.
dave-fl
on 27 Sep 2018
Spring Framework 5.1 was released last week
smaldini
on 27 Sep 2018
Hi,
Were you able to test the latest Spring Framework release?
Can you provide a feedback?
Thanks,
Violeta
violetagg
on 8 Oct 2018
I am running Spring 5.1.0.RELEASE and reactor-netty 8.1.0.RELEASE, and I am able to reproduce similar leak reports after just a few requests (in paranoid mode) after introducing a short random timeout via Flux/Mono#timeout. I had understood from another thread that this was the recommended timeout strategy as of version 8.* - but it's not clear to me from the discussion here whether the buffer leaks resulting from this timeout strategy are thought to have been previously resolved or not. Here are a couple of examples of the leak reports I've seen for reference:
ERROR: LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records:
#1:
io.netty.handler.codec.http.DefaultHttpContent.release(DefaultHttpContent.java:94)
io.netty.util.ReferenceCountUtil.release(ReferenceCountUtil.java:88)
reactor.netty.channel.FluxReceive.drainReceiver(FluxReceive.java:214)
reactor.netty.channel.FluxReceive.onInboundNext(FluxReceive.java:327)
reactor.netty.channel.ChannelOperations.onInboundNext(ChannelOperations.java:310)
reactor.netty.http.client.HttpClientOperations.onInboundNext(HttpClientOperations.java:514)
reactor.netty.channel.ChannelOperationsHandler.channelRead(ChannelOperationsHandler.java:141)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#2:
org.springframework.http.client.reactive.ReactorClientHttpResponse.lambda$getBody$0(ReactorClientHttpResponse.java:62)
reactor.core.publisher.FluxMap$MapSubscriber.onNext(FluxMap.java:100)
org.springframework.cloud.sleuth.instrument.reactor.ScopePassingSpanSubscriber.onNext(ScopePassingSpanSubscriber.java:81)
reactor.core.publisher.FluxMap$MapSubscriber.onNext(FluxMap.java:114)
reactor.netty.channel.FluxReceive.drainReceiver(FluxReceive.java:211)
reactor.netty.channel.FluxReceive.onInboundNext(FluxReceive.java:327)
reactor.netty.channel.ChannelOperations.onInboundNext(ChannelOperations.java:310)
reactor.netty.http.client.HttpClientOperations.onInboundNext(HttpClientOperations.java:514)
reactor.netty.channel.ChannelOperationsHandler.channelRead(ChannelOperationsHandler.java:141)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#3:
Hint: Buffered ByteBufHolder in Inbound Flux Queue
io.netty.handler.codec.http.DefaultHttpContent.touch(DefaultHttpContent.java:88)
io.netty.handler.codec.http.DefaultLastHttpContent.touch(DefaultLastHttpContent.java:88)
io.netty.handler.codec.http.DefaultLastHttpContent.touch(DefaultLastHttpContent.java:28)
reactor.netty.channel.FluxReceive.onInboundNext(FluxReceive.java:322)
reactor.netty.channel.ChannelOperations.onInboundNext(ChannelOperations.java:310)
reactor.netty.http.client.HttpClientOperations.onInboundNext(HttpClientOperations.java:514)
reactor.netty.channel.ChannelOperationsHandler.channelRead(ChannelOperationsHandler.java:141)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#4:
Hint: 'reactor.right.reactiveBridge' will handle the message from this point.
io.netty.handler.codec.http.DefaultHttpContent.touch(DefaultHttpContent.java:88)
io.netty.handler.codec.http.DefaultLastHttpContent.touch(DefaultLastHttpContent.java:88)
io.netty.handler.codec.http.DefaultLastHttpContent.touch(DefaultLastHttpContent.java:28)
io.netty.channel.DefaultChannelPipeline.touch(DefaultChannelPipeline.java:116)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438)
io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
Created at:
io.netty.buffer.SimpleLeakAwareByteBuf.unwrappedDerived(SimpleLeakAwareByteBuf.java:143)
io.netty.buffer.SimpleLeakAwareByteBuf.readRetainedSlice(SimpleLeakAwareByteBuf.java:67)
io.netty.buffer.AdvancedLeakAwareByteBuf.readRetainedSlice(AdvancedLeakAwareByteBuf.java:107)
io.netty.handler.codec.http.HttpObjectDecoder.decode(HttpObjectDecoder.java:305)
io.netty.handler.codec.http.HttpClientCodec$Decoder.decode(HttpClientCodec.java:202)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
ERROR: LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records:
#1:
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:273)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#2:
io.netty.buffer.AdvancedLeakAwareByteBuf.readRetainedSlice(AdvancedLeakAwareByteBuf.java:106)
io.netty.handler.codec.http.HttpObjectDecoder.decode(HttpObjectDecoder.java:305)
io.netty.handler.codec.http.HttpClientCodec$Decoder.decode(HttpClientCodec.java:202)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#3:
io.netty.buffer.AdvancedLeakAwareByteBuf.forEachByte(AdvancedLeakAwareByteBuf.java:670)
io.netty.handler.codec.http.HttpObjectDecoder$HeaderParser.parse(HttpObjectDecoder.java:801)
io.netty.handler.codec.http.HttpObjectDecoder.readHeaders(HttpObjectDecoder.java:601)
io.netty.handler.codec.http.HttpObjectDecoder.decode(HttpObjectDecoder.java:227)
io.netty.handler.codec.http.HttpClientCodec$Decoder.decode(HttpClientCodec.java:202)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#4:
io.netty.buffer.AdvancedLeakAwareByteBuf.forEachByte(AdvancedLeakAwareByteBuf.java:670)
io.netty.handler.codec.http.HttpObjectDecoder$HeaderParser.parse(HttpObjectDecoder.java:801)
io.netty.handler.codec.http.HttpObjectDecoder.readHeaders(HttpObjectDecoder.java:581)
io.netty.handler.codec.http.HttpObjectDecoder.decode(HttpObjectDecoder.java:227)
io.netty.handler.codec.http.HttpClientCodec$Decoder.decode(HttpClientCodec.java:202)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#5:
io.netty.buffer.AdvancedLeakAwareByteBuf.getUnsignedByte(AdvancedLeakAwareByteBuf.java:160)
io.netty.handler.codec.http.HttpObjectDecoder.skipControlCharacters(HttpObjectDecoder.java:566)
io.netty.handler.codec.http.HttpObjectDecoder.decode(HttpObjectDecoder.java:202)
io.netty.handler.codec.http.HttpClientCodec$Decoder.decode(HttpClientCodec.java:202)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#6:
Hint: 'reactor.left.httpCodec' will handle the message from this point.
io.netty.channel.DefaultChannelPipeline.touch(DefaultChannelPipeline.java:116)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#7:
Hint: 'DefaultChannelPipeline$HeadContext#0' will handle the message from this point.
io.netty.channel.DefaultChannelPipeline.touch(DefaultChannelPipeline.java:116)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
Created at:
io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:331)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:185)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:176)
io.netty.buffer.AbstractByteBufAllocator.ioBuffer(AbstractByteBufAllocator.java:137)
io.netty.channel.DefaultMaxMessagesRecvByteBufAllocator$MaxMessageHandle.allocate(DefaultMaxMessagesRecvByteBufAllocator.java:114)
io.netty.channel.epoll.EpollRecvByteAllocatorHandle.allocate(EpollRecvByteAllocatorHandle.java:72)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:793)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
: 4 leak records were discarded because the leak record count is targeted to 4. Use system property io.netty.leakDetection.targetRecords to increase the limit.
danielra
on 16 Oct 2018
@danielra Can you provide some reproducible example/scenario? (This issue is for pure Reactor Netty Http Client while in your scenario most probably the WebClient from Spring Framework is used)
This one is suspicious:
#2:
org.springframework.http.client.reactive.ReactorClientHttpResponse.lambda$getBody$0(ReactorClientHttpResponse.java:62)
reactor.core.publisher.FluxMap$MapSubscriber.onNext(FluxMap.java:100)
org.springframework.cloud.sleuth.instrument.reactor.ScopePassingSpanSubscriber.onNext(ScopePassingSpanSubscriber.java:81)
reactor.core.publisher.FluxMap$MapSubscriber.onNext(FluxMap.java:114)
Indeed in this case I was using the WebClient with the ReactorClientHttpConnector. I will try to pare down to a more isolated repro case and report back.
danielra
on 17 Oct 2018
I was able to reproduce emitting similar leak reports using the reactor-netty HttpClient directly in a more isolated test case. A demo project with the repro test case can be found here: https://github.com/danielra/buffer-leak-repro-reactor-netty
Running this via ./gradlew clean test --debug I fairly reliably see leak reports like this:
10:44:54.298 [DEBUG] [TestEventLogger] 2018-10-17 10:44:54.255 ERROR 14296 --- [ Test worker] io.netty.util.ResourceLeakDetector : LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
10:44:54.298 [DEBUG] [TestEventLogger] Recent access records:
10:44:54.298 [DEBUG] [TestEventLogger] #1:
10:44:54.298 [DEBUG] [TestEventLogger] io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:273)
10:44:54.298 [DEBUG] [TestEventLogger] io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
10:44:54.298 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
10:44:54.298 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
10:44:54.298 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
10:44:54.298 [DEBUG] [TestEventLogger] io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
10:44:54.298 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
10:44:54.298 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
10:44:54.298 [DEBUG] [TestEventLogger] io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
10:44:54.299 [DEBUG] [TestEventLogger] java.lang.Thread.run(Thread.java:748)
10:44:54.299 [DEBUG] [TestEventLogger] #2:
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.buffer.AdvancedLeakAwareByteBuf.getByte(AdvancedLeakAwareByteBuf.java:154)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.handler.codec.http.HttpObjectDecoder.decode(HttpObjectDecoder.java:360)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.handler.codec.http.HttpClientCodec$Decoder.decode(HttpClientCodec.java:202)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
10:44:54.299 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
10:44:54.300 [DEBUG] [TestEventLogger] java.lang.Thread.run(Thread.java:748)
10:44:54.300 [DEBUG] [TestEventLogger] #3:
10:44:54.300 [DEBUG] [TestEventLogger] Hint: 'reactor.left.httpCodec' will handle the message from this point.
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.channel.DefaultChannelPipeline.touch(DefaultChannelPipeline.java:116)
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
10:44:54.300 [DEBUG] [TestEventLogger] io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
10:44:54.301 [DEBUG] [TestEventLogger] io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
10:44:54.301 [DEBUG] [TestEventLogger] java.lang.Thread.run(Thread.java:748)
10:44:54.301 [DEBUG] [TestEventLogger] #4:
10:44:54.301 [DEBUG] [TestEventLogger] Hint: 'DefaultChannelPipeline$HeadContext#0' will handle the message from this point.
10:44:54.301 [DEBUG] [TestEventLogger] io.netty.channel.DefaultChannelPipeline.touch(DefaultChannelPipeline.java:116)
10:44:54.301 [DEBUG] [TestEventLogger] io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
10:44:54.301 [DEBUG] [TestEventLogger] io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
10:44:54.301 [DEBUG] [TestEventLogger] io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:808)
10:44:54.301 [DEBUG] [TestEventLogger] io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
10:44:54.301 [DEBUG] [TestEventLogger] io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
10:44:54.301 [DEBUG] [TestEventLogger] io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
10:44:54.301 [DEBUG] [TestEventLogger] java.lang.Thread.run(Thread.java:748)
10:44:54.301 [DEBUG] [TestEventLogger] Created at:
10:44:54.301 [DEBUG] [TestEventLogger] io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:331)
10:44:54.301 [DEBUG] [TestEventLogger] io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:185)
10:44:54.301 [DEBUG] [TestEventLogger] io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:176)
10:44:54.301 [DEBUG] [TestEventLogger] io.netty.buffer.AbstractByteBufAllocator.ioBuffer(AbstractByteBufAllocator.java:137)
10:44:54.301 [DEBUG] [TestEventLogger] io.netty.channel.DefaultMaxMessagesRecvByteBufAllocator$MaxMessageHandle.allocate(DefaultMaxMessagesRecvByteBufAllocator.java:114)
10:44:54.302 [DEBUG] [TestEventLogger] io.netty.channel.epoll.EpollRecvByteAllocatorHandle.allocate(EpollRecvByteAllocatorHandle.java:72)
10:44:54.302 [DEBUG] [TestEventLogger] io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:793)
10:44:54.302 [DEBUG] [TestEventLogger] io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:410)
10:44:54.302 [DEBUG] [TestEventLogger] io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:310)
10:44:54.302 [DEBUG] [TestEventLogger] io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
10:44:54.302 [DEBUG] [TestEventLogger] java.lang.Thread.run(Thread.java:748)
Notably I also see some errors like this, which I hadn't seen before:
10:44:54.344 [DEBUG] [TestEventLogger] io.netty.util.IllegalReferenceCountException: refCnt: 0, decrement: 1
10:44:54.344 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractReferenceCountedByteBuf.release0(AbstractReferenceCountedByteBuf.java:100) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
10:44:54.344 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractReferenceCountedByteBuf.release(AbstractReferenceCountedByteBuf.java:84) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
10:44:54.344 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractPooledDerivedByteBuf$PooledNonRetainedSlicedByteBuf.release0(AbstractPooledDerivedByteBuf.java:271) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
10:44:54.344 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractDerivedByteBuf.release(AbstractDerivedByteBuf.java:85) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
10:44:54.344 [DEBUG] [TestEventLogger] at io.netty.buffer.WrappedByteBuf.release(WrappedByteBuf.java:1029) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
10:44:54.344 [DEBUG] [TestEventLogger] at io.netty.buffer.SimpleLeakAwareByteBuf.release(SimpleLeakAwareByteBuf.java:102) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
10:44:54.344 [DEBUG] [TestEventLogger] at io.netty.buffer.AdvancedLeakAwareByteBuf.release(AdvancedLeakAwareByteBuf.java:941) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
10:44:54.344 [DEBUG] [TestEventLogger] at io.netty.buffer.CompositeByteBuf$Component.freeIfNecessary(CompositeByteBuf.java:1683) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
10:44:54.344 [DEBUG] [TestEventLogger] at io.netty.buffer.CompositeByteBuf.deallocate(CompositeByteBuf.java:1948) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
10:44:54.345 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractReferenceCountedByteBuf.release0(AbstractReferenceCountedByteBuf.java:95) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
10:44:54.345 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractReferenceCountedByteBuf.release(AbstractReferenceCountedByteBuf.java:84) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
10:44:54.345 [DEBUG] [TestEventLogger] at io.netty.buffer.WrappedCompositeByteBuf.release(WrappedCompositeByteBuf.java:43) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
10:44:54.345 [DEBUG] [TestEventLogger] at io.netty.buffer.SimpleLeakAwareCompositeByteBuf.release(SimpleLeakAwareCompositeByteBuf.java:38) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
10:44:54.345 [DEBUG] [TestEventLogger] at io.netty.buffer.AdvancedLeakAwareCompositeByteBuf.release(AdvancedLeakAwareCompositeByteBuf.java:1023) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxUsing$UsingFuseableSubscriber.cleanup(FluxUsing.java:331) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxUsing$UsingFuseableSubscriber.cancel(FluxUsing.java:325) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.cancel(FluxMapFuseable.java:167) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxMap$MapSubscriber.cancel(FluxMap.java:160) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.publisher.Operators$MultiSubscriptionSubscriber.drainLoop(Operators.java:1905) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.publisher.Operators$MultiSubscriptionSubscriber.drain(Operators.java:1874) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.publisher.Operators$MultiSubscriptionSubscriber.cancel(Operators.java:1686) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxTimeout$TimeoutMainSubscriber.handleTimeout(FluxTimeout.java:287) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxTimeout$TimeoutMainSubscriber.doTimeout(FluxTimeout.java:273) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxTimeout$TimeoutTimeoutSubscriber.onNext(FluxTimeout.java:390) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.publisher.StrictSubscriber.onNext(StrictSubscriber.java:89) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:73) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.publisher.MonoDelay$MonoDelayRunnable.run(MonoDelay.java:117) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.scheduler.SchedulerTask.call(SchedulerTask.java:50) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at reactor.core.scheduler.SchedulerTask.call(SchedulerTask.java:27) [reactor-core-3.2.1.RELEASE.jar:3.2.1.RELEASE]
10:44:54.345 [DEBUG] [TestEventLogger] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_171]
10:44:54.345 [DEBUG] [TestEventLogger] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_171]
10:44:54.345 [DEBUG] [TestEventLogger] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.8.0_171]
10:44:54.345 [DEBUG] [TestEventLogger] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_171]
10:44:54.345 [DEBUG] [TestEventLogger] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_171]
10:44:54.345 [DEBUG] [TestEventLogger] at java.lang.Thread.run(Thread.java:748) [na:1.8.0_171]
I do not see either of these if I remove the line in the test specifying the timeout: https://github.com/danielra/buffer-leak-repro-reactor-netty/blob/master/src/test/java/com/example/demo/DemoApplicationTests.java#L43
danielra
on 17 Oct 2018
@danielra and this is 0.7.x or 0.8.x?
violetagg
on 17 Oct 2018
The demo project is using reactor-netty 0.8.1.RELEASE according to the ./gradlew dependencies output.
danielra
on 17 Oct 2018
Thanks for the sample @danielra. That helps a lot.
it's not clear to me from the discussion here whether the buffer leaks resulting from this timeout strategy are thought to have been previously resolved or not.
There were some recent changes in the Spring Framework in 5.1 and 5.0.x, but we need some more. I've created SPR-17408 that you can read and follow.
rstoyanchev
on 18 Oct 2018
👍1
@danielra I would say a strong NO to them being specific to timeouts. I have seen them through regular use of Spring Cloud Gateway.
Unfortunately I have not been able to fully test again with 5.1 but I did test a beta and still saw issues. Will test again as soon as I can.
dave-fl
on 18 Oct 2018
Indeed timeout is just one example. More generally anything to do with error or cancellation.
rstoyanchev
on 18 Oct 2018
@rstoyanchev Where do things currently stand as of now and what are the plans to move this forward?
I realize that tracking these bugs down are extremely difficult but I think the majority of users encountering this are simply using WebFlux, WebClient or Spring Cloud Gateway and not doing anything particularly specific.
Every time someone strolls through here and says memory leak - we rarely have a way to reproduce as its all happening randomly. With that said, what can be done to get this resolved. I prefer not to have to plan for daily restarts to clear up memory.
dave-fl
on 18 Oct 2018
@dave-fl, see my earlier comment and reference to SPR-17408. We plan a comprehensive review at the Spring Framework level. A similar effort is under way in Reactor Netty. Watch for fixes here and under SPR-17408 targeting Spring Framework 5.1.2 (as a high priority) due on 10/29 with Reactor Netty releases prior to that. We'll then follow up with Spring Cloud Gateway as well but I expect most changes will come from Reactor Netty and WebFlux.
Apologies this has taken as long to address. For sure the goal is for applications to not have to do anything special, unless working with data buffers directly for which we'll provide specific guidance too.
rstoyanchev
on 18 Oct 2018
@danielra I believe this might be the cause for the memory issue in your use case:
2018-10-19 14:32:59.834 ERROR 67399 --- [ctor-http-nio-6] reactor.core.publisher.Operators : Operator called default onErrorDropped
java.lang.NullPointerException: null
at reactor.netty.ByteBufFlux.lambda$aggregate$5(ByteBufFlux.java:230) ~[reactor-netty-0.8.2.BUILD-SNAPSHOT.jar:0.8.2.BUILD-SNAPSHOT]
at reactor.core.publisher.MonoReduceSeed$ReduceSeedSubscriber.onNext(MonoReduceSeed.java:120) ~[reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
at reactor.core.publisher.FluxMap$MapSubscriber.onNext(FluxMap.java:114) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
at reactor.core.publisher.FluxPeek$PeekSubscriber.onNext(FluxPeek.java:192) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
at reactor.core.publisher.MonoFlatMapMany$FlatMapManyInner.onNext(MonoFlatMapMany.java:238) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
at reactor.core.publisher.FluxMap$MapSubscriber.onNext(FluxMap.java:114) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
at reactor.netty.channel.FluxReceive.drainReceiver(FluxReceive.java:211) [reactor-netty-0.8.2.BUILD-SNAPSHOT.jar:0.8.2.BUILD-SNAPSHOT]
at reactor.netty.channel.FluxReceive.onInboundNext(FluxReceive.java:327) [reactor-netty-0.8.2.BUILD-SNAPSHOT.jar:0.8.2.BUILD-SNAPSHOT]
at reactor.netty.channel.ChannelOperations.onInboundNext(ChannelOperations.java:310) [reactor-netty-0.8.2.BUILD-SNAPSHOT.jar:0.8.2.BUILD-SNAPSHOT]
at reactor.netty.http.client.HttpClientOperations.onInboundNext(HttpClientOperations.java:537) [reactor-netty-0.8.2.BUILD-SNAPSHOT.jar:0.8.2.BUILD-SNAPSHOT]
at reactor.netty.channel.ChannelOperationsHandler.channelRead(ChannelOperationsHandler.java:141) [reactor-netty-0.8.2.BUILD-SNAPSHOT.jar:0.8.2.BUILD-SNAPSHOT]
I'm investigating it ...
violetagg
on 19 Oct 2018
@danielra Try with the latest Reactor Netty 0.8.2 and Reactor Core 3.2.2 snapshots.
https://github.com/reactor/reactor-core/issues/1393
470
violetagg
on 19 Oct 2018
Running the demo test project linked previously a few times built against the suggested snapshot versions, I haven't seen any buffer leak reports so far - but I do still sometimes see an IllegalReferenceCountException:
09:21:56.522 [DEBUG] [TestEventLogger] 2018-10-22 09:21:56.521 ERROR 10881 --- [ parallel-2] reactor.core.publisher.Operators : Operator called default onErrorDropped
09:21:56.522 [DEBUG] [TestEventLogger]
09:21:56.522 [DEBUG] [TestEventLogger] io.netty.util.IllegalReferenceCountException: refCnt: 0, decrement: 1
09:21:56.522 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractReferenceCountedByteBuf.release0(AbstractReferenceCountedByteBuf.java:100) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.522 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractReferenceCountedByteBuf.release(AbstractReferenceCountedByteBuf.java:84) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.522 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractPooledDerivedByteBuf$PooledNonRetainedSlicedByteBuf.release0(AbstractPooledDerivedByteBuf.java:271) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.522 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractDerivedByteBuf.release(AbstractDerivedByteBuf.java:85) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.522 [DEBUG] [TestEventLogger] at io.netty.buffer.WrappedByteBuf.release(WrappedByteBuf.java:1029) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.522 [DEBUG] [TestEventLogger] at io.netty.buffer.SimpleLeakAwareByteBuf.release(SimpleLeakAwareByteBuf.java:102) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.522 [DEBUG] [TestEventLogger] at io.netty.buffer.AdvancedLeakAwareByteBuf.release(AdvancedLeakAwareByteBuf.java:941) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.522 [DEBUG] [TestEventLogger] at io.netty.buffer.CompositeByteBuf$Component.freeIfNecessary(CompositeByteBuf.java:1683) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.522 [DEBUG] [TestEventLogger] at io.netty.buffer.CompositeByteBuf.deallocate(CompositeByteBuf.java:1948) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.522 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractReferenceCountedByteBuf.release0(AbstractReferenceCountedByteBuf.java:95) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.523 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractReferenceCountedByteBuf.release(AbstractReferenceCountedByteBuf.java:84) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.523 [DEBUG] [TestEventLogger] at io.netty.buffer.WrappedCompositeByteBuf.release(WrappedCompositeByteBuf.java:43) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.523 [DEBUG] [TestEventLogger] at io.netty.buffer.SimpleLeakAwareCompositeByteBuf.release(SimpleLeakAwareCompositeByteBuf.java:38) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.523 [DEBUG] [TestEventLogger] at io.netty.buffer.AdvancedLeakAwareCompositeByteBuf.release(AdvancedLeakAwareCompositeByteBuf.java:1023) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.523 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxUsing$UsingFuseableSubscriber.cleanup(FluxUsing.java:331) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.523 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxUsing$UsingFuseableSubscriber.cancel(FluxUsing.java:325) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.523 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxMapFuseable$MapFuseableConditionalSubscriber.cancel(FluxMapFuseable.java:351) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.523 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxHandle$HandleSubscriber.cancel(FluxHandle.java:260) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.523 [DEBUG] [TestEventLogger] at reactor.core.publisher.Operators$MultiSubscriptionSubscriber.drainLoop(Operators.java:1905) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.523 [DEBUG] [TestEventLogger] at reactor.core.publisher.Operators$MultiSubscriptionSubscriber.drain(Operators.java:1874) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.523 [DEBUG] [TestEventLogger] at reactor.core.publisher.Operators$MultiSubscriptionSubscriber.cancel(Operators.java:1686) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.523 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxTimeout$TimeoutMainSubscriber.handleTimeout(FluxTimeout.java:287) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.523 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxTimeout$TimeoutMainSubscriber.doTimeout(FluxTimeout.java:273) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.523 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxTimeout$TimeoutTimeoutSubscriber.onNext(FluxTimeout.java:390) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.523 [DEBUG] [TestEventLogger] at reactor.core.publisher.StrictSubscriber.onNext(StrictSubscriber.java:89) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.524 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:73) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.524 [DEBUG] [TestEventLogger] at reactor.core.publisher.MonoDelay$MonoDelayRunnable.run(MonoDelay.java:117) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.524 [DEBUG] [TestEventLogger] at reactor.core.scheduler.SchedulerTask.call(SchedulerTask.java:50) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.524 [DEBUG] [TestEventLogger] at reactor.core.scheduler.SchedulerTask.call(SchedulerTask.java:27) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.524 [DEBUG] [TestEventLogger] at java.util.concurrent.FutureTask.run(FutureTask.java:266) [na:1.8.0_171]
09:21:56.524 [DEBUG] [TestEventLogger] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) [na:1.8.0_171]
09:21:56.524 [DEBUG] [TestEventLogger] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) [na:1.8.0_171]
09:21:56.524 [DEBUG] [TestEventLogger] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [na:1.8.0_171]
09:21:56.524 [DEBUG] [TestEventLogger] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [na:1.8.0_171]
09:21:56.524 [DEBUG] [TestEventLogger] at java.lang.Thread.run(Thread.java:748) [na:1.8.0_171]
09:21:56.524 [DEBUG] [TestEventLogger]
09:21:56.524 [DEBUG] [TestEventLogger] 2018-10-22 09:21:56.523 ERROR 10881 --- [ parallel-2] reactor.core.scheduler.Schedulers : Scheduler worker in group main failed with an uncaught exception
09:21:56.524 [DEBUG] [TestEventLogger]
09:21:56.524 [DEBUG] [TestEventLogger] io.netty.util.IllegalReferenceCountException: refCnt: 0, decrement: 1
09:21:56.524 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractReferenceCountedByteBuf.release0(AbstractReferenceCountedByteBuf.java:100) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.527 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractReferenceCountedByteBuf.release(AbstractReferenceCountedByteBuf.java:84) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.527 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractPooledDerivedByteBuf$PooledNonRetainedSlicedByteBuf.release0(AbstractPooledDerivedByteBuf.java:271) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.527 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractDerivedByteBuf.release(AbstractDerivedByteBuf.java:85) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.527 [DEBUG] [TestEventLogger] at io.netty.buffer.WrappedByteBuf.release(WrappedByteBuf.java:1029) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.527 [DEBUG] [TestEventLogger] at io.netty.buffer.SimpleLeakAwareByteBuf.release(SimpleLeakAwareByteBuf.java:102) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.527 [DEBUG] [TestEventLogger] at io.netty.buffer.AdvancedLeakAwareByteBuf.release(AdvancedLeakAwareByteBuf.java:941) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.527 [DEBUG] [TestEventLogger] at io.netty.buffer.CompositeByteBuf$Component.freeIfNecessary(CompositeByteBuf.java:1683) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.527 [DEBUG] [TestEventLogger] at io.netty.buffer.CompositeByteBuf.deallocate(CompositeByteBuf.java:1948) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.527 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractReferenceCountedByteBuf.release0(AbstractReferenceCountedByteBuf.java:95) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.527 [DEBUG] [TestEventLogger] at io.netty.buffer.AbstractReferenceCountedByteBuf.release(AbstractReferenceCountedByteBuf.java:84) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.527 [DEBUG] [TestEventLogger] at io.netty.buffer.WrappedCompositeByteBuf.release(WrappedCompositeByteBuf.java:43) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.527 [DEBUG] [TestEventLogger] at io.netty.buffer.SimpleLeakAwareCompositeByteBuf.release(SimpleLeakAwareCompositeByteBuf.java:38) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.527 [DEBUG] [TestEventLogger] at io.netty.buffer.AdvancedLeakAwareCompositeByteBuf.release(AdvancedLeakAwareCompositeByteBuf.java:1023) ~[netty-buffer-4.1.29.Final.jar:4.1.29.Final]
09:21:56.527 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxUsing$UsingFuseableSubscriber.cleanup(FluxUsing.java:331) ~[reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.527 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxUsing$UsingFuseableSubscriber.cancel(FluxUsing.java:325) ~[reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.528 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxMapFuseable$MapFuseableConditionalSubscriber.cancel(FluxMapFuseable.java:351) ~[reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.528 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxHandle$HandleSubscriber.cancel(FluxHandle.java:260) ~[reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.528 [DEBUG] [TestEventLogger] at reactor.core.publisher.Operators$MultiSubscriptionSubscriber.drainLoop(Operators.java:1905) ~[reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.528 [DEBUG] [TestEventLogger] at reactor.core.publisher.Operators$MultiSubscriptionSubscriber.drain(Operators.java:1874) ~[reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.528 [DEBUG] [TestEventLogger] at reactor.core.publisher.Operators$MultiSubscriptionSubscriber.cancel(Operators.java:1686) ~[reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.528 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxTimeout$TimeoutMainSubscriber.handleTimeout(FluxTimeout.java:287) ~[reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.528 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxTimeout$TimeoutMainSubscriber.doTimeout(FluxTimeout.java:273) ~[reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.528 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxTimeout$TimeoutTimeoutSubscriber.onNext(FluxTimeout.java:390) ~[reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.528 [DEBUG] [TestEventLogger] at reactor.core.publisher.StrictSubscriber.onNext(StrictSubscriber.java:89) ~[reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.528 [DEBUG] [TestEventLogger] at reactor.core.publisher.FluxOnErrorResume$ResumeSubscriber.onNext(FluxOnErrorResume.java:73) ~[reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.528 [DEBUG] [TestEventLogger] at reactor.core.publisher.MonoDelay$MonoDelayRunnable.run(MonoDelay.java:117) ~[reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.528 [DEBUG] [TestEventLogger] at reactor.core.scheduler.SchedulerTask.call(SchedulerTask.java:50) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.528 [DEBUG] [TestEventLogger] at reactor.core.scheduler.SchedulerTask.call(SchedulerTask.java:27) [reactor-core-3.2.2.BUILD-SNAPSHOT.jar:3.2.2.BUILD-SNAPSHOT]
09:21:56.528 [DEBUG] [TestEventLogger] at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[na:1.8.0_171]
09:21:56.528 [DEBUG] [TestEventLogger] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) ~[na:1.8.0_171]
09:21:56.528 [DEBUG] [TestEventLogger] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) ~[na:1.8.0_171]
09:21:56.528 [DEBUG] [TestEventLogger] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[na:1.8.0_171]
09:21:56.529 [DEBUG] [TestEventLogger] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[na:1.8.0_171]
09:21:56.529 [DEBUG] [TestEventLogger] at java.lang.Thread.run(Thread.java:748) ~[na:1.8.0_171]
@violetagg Any comments on these latest logs and new errors?
dave-fl
on 25 Oct 2018
Will upgrade to SB 2.1.0 today and see if we still see the errors.
madgnome
on 30 Oct 2018
@madgnome just a quick note, if you're using any of: request body predicate, request body transformer, response body transformer, please turn those off for now before testing. There are expected fixes for those in the next Spring Cloud Gateway release.
rstoyanchev
on 1 Nov 2018
We're not using Spring Cloud Gateway.
Unfortunately the issue seems worse in Spring Boot 2.1
We use timeout() operator.
LEAK: ByteBuf.release()
was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records:
#1: io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:273)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1199)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1243)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#2: io.netty.buffer.AdvancedLeakAwareByteBuf.forEachByte(AdvancedLeakAwareByteBuf.java:670)
io.netty.handler.codec.http.HttpObjectDecoder$HeaderParser.parse(HttpObjectDecoder.java:801)
io.netty.handler.codec.http.HttpObjectDecoder$LineParser.parse(HttpObjectDecoder.java:850)
io.netty.handler.codec.http.HttpObjectDecoder.decode(HttpObjectDecoder.java:208)
io.netty.handler.codec.http.HttpClientCodec$Decoder.decode(HttpClientCodec.java:202)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1199)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1243)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#3: Hint: 'reactor.left.httpCodec' will handle the message from this point. io.netty.channel.DefaultChannelPipeline.touch(DefaultChannelPipeline.java:116)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:345)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1429)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1199)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1243)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#4: io.netty.buffer.AdvancedLeakAwareByteBuf.internalNioBuffer(AdvancedLeakAwareByteBuf.java:736)
io.netty.handler.ssl.SslHandler.toByteBuffer(SslHandler.java:1439)
io.netty.handler.ssl.SslHandler.access$300(SslHandler.java:165)
io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:295)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1297)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1199)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1243)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
#5: io.netty.buffer.AdvancedLeakAwareByteBuf.nioBufferCount(AdvancedLeakAwareByteBuf.java:706)
io.netty.handler.ssl.SslHandler.toByteBuffer(SslHandler.java:1439)
io.netty.handler.ssl.SslHandler.access$300(SslHandler.java:165)
io.netty.handler.ssl.SslHandler$SslEngineType$3.unwrap(SslHandler.java:295)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1297)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1199)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1243)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
Created at: io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:331)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:185)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:176)
io.netty.buffer.AbstractByteBufAllocator.buffer(AbstractByteBufAllocator.java:113)
io.netty.handler.ssl.SslHandler.allocate(SslHandler.java:1911)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1292)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1199)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1243)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:489)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:428)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:265)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:628)
io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:563)
io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480)
io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
java.lang.Thread.run(Thread.java:748)
: 13 leak records were discarded because the leak record count is targeted to 4. Use system property io.netty.leakDetection.targetRecords to increase the limit.
👍2
@madgnome, do you have anything reproducible that we can use to investigate? if not can you describe the code in high level terms (e.g. Spring MVC / Boot app, using the WebClient, etc), and provide some relevant code snippets? Note also SPR-17473 just addressed but without any sample code I can't be sure if you use onStatus hooks or not.
rstoyanchev
on 9 Nov 2018
❤1
🎉1
👍1
Thanks @rstoyanchev we do use onStatus without consuming the body so I'll fix that and see if the leak disappears. 🤞
madgnome
on 13 Nov 2018
One issue identified when sending data and request method GET
512
violetagg
on 19 Nov 2018
👍1
Two new fixes related to web sockets and redirection
513 #515
violetagg
on 20 Nov 2018
Making sure we consume the responseBody on onStatus didn't fix the leaks. (It may have reduced the number but not enough to be sure)
I will see if I can try the latest snapshot to get #512 fix
madgnome
on 20 Nov 2018
Deployed the latest snapshot and still seeing the errors unfortunately =(
I've ran the service for DEBUG log for reactor.netty.http here's the log file starting from the leak
And here's another one with all reactor.netty in DEBUG mode.
madgnome
on 21 Nov 2018
We are testing again. Latest build continues to leak. Spring 2.1.0 Release.
dave-fl
on 28 Nov 2018
For the latest version, it seems like using option (on Spring Cloud Gateway):
gateway:
httpclient:
response-timeout: 20s
Is what is causing possibly a lot of the memory leak errors. What I wonder though is that if that is corrected on the Spring Cloud Gateway side, will the leaks still occur when legitimate timeouts are taking place.
dave-fl
on 28 Nov 2018
@violetagg Good news! I was able to reproduce the issue locally and I think it was fixed by https://github.com/reactor/reactor-netty/pull/554
I'll deploy to production to confirm.
I confirm no more leaks in production when using v0.8.4
For people facing a related leak issue and trying to reproduce locally, here are things that helped us reproducing locally:
- Switch to paranoid mode
-Dio.netty.leakDetection.level=paranoid - Introduce gc call (
System.gc()) at the end of your http calls, we put it manually but you could put it more systematically with an handler.
madgnome
on 18 Jan 2019
🎉3
Same problem using Spring 2.1.2.RELEASE which uses reactor-netty-v0.8.4.RELEASE
Casvad
on 9 Feb 2019
I also see the same issue in Spring Boot 2.1.2.RELEASE
2019-02-19 16:10:32.947 ERROR 20234 --- [or-http-epoll-1] io.netty.util.ResourceLeakDetector : LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records:
Created at:
io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:331)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:185)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:176)
io.netty.buffer.AbstractByteBufAllocator.buffer(AbstractByteBufAllocator.java:113)
org.springframework.core.io.buffer.NettyDataBufferFactory.allocateBuffer(NettyDataBufferFactory.java:71)
org.springframework.core.io.buffer.NettyDataBufferFactory.allocateBuffer(NettyDataBufferFactory.java:39)
org.springframework.core.codec.CharSequenceEncoder.lambda$encode$1(CharSequenceEncoder.java:85)
reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.onNext(FluxMapFuseable.java:107)
reactor.core.publisher.FluxJust$WeakScalarSubscription.request(FluxJust.java:99)
reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.request(FluxMapFuseable.java:162)
reactor.core.publisher.MonoNext$NextSubscriber.request(MonoNext.java:102)
reactor.core.publisher.Operators$MultiSubscriptionSubscriber.set(Operators.java:1878)
reactor.core.publisher.Operators$MultiSubscriptionSubscriber.onSubscribe(Operators.java:1752)
reactor.core.publisher.MonoNext$NextSubscriber.onSubscribe(MonoNext.java:64)
reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.onSubscribe(FluxMapFuseable.java:90)
reactor.core.publisher.FluxJust.subscribe(FluxJust.java:70)
reactor.core.publisher.FluxMapFuseable.subscribe(FluxMapFuseable.java:63)
reactor.core.publisher.MonoNext.subscribe(MonoNext.java:40)
reactor.core.publisher.MonoSwitchIfEmpty.subscribe(MonoSwitchIfEmpty.java:44)
reactor.core.publisher.MonoFlatMap.subscribe(MonoFlatMap.java:60)
reactor.core.publisher.MonoFlatMap$FlatMapMain.onNext(MonoFlatMap.java:150)
reactor.core.publisher.MonoCreate$DefaultMonoSink.success(MonoCreate.java:147)
kotlinx.coroutines.reactor.MonoCoroutine.onCompleted(Mono.kt:51)
kotlinx.coroutines.AbstractCoroutine.onCompletionInternal$kotlinx_coroutines_core(AbstractCoroutine.kt:108)
kotlinx.coroutines.JobSupport.completeStateFinalization(JobSupport.kt:305)
kotlinx.coroutines.JobSupport.tryFinalizeSimpleState(JobSupport.kt:264)
kotlinx.coroutines.JobSupport.tryMakeCompleting(JobSupport.kt:762)
kotlinx.coroutines.JobSupport.makeCompletingOnce$kotlinx_coroutines_core(JobSupport.kt:742)
kotlinx.coroutines.AbstractCoroutine.resumeWith(AbstractCoroutine.kt:117)
kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:45)
kotlinx.coroutines.DispatchedTask.run(Dispatched.kt:233)
kotlinx.coroutines.scheduling.CoroutineScheduler.runSafely(CoroutineScheduler.kt:594)
kotlinx.coroutines.scheduling.CoroutineScheduler.access$runSafely(CoroutineScheduler.kt:60)
kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.run(CoroutineScheduler.kt:742)
wdroste
on 19 Feb 2019
Same issue in Spring Boot 2.1.2 RELEASE
Basic code is feeding bytes into parser.
return body
.reduce(ParserContainer()) { c, buf ->
try {
// parse each fragment
c.next(buf.asByteBuffer())
} finally {
// memory leak detector says it needs to be released
DataBufferUtils.release(buf)
}
}
// extract the response
.map { it.response() ?: EmptyResponse() }
Exception:
2019-02-19 16:10:32.947 ERROR 20234 --- [or-http-epoll-1] io.netty.util.ResourceLeakDetector : LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records:
Created at:
io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:331)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:185)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:176)
io.netty.buffer.AbstractByteBufAllocator.buffer(AbstractByteBufAllocator.java:113)
org.springframework.core.io.buffer.NettyDataBufferFactory.allocateBuffer(NettyDataBufferFactory.java:71)
org.springframework.core.io.buffer.NettyDataBufferFactory.allocateBuffer(NettyDataBufferFactory.java:39)
org.springframework.core.codec.CharSequenceEncoder.lambda$encode$1(CharSequenceEncoder.java:85)
reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.onNext(FluxMapFuseable.java:107)
reactor.core.publisher.FluxJust$WeakScalarSubscription.request(FluxJust.java:99)
reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.request(FluxMapFuseable.java:162)
reactor.core.publisher.MonoNext$NextSubscriber.request(MonoNext.java:102)
reactor.core.publisher.Operators$MultiSubscriptionSubscriber.set(Operators.java:1878)
reactor.core.publisher.Operators$MultiSubscriptionSubscriber.onSubscribe(Operators.java:1752)
reactor.core.publisher.MonoNext$NextSubscriber.onSubscribe(MonoNext.java:64)
reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.onSubscribe(FluxMapFuseable.java:90)
reactor.core.publisher.FluxJust.subscribe(FluxJust.java:70)
reactor.core.publisher.FluxMapFuseable.subscribe(FluxMapFuseable.java:63)
reactor.core.publisher.MonoNext.subscribe(MonoNext.java:40)
reactor.core.publisher.MonoSwitchIfEmpty.subscribe(MonoSwitchIfEmpty.java:44)
reactor.core.publisher.MonoFlatMap.subscribe(MonoFlatMap.java:60)
reactor.core.publisher.MonoFlatMap$FlatMapMain.onNext(MonoFlatMap.java:150)
reactor.core.publisher.MonoCreate$DefaultMonoSink.success(MonoCreate.java:147)
kotlinx.coroutines.reactor.MonoCoroutine.onCompleted(Mono.kt:51)
kotlinx.coroutines.AbstractCoroutine.onCompletionInternal$kotlinx_coroutines_core(AbstractCoroutine.kt:108)
kotlinx.coroutines.JobSupport.completeStateFinalization(JobSupport.kt:305)
kotlinx.coroutines.JobSupport.tryFinalizeSimpleState(JobSupport.kt:264)
kotlinx.coroutines.JobSupport.tryMakeCompleting(JobSupport.kt:762)
kotlinx.coroutines.JobSupport.makeCompletingOnce$kotlinx_coroutines_core(JobSupport.kt:742)
kotlinx.coroutines.AbstractCoroutine.resumeWith(AbstractCoroutine.kt:117)
kotlin.coroutines.jvm.internal.BaseContinuationImpl.resumeWith(ContinuationImpl.kt:45)
kotlinx.coroutines.DispatchedTask.run(Dispatched.kt:233)
kotlinx.coroutines.scheduling.CoroutineScheduler.runSafely(CoroutineScheduler.kt:594)
kotlinx.coroutines.scheduling.CoroutineScheduler.access$runSafely(CoroutineScheduler.kt:60)
kotlinx.coroutines.scheduling.CoroutineScheduler$Worker.run(CoroutineScheduler.kt:742)
I'd try with boot 2.1.3.RELEASE as the reactor team has worked hard on these issues.
spencergibb
on 19 Feb 2019
@spencergibb will give it a try.
wdroste
on 20 Feb 2019
Unfortunately I have a similar error in Spring Boot 2.1.3.RELEASE, occurring in production. It may be related to the other side closing the connection just before, as we saw this error just before: reactor.netty.http.client.PrematureCloseException: Connection has been closed BEFORE response, while sending request body
Not doing anything fancy. Very basic high-level code.
Mono<ClientResponse> doRequest(WebhookCall webhookCall) {
WebhookSetting webhookSetting = webhookCall.getWebhookSetting();
String authString = webhookSetting.getAuthorizationHeader();
return webClient.post()
.uri(webhookSetting.getEndpoint())
.headers(httpHeaders -> {
if (authString != null)
httpHeaders.add("Authorization", authString);
})
.body(BodyInserters.fromObject(getEventDTO(webhookCall.getEvent())))
.exchange();
}
and that code being called in this way:
Mono<ClientResponse> clientResponseMono = webhookRequestPoster.doRequest(webhookCall);
clientResponseMono.subscribe(
clientResponse -> clientResponseWritingService.insertClientResponse(webhookCall, clientResponse),
throwable -> clientResponseWritingService.insertConnectionErrorResponse(webhookCall, throwable));
Error:
LEAK: ByteBuf.release() was not called before it's garbage-collected. See http://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records:
Created at:
io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:339)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:185)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:171)
io.netty.buffer.AbstractByteBufAllocator.buffer(AbstractByteBufAllocator.java:105)
org.springframework.core.io.buffer.NettyDataBufferFactory.allocateBuffer(NettyDataBufferFactory.java:65)
org.springframework.core.io.buffer.NettyDataBufferFactory.allocateBuffer(NettyDataBufferFactory.java:39)
org.springframework.http.codec.json.AbstractJackson2Encoder.encodeValue(AbstractJackson2Encoder.java:172)
org.springframework.http.codec.json.AbstractJackson2Encoder.lambda$encode$0(AbstractJackson2Encoder.java:122)
reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.onNext(FluxMapFuseable.java:107)
reactor.core.publisher.Operators$ScalarSubscription.request(Operators.java:2070)
reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.request(FluxMapFuseable.java:162)
reactor.core.publisher.Operators$MultiSubscriptionSubscriber.set(Operators.java:1878)
reactor.core.publisher.Operators$MultiSubscriptionSubscriber.onSubscribe(Operators.java:1752)
reactor.core.publisher.FluxMapFuseable$MapFuseableSubscriber.onSubscribe(FluxMapFuseable.java:90)
reactor.core.publisher.MonoJust.subscribe(MonoJust.java:54)
reactor.core.publisher.MonoMapFuseable.subscribe(MonoMapFuseable.java:59)
reactor.core.publisher.MonoSwitchIfEmpty.subscribe(MonoSwitchIfEmpty.java:44)
reactor.core.publisher.MonoFlatMap.subscribe(MonoFlatMap.java:60)
reactor.core.publisher.MonoDefer.subscribe(MonoDefer.java:52)
reactor.netty.http.client.HttpClientConnect$HttpObserver.onStateChange(HttpClientConnect.java:421)
reactor.netty.resources.PooledConnectionProvider$DisposableAcquire.onStateChange(PooledConnectionProvider.java:514)
reactor.netty.resources.PooledConnectionProvider$DisposableAcquire.run(PooledConnectionProvider.java:544)
io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:163)
io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:404)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:333)
io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:905)
java.base/java.lang.Thread.run(Thread.java:834)
Bas83
on 4 Apr 2019
@Bas83 Try with the latest:
- Reactor Core 3.2.9.BUILD-SNAPSHOT
- Reactor Netty 0.8.7.BUILD-SNAPSHOT
- Spring Framework 5.1.7.BUILD-SNAPSHOT
violetagg
on 4 Apr 2019
@violetagg have there been additional fixes in those versions or is that just to make sure it still exists in the current version? I would love to reproduce it in those versions but since the error is in production it's going to take some time.
Bas83
on 4 Apr 2019
@Bas83 yes there are
- Reactor Core - https://github.com/reactor/reactor-core/issues/1634 ; https://github.com/reactor/reactor-core/issues/1622
- Reactor Netty - #585 #694 #698
- Spring Framework - https://github.com/spring-projects/spring-framework/issues/22720
violetagg
on 4 Apr 2019
👍4
@violetagg I have the same issue in production using elastic Scheduler (unbounded) and the following version of the frameworks :
- Reactor Core 3.3.0.RELEASE
- Reactor Netty 0.9.1.RELEASE
- Spring Framework 5.2.1.RELEASE
any help or explanation is appreciated. here is the stack trace
ERROR io.netty.util.ResourceLeakDetector.reportTracedLeak - LEAK: ByteBuf.release() was not called before it's garbage-collected. See https://netty.io/wiki/reference-counted-objects.html for more information.
Recent access records:
Created at:
io.netty.buffer.PooledByteBufAllocator.newDirectBuffer(PooledByteBufAllocator.java:349)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:187)
io.netty.buffer.AbstractByteBufAllocator.directBuffer(AbstractByteBufAllocator.java:178)
io.netty.buffer.AbstractByteBufAllocator.buffer(AbstractByteBufAllocator.java:115)
io.netty.handler.ssl.SslHandler.allocate(SslHandler.java:2146)
io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1327)
io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1227)
io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1274)
io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:503)
io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:442)
io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:281)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:352)
io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1422)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:374)
io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:360)
io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:931)
io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:792)
io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:502)
io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:407)
io.netty.util.concurrent.SingleThreadEventExecutor$6.run(SingleThreadEventExecutor.java:1050)
io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)
java.base/java.lang.Thread.run(Thread.java:834)
saad14092
on 26 Nov 2019
@saad14092 Please open a new issue. It will be really helpful if you provide more info for the use case and/or a test example
violetagg
on 26 Nov 2019
👍3
Was this page helpful?
0 / 5 - 0 ratings
Related issues
flisky
·
6Comments
anh-dev
·
3Comments
smaldini
·
3Comments
be-hase
·
6Comments
hisener
·
4Comments
Most helpful comment
@dave-fl yep I think we have an issue. I'm on it.