Rabbitmq-server: Management plugin, JSON encode error "bad term" x-cancel-on-ha-failover starting from v3.6.7

[micdenny]

When there's a consumer connected to a queue with x-cancel-on-ha-failover, is not possible to open the queue detail from the management plugin.

RabbitMQ 3.6.7
Erlang: 19.0

With RabbitMQ 3.6.6 it works with the same erlang version 19.0

Here's the error:

Got response code 500 with body {"error":"JSON encode error: {bad_term,{<<\"x-cancel-on-ha-failover\">>,bool,false}}","reason":"While encoding: \n[{incoming,[]},\n {deliveries,[]},\n {messages_details,\n [{avg,0.0},\n {avg_rate,0.0},\n {samples,\n [[{timestamp,1490265830000},{sample,0}],\n [{timestamp,1490265825000},{sample,0}],\n [{timestamp,1490265820000},{sample,0}],\n [{timestamp,1490265815000},{sample,0}],\n [{timestamp,1490265810000},{sample,0}],\n [{timestamp,1490265805000},{sample,0}],\n [{timestamp,1490265800000},{sample,0}],\n [{timestamp,1490265795000},{sample,0}],\n [{timestamp,1490265790000},{sample,0}],\n [{timestamp,1490265785000},{sample,0}],\n [{timestamp,1490265780000},{sample,0}],\n [{timestamp,1490265775000},{sample,0}],\n [{timestamp,1490265770000},{sample,0}]]},\n {rate,0.0}]},\n {messages,0},\n {messages_unacknowledged_details,\n [{avg,0.0},\n {avg_rate,0.0},\n {samples,\n [[{timestamp,1490265830000},{sample,0}],\n [{timestamp,1490265825000},{sample,0}],\n [{timestamp,1490265820000},{sample,0}],\n [{timestamp,1490265815000},{sample,0}],\n [{timestamp,1490265810000},{sample,0}],\n [{timestamp,1490265805000},{sample,0}],\n [{timestamp,1490265800000},{sample,0}],\n [{timestamp,1490265795000},{sample,0}],\n [{timestamp,1490265790000},{sample,0}],\n [{timestamp,1490265785000},{sample,0}],\n [{timestamp,1490265780000},{sample,0}],\n [{timestamp,1490265775000},{sample,0}],\n [{timestamp,1490265770000},{sample,0}]]},\n {rate,0.0}]},\n {messages_unacknowledged,0},\n {messages_ready_details,\n [{avg,0.0},\n {avg_rate,0.0},\n {samples,\n [[{timestamp,1490265830000},{sample,0}],\n [{timestamp,1490265825000},{sample,0}],\n [{timestamp,1490265820000},{sample,0}],\n [{timestamp,1490265815000},{sample,0}],\n [{timestamp,1490265810000},{sample,0}],\n [{timestamp,1490265805000},{sample,0}],\n [{timestamp,1490265800000},{sample,0}],\n [{timestamp,1490265795000},{sample,0}],\n [{timestamp,1490265790000},{sample,0}],\n [{timestamp,1490265785000},{sample,0}],\n [{timestamp,1490265780000},{sample,0}],\n [{timestamp,1490265775000},{sample,0}],\n [{timestamp,1490265770000},{sample,0}]]},\n {rate,0.0}]},\n {messages_ready,0},\n {reductions_details,\n [{avg,443746.0},\n {avg_rate,0.0},\n {samples,\n [[{timestamp,1490265825000},{sample,443746}],\n [{timestamp,1490265820000},{sample,443746}],\n [{timestamp,1490265815000},{sample,443746}],\n [{timestamp,1490265810000},{sample,443746}],\n [{timestamp,1490265805000},{sample,443746}],\n [{timestamp,1490265800000},{sample,443746}],\n [{timestamp,1490265795000},{sample,443746}],\n [{timestamp,1490265790000},{sample,443746}],\n [{timestamp,1490265785000},{sample,443746}],\n [{timestamp,1490265780000},{sample,443746}],\n [{timestamp,1490265775000},{sample,443746}],\n [{timestamp,1490265770000},{sample,443746}],\n [{timestamp,1490265765000},{sample,443746}]]},\n {rate,0.0}]},\n {reductions,443746},\n {message_stats,\n [{deliver_get_details,\n [{avg,369.0},\n {avg_rate,0.0},\n {samples,\n [[{timestamp,1490265825000},{sample,369}],\n [{timestamp,1490265820000},{sample,369}],\n [{timestamp,1490265815000},{sample,369}],\n [{timestamp,1490265810000},{sample,369}],\n [{timestamp,1490265805000},{sample,369}],\n [{timestamp,1490265800000},{sample,369}],\n [{timestamp,1490265795000},{sample,369}],\n [{timestamp,1490265790000},{sample,369}],\n [{timestamp,1490265785000},{sample,369}],\n [{timestamp,1490265780000},{sample,369}],\n [{timestamp,1490265775000},{sample,369}],\n [{timestamp,1490265770000},{sample,369}],\n [{timestamp,1490265765000},{sample,369}]]},\n {rate,0.0}]},\n {deliver_get,369},\n {ack_details,\n [{avg,369.0},\n {avg_rate,0.0},\n {samples,\n [[{timestamp,1490265825000},{sample,369}],\n [{timestamp,1490265820000},{sample,369}],\n [{timestamp,1490265815000},{sample,369}],\n [{timestamp,1490265810000},{sample,369}],\n [{timestamp,1490265805000},{sample,369}],\n [{timestamp,1490265800000},{sample,369}],\n [{timestamp,1490265795000},{sample,369}],\n [{timestamp,1490265790000},{sample,369}],\n [{timestamp,1490265785000},{sample,369}],\n [{timestamp,1490265780000},{sample,369}],\n [{timestamp,1490265775000},{sample,369}],\n [{timestamp,1490265770000},{sample,369}],\n [{timestamp,1490265765000},{sample,369}]]},\n {rate,0.0}]},\n {ack,369},\n {redeliver_details,\n [{avg,0.0},\n {avg_rate,0.0},\n {samples,\n [[{timestamp,1490265825000},{sample,0}],\n [{timestamp,1490265820000},{sample,0}],\n [{timestamp,1490265815000},{sample,0}],\n [{timestamp,1490265810000},{sample,0}],\n [{timestamp,1490265805000},{sample,0}],\n [{timestamp,1490265800000},{sample,0}],\n [{timestamp,1490265795000},{sample,0}],\n [{timestamp,1490265790000},{sample,0}],\n [{timestamp,1490265785000},{sample,0}],\n [{timestamp,1490265780000},{sample,0}],\n [{timestamp,1490265775000},{sample,0}],\n [{timestamp,1490265770000},{sample,0}],\n [{timestamp,1490265765000},{sample,0}]]},\n {rate,0.0}]},\n {redeliver,0},\n {deliver_no_ack_details,\n [{avg,0.0},\n {avg_rate,0.0},\n {samples,\n [[{timestamp,1490265825000},{sample,0}],\n [{timestamp,1490265820000},{sample,0}],\n [{timestamp,1490265815000},{sample,0}],\n [{timestamp,1490265810000},{sample,0}],\n [{timestamp,1490265805000},{sample,0}],\n [{timestamp,1490265800000},{sample,0}],\n [{timestamp,1490265795000},{sample,0}],\n [{timestamp,1490265790000},{sample,0}],\n [{timestamp,1490265785000},{sample,0}],\n [{timestamp,1490265780000},{sample,0}],\n [{timestamp,1490265775000},{sample,0}],\n [{timestamp,1490265770000},{sample,0}],\n [{timestamp,1490265765000},{sample,0}]]},\n {rate,0.0}]},\n {deliver_no_ack,0},\n {deliver_details,\n [{avg,369.0},\n {avg_rate,0.0},\n {samples,\n [[{timestamp,1490265825000},{sample,369}],\n [{timestamp,1490265820000},{sample,369}],\n [{timestamp,1490265815000},{sample,369}],\n [{timestamp,1490265810000},{sample,369}],\n [{timestamp,1490265805000},{sample,369}],\n [{timestamp,1490265800000},{sample,369}],\n [{timestamp,1490265795000},{sample,369}],\n [{timestamp,1490265790000},{sample,369}],\n [{timestamp,1490265785000},{sample,369}],\n [{timestamp,1490265780000},{sample,369}],\n [{timestamp,1490265775000},{sample,369}],\n [{timestamp,1490265770000},{sample,369}],\n [{timestamp,1490265765000},{sample,369}]]},\n {rate,0.0}]},\n {deliver,369},\n {get_no_ack_details,\n [{avg,0.0},\n {avg_rate,0.0},\n {samples,\n [[{timestamp,1490265825000},{sample,0}],\n [{timestamp,1490265820000},{sample,0}],\n [{timestamp,1490265815000},{sample,0}],\n [{timestamp,1490265810000},{sample,0}],\n [{timestamp,1490265805000},{sample,0}],\n [{timestamp,1490265800000},{sample,0}],\n [{timestamp,1490265795000},{sample,0}],\n [{timestamp,1490265790000},{sample,0}],\n [{timestamp,1490265785000},{sample,0}],\n [{timestamp,1490265780000},{sample,0}],\n [{timestamp,1490265775000},{sample,0}],\n [{timestamp,1490265770000},{sample,0}],\n [{timestamp,1490265765000},{sample,0}]]},\n {rate,0.0}]},\n {get_no_ack,0},\n {get_details,\n [{avg,0.0},\n {avg_rate,0.0},\n {samples,\n [[{timestamp,1490265825000},{sample,0}],\n [{timestamp,1490265820000},{sample,0}],\n [{timestamp,1490265815000},{sample,0}],\n [{timestamp,1490265810000},{sample,0}],\n [{timestamp,1490265805000},{sample,0}],\n [{timestamp,1490265800000},{sample,0}],\n [{timestamp,1490265795000},{sample,0}],\n [{timestamp,1490265790000},{sample,0}],\n [{timestamp,1490265785000},{sample,0}],\n [{timestamp,1490265780000},{sample,0}],\n [{timestamp,1490265775000},{sample,0}],\n [{timestamp,1490265770000},{sample,0}],\n [{timestamp,1490265765000},{sample,0}]]},\n {rate,0.0}]},\n {get,0},\n {publish_details,\n [{avg,369.0},\n {avg_rate,0.0},\n {samples,\n [[{timestamp,1490265825000},{sample,369}],\n [{timestamp,1490265820000},{sample,369}],\n [{timestamp,1490265815000},{sample,369}],\n [{timestamp,1490265810000},{sample,369}],\n [{timestamp,1490265805000},{sample,369}],\n [{timestamp,1490265800000},{sample,369}],\n [{timestamp,1490265795000},{sample,369}],\n [{timestamp,1490265790000},{sample,369}],\n [{timestamp,1490265785000},{sample,369}],\n [{timestamp,1490265780000},{sample,369}],\n [{timestamp,1490265775000},{sample,369}],\n [{timestamp,1490265770000},{sample,369}],\n [{timestamp,1490265765000},{sample,369}]]},\n {rate,0.0}]},\n {publish,369}]},\n {node,'rabbit@mdenny-ws'},\n {arguments,{struct,[]}},\n {exclusive,false},\n {auto_delete,false},\n {durable,true},\n {vhost,<<\"/\">>},\n {name,<<\"TEST\">>},\n {message_bytes_paged_out,0},\n {messages_paged_out,0},\n {backing_queue_status,\n {struct,\n [{mode,default},\n {q1,0},\n {q2,0},\n {delta,[delta,undefined,0,0,undefined]},\n {q3,0},\n {q4,0},\n {len,0},\n {target_ram_count,infinity},\n {next_seq_id,369},\n {avg_ingress_rate,0.7291118917432003},\n {avg_egress_rate,0.7291118917432003},\n {avg_ack_ingress_rate,0.7291118917432003},\n {avg_ack_egress_rate,0.7291118917432003}]}},\n {head_message_timestamp,''},\n {message_bytes_persistent,0},\n {message_bytes_ram,0},\n {message_bytes_unacknowledged,0},\n {message_bytes_ready,0},\n {message_bytes,0},\n {messages_persistent,0},\n {messages_unacknowledged_ram,0},\n {messages_ready_ram,0},\n {messages_ram,0},\n {garbage_collection,\n [{minor_gcs,194},\n {fullsweep_after,65535},\n {min_heap_size,233},\n {min_bin_vheap_size,46422},\n {max_heap_size,0}]},\n {state,running},\n {recoverable_slaves,''},\n {memory,143136},\n {consumer_utilisation,''},\n {consumers,0},\n {exclusive_consumer_tag,''},\n {policy,''},\n {consumer_details,\n [[{arguments,\n [{<<\"x-cancel-on-ha-failover\">>,bool,false},\n {<<\"x-priority\">>,signedint,0}]},\n {prefetch_count,50},\n {ack_required,true},\n {exclusive,false},\n {consumer_tag,<<\"dd7742d6-acc3-4905-bf40-f6a46de135d9\">>},\n {channel_details,\n [{peer_host,<<\"127.0.0.1\">>},\n {peer_port,50657},\n {connection_name,<<\"127.0.0.1:50657 -> 127.0.0.1:5672\">>},\n {user,<<\"guest\">>},\n {number,2},\n {node,'rabbit@mdenny-ws'},\n {name,<<\"127.0.0.1:50657 -> 127.0.0.1:5672 (2)\">>}]},\n {queue,[{vhost,<<\"/\">>},{name,<<\"TEST\">>}]}]]}]"}

[micdenny]

Sorry I had to push this in https://github.com/rabbitmq/rabbitmq-management/issues

Maybe is a duplicate of https://github.com/rabbitmq/rabbitmq-management-agent/issues/34? but I'm using erlang 19.0 and I also tested RabbitMQ v3.6.8 with same error

If there's no consumer the queue is accessible.

[michaelklishin]

It's the same issue as rabbitmq/rabbitmq-management-agent#34 but in a different place. How can this be reproduced so that I can file a new issue there?

[michaelklishin]

OK, so looks like adding a consumer with extra arguments should be enough.

[micdenny]

How can this be reproduced so that I can file a new issue there?

I'm using EasyNetQ, just install the latest version from nuget, then this is the c# code:

using System;
using EasyNetQ;

namespace BadTermIssue
{
    public class Program
    {
        public static void Main(string[] args)
        {
            var bus = RabbitHutch.CreateBus("host=127.0.0.1", x => x.Register<IEasyNetQLogger>(s => new EasyNetQ.Loggers.ConsoleLogger()));

            bus.Receive<MyMessage>("BAD_TERM_ISSUE", message => { });

            Console.WriteLine("Press enter to stop the consumer and exit the application...");
            Console.ReadLine();

            bus.Dispose();
        }
    }

    public class MyMessage
    {
    }
}

When it's running try to access the queue -> http://localhost:15672/#/queues/%2F/BAD_TERM_ISSUE

[michaelklishin]

Moved to https://github.com/rabbitmq/rabbitmq-management-agent/issues/39, we can reproduce.

[michaelklishin]

@micdenny https://github.com/rabbitmq/rabbitmq-management-agent/issues/39 is fixed and will be in 3.6.9, thank you for reporting.

[micdenny]

tested and it works like a charm