Jormungandr: 0.8.5-alpha1 panics observed

Created on 21 Dec 2019 · 2Comments · Source: input-output-hk/jormungandr

Panics observed with 0.8.5-alpha1 across a cluster of several nodes:

1) thread 'block0' panicked at 'missed chain pull root candidate

received block announcement
received block announcement from network
receiving header stream from network
receiving block stream from network
thread 'block0' panicked at 'missed chain pull root candidate c2659efb12515586e4081afde99384e229af46e599bfb4c6b4509fd1d5e7b7bd that got committed to storage', jormungandr/src/blockchain/candidate.rs:553:13
stack backtrace:
received block announcement
received block announcement from network                                                                                                                                                                          |
   0:     0x556cdaa71ceb - std::panicking::default_hook::{{closure}}::h96cbf7b454e3f557
   1:     0x556cdaa719c6 - std::panicking::default_hook::h95a8f00337383d83
   2:     0x556cdaa7245d - std::panicking::rust_panic_with_hook::h92f98b46e22f14ed
   3:     0x556cdaa71fe2 - std::panicking::continue_panic_fmt::h25abfbb4e5b7043a
   4:     0x556cdaa71f2f - std::panicking::begin_panic_fmt::hc33ce01c7f7cd52d
   5:     0x556cda5390b7 - jormungandr::blockchain::candidate::CandidateForestThickets::apply_block::h4b43880cffc3e895
   6:     0x556cda3fa207 - futures::future::chain::Chain<A,B,C>::poll::he889e97da91b38c7
   7:     0x556cda3e2f49 - futures::future::chain::Chain<A,B,C>::poll::hb3f812da50abc64e
   8:     0x556cda21bec9 - <futures::future::either::Either<A,B> as futures::future::Future>::poll::h8669a0514afdced1
   9:     0x556cda3ec882 - futures::future::chain::Chain<A,B,C>::poll::hc068ec2e61b14ac9
  10:     0x556cda3fd8c2 - futures::future::chain::Chain<A,B,C>::poll::hec11cc816ff86ecb
  11:     0x556cda3c9b62 - futures::future::chain::Chain<A,B,C>::poll::h8ff44977d3314539
  12:     0x556cda32d2d9 - <futures::future::loop_fn::LoopFn<A,F> as futures::future::Future>::poll::h6b487f1eb0ef581e
  13:     0x556cda3e2303 - futures::future::chain::Chain<A,B,C>::poll::hb2d9c58ee76417cc
  14:     0x556cda33651b - <futures::future::map_err::MapErr<A,F> as futures::future::Future>::poll::h852820e2dcf93c0e
  15:     0x556cdaa4d802 - futures::task_impl::std::set::h1e2c0f463cccad1f
  16:     0x556cdaa4d462 - std::panicking::try::do_call::hfd59f9795844b37e
  17:     0x556cdaa76c1a - __rust_maybe_catch_panic
  18:     0x556cdaa4b539 - tokio_threadpool::task::Task::run::hdd33ce00a55c22d0
  19:     0x556cdaa47d76 - tokio_threadpool::worker::Worker::run_task::h3a0825ded0a3ebf6
  20:     0x556cdaa47210 - tokio_threadpool::worker::Worker::run::h4ca8c7c856069205
  21:     0x556cdaa2357e - tokio_timer::clock::clock::with_default::h3ebf42f0968bec7b
  22:     0x556cdaa28093 - tokio::runtime::threadpool::builder::Builder::build::{{closure}}::h29b84bc8a6aa6432
  23:     0x556cdaa4d0ca - std::thread::local::LocalKey<T>::with::h1c49b02109ad3d66
  24:     0x556cdaa4d1e9 - std::thread::local::LocalKey<T>::with::h597a6a28ea622399
  25:     0x556cdaa448d8 - std::sys_common::backtrace::__rust_begin_short_backtrace::h6487e40255b569de
  26:     0x556cdaa4d3bc - std::panicking::try::do_call::h522fd0d86c21226d
  27:     0x556cdaa76c1a - __rust_maybe_catch_panic
  28:     0x556cdaa45880 - core::ops::function::FnOnce::call_once{{vtable.shim}}::hf52307807037b5df
  29:     0x556cdaa6277f - <alloc::boxed::Box<F> as core::ops::function::FnOnce<A>>::call_once::h2877b2aefb0fd627
  30:     0x556cdaa75f60 - std::sys::unix::thread::Thread::new::thread_start::h382c6ae14181d163
  31:     0x7ff9f2f0aef7 - start_thread
  32:     0x7ff9f2c2822f - __GI___clone
  33:                0x0 - <unknown>
Task panicked

2) thread 'block1' panicked at 'referential integrity failure in CandidateForest', src/libcore/option.rs:1166:5

received block announcement
received block announcement
received block announcement from network
received block announcement from network
received block announcement
received block announcement from network
thread 'block1' panicked at 'referential integrity failure in CandidateForest', src/libcore/option.rs:1166:5
stack backtrace:
   0:     0x55abe53cfceb - std::panicking::default_hook::{{closure}}::h96cbf7b454e3f557
   1:     0x55abe53cf9c6 - std::panicking::default_hook::h95a8f00337383d83
   2:     0x55abe53d045d - std::panicking::rust_panic_with_hook::h92f98b46e22f14ed
   3:     0x55abe53cffe2 - std::panicking::continue_panic_fmt::h25abfbb4e5b7043a
   4:     0x55abe53cfec6 - rust_begin_unwind
   5:     0x55abe53ed52d - core::panicking::panic_fmt::h7e9f94035af782b3
   6:     0x55abe53ed597 - core::option::expect_failed::h292a6217b603c24f
   7:     0x55abe4e996c0 - jormungandr::blockchain::candidate::CandidateForestThickets::poll_purge::h8b64bb39829a4fa4
   8:     0x55abe4d52626 - futures::future::chain::Chain<A,B,C>::poll::hd545a9a7ca9f05a6
   9:     0x55abe4efc4e9 - <futures::stream::for_each::ForEach<S,F,U> as futures::future::Future>::poll::h42f555f02309d7d1
  10:     0x55abe4c8e61c - <futures::future::map_err::MapErr<A,F> as futures::future::Future>::poll::h183b886f098f2ceb
  11:     0x55abe53ab802 - futures::task_impl::std::set::h1e2c0f463cccad1f
  12:     0x55abe53ab462 - std::panicking::try::do_call::hfd59f9795844b37e
  13:     0x55abe53d4c1a - __rust_maybe_catch_panic
  14:     0x55abe53a9539 - tokio_threadpool::task::Task::run::hdd33ce00a55c22d0
  15:     0x55abe53a5d76 - tokio_threadpool::worker::Worker::run_task::h3a0825ded0a3ebf6
  16:     0x55abe53a53e8 - tokio_threadpool::worker::Worker::run::h4ca8c7c856069205
  17:     0x55abe538157e - tokio_timer::clock::clock::with_default::h3ebf42f0968bec7b
  18:     0x55abe5386093 - tokio::runtime::threadpool::builder::Builder::build::{{closure}}::h29b84bc8a6aa6432
  19:     0x55abe53ab0ca - std::thread::local::LocalKey<T>::with::h1c49b02109ad3d66
  20:     0x55abe53ab1e9 - std::thread::local::LocalKey<T>::with::h597a6a28ea622399
  21:     0x55abe53a28d8 - std::sys_common::backtrace::__rust_begin_short_backtrace::h6487e40255b569de
  22:     0x55abe53ab3bc - std::panicking::try::do_call::h522fd0d86c21226d
  23:     0x55abe53d4c1a - __rust_maybe_catch_panic
  24:     0x55abe53a3880 - core::ops::function::FnOnce::call_once{{vtable.shim}}::hf52307807037b5df
  25:     0x55abe53c077f - <alloc::boxed::Box<F> as core::ops::function::FnOnce<A>>::call_once::h2877b2aefb0fd627
  26:     0x55abe53d3f60 - std::sys::unix::thread::Thread::new::thread_start::h382c6ae14181d163
  27:     0x7f6853afdef7 - start_thread
  28:     0x7f685381b22f - __GI___clone
  29:                0x0 - <unknown>
Task panicked

3) thread 'block2' panicked at 'internal error: entered unreachable code', jormungandr/src/blockchain/storage.rs:92:49

receiving header stream from network
receiving header stream from network
receiving block stream from network
receiving header stream from network
receiving block stream from network
receiving header stream from network
receiving header stream from network
thread 'block2' panicked at 'internal error: entered unreachable code', jormungandr/src/blockchain/storage.rs:92:49
stack backtrace:
   0:     0x55bc8f0f1ceb - std::panicking::default_hook::{{closure}}::h96cbf7b454e3f557
   1:     0x55bc8f0f19c6 - std::panicking::default_hook::h95a8f00337383d83
   2:     0x55bc8f0f245d - std::panicking::rust_panic_with_hook::h92f98b46e22f14ed
   3:     0x55bc8ebe8e25 - std::panicking::begin_panic::h7387897093777db5
   4:     0x55bc8e8a511d - jormungandr::blockchain::storage::Storage::put_block::h83d43416bb6d40ce
   5:     0x55bc8ea7109a - futures::future::chain::Chain<A,B,C>::poll::hc9b4a80c9fa6fc5e
   6:     0x55bc8ea6e5ed - futures::future::chain::Chain<A,B,C>::poll::hc42541da1af9f651
   7:     0x55bc8ea13da9 - futures::future::chain::Chain<A,B,C>::poll::h31054d23a94f13f0
   8:     0x55bc8e9332f7 - <futures::stream::fold::Fold<S,F,Fut,T> as futures::future::Future>::poll::h29932901fcadaaa9
   9:     0x55bc8ea634ed - futures::future::chain::Chain<A,B,C>::poll::hb3f812da50abc64e
  10:     0x55bc8e89bec9 - <futures::future::either::Either<A,B> as futures::future::Future>::poll::h8669a0514afdced1
  11:     0x55bc8ea6c882 - futures::future::chain::Chain<A,B,C>::poll::hc068ec2e61b14ac9
  12:     0x55bc8ea7d8c2 - futures::future::chain::Chain<A,B,C>::poll::hec11cc816ff86ecb
  13:     0x55bc8ea49b62 - futures::future::chain::Chain<A,B,C>::poll::h8ff44977d3314539
  14:     0x55bc8e9ad2d9 - <futures::future::loop_fn::LoopFn<A,F> as futures::future::Future>::poll::h6b487f1eb0ef581e
  15:     0x55bc8ea62303 - futures::future::chain::Chain<A,B,C>::poll::hb2d9c58ee76417cc
  16:     0x55bc8e9b651b - <futures::future::map_err::MapErr<A,F> as futures::future::Future>::poll::h852820e2dcf93c0e
  17:     0x55bc8f0cd802 - futures::task_impl::std::set::h1e2c0f463cccad1f
  18:     0x55bc8f0cd462 - std::panicking::try::do_call::hfd59f9795844b37e
  19:     0x55bc8f0f6c1a - __rust_maybe_catch_panic
  20:     0x55bc8f0cb539 - tokio_threadpool::task::Task::run::hdd33ce00a55c22d0
  21:     0x55bc8f0c7d76 - tokio_threadpool::worker::Worker::run_task::h3a0825ded0a3ebf6
  22:     0x55bc8f0c73e8 - tokio_threadpool::worker::Worker::run::h4ca8c7c856069205
  23:     0x55bc8f0a357e - tokio_timer::clock::clock::with_default::h3ebf42f0968bec7b
  24:     0x55bc8f0a8093 - tokio::runtime::threadpool::builder::Builder::build::{{closure}}::h29b84bc8a6aa6432
  25:     0x55bc8f0cd0ca - std::thread::local::LocalKey<T>::with::h1c49b02109ad3d66
  26:     0x55bc8f0cd1e9 - std::thread::local::LocalKey<T>::with::h597a6a28ea622399
  27:     0x55bc8f0c48d8 - std::sys_common::backtrace::__rust_begin_short_backtrace::h6487e40255b569de
  28:     0x55bc8f0cd3bc - std::panicking::try::do_call::h522fd0d86c21226d
  29:     0x55bc8f0f6c1a - __rust_maybe_catch_panic
  30:     0x55bc8f0c5880 - core::ops::function::FnOnce::call_once{{vtable.shim}}::hf52307807037b5df
  31:     0x55bc8f0e277f - <alloc::boxed::Box<F> as core::ops::function::FnOnce<A>>::call_once::h2877b2aefb0fd627
  32:     0x55bc8f0f5f60 - std::sys::unix::thread::Thread::new::thread_start::h382c6ae14181d163
  33:     0x7f47d0e2bef7 - start_thread
  34:     0x7f47d0b4922f - __GI___clone
  35:                0x0 - <unknown>

bug subsys-block Priority - High

Source