Tokio: Semaphore Option::unwrap() on a None value causes panic
Version
0.2.13
$ cargo tree | grep tokio
โ โโโ tokio v0.2.13
โ โโโ tokio-macros v0.2.5
โ โ โโโ tokio v0.2.13 (*)
โ โ โโโ tokio-util v0.2.0
โ โ โโโ tokio v0.2.13 (*)
โ โโโ tokio v0.2.13 (*)
โ โโโ tokio v0.2.13 (*)
โ โโโ tokio-rustls v0.13.0
โ โ โโโ tokio v0.2.13 (*)
โ โโโ tokio v0.2.13 (*)
โ โโโ tokio-rustls v0.13.0 (*)
โโโ tokio v0.2.13 (*)
โโโ tokio-io-timeout v0.4.0
โ โโโ tokio v0.2.13 (*)
โโโ tokio-rustls v0.13.0 (*)
โ โโโ tokio v0.2.13 (*)
โ โโโ tokio v0.2.13 (*)
Platform
Linux 4.18.0-21-generic
Description
Getting a panic on an Option::unwrap. I believe it's this one: https://github.com/tokio-rs/tokio/blob/master/tokio/src/sync/semaphore_ll.rs#L475
Backtrace
called `Option::unwrap()` on a `None` value
Thread 0
0 core 0xce624d core::panicking::panic (panicking.rs:52)
1 tokio 0xb4da51 [inlined] tokio::sync::semaphore_ll::Semaphore::add_permits_locked2
2 tokio 0xb4da51 tokio::sync::semaphore_ll::Semaphore::add_permits_locked (semaphore_ll.rs:363)
3 core 0x4603e4 [inlined] core::ptr::drop_in_place (mod.rs:174)
4 core 0x4603e4 core::ptr::drop_in_place (mod.rs:174)
5 core 0x470010 [inlined] core::ptr::drop_in_place (mod.rs:174)
6 core 0x470010 [inlined] core::ptr::drop_in_place (mod.rs:174)
7 alloc 0x470010 alloc::sync::Arc<T>::drop_slow (sync.rs:739)
8 alloc 0x632263 [inlined] alloc::sync::Arc<T>::drop (sync.rs:1249)
9 core 0x632263 [inlined] core::ptr::drop_in_place (mod.rs:174)
10 core 0x632263 [inlined] core::ptr::drop_in_place (mod.rs:174)
11 core 0x632263 [inlined] core::ptr::drop_in_place (mod.rs:174)
12 fly_proxy 0x632263 fly_proxy::handlers::utils::transfer::{{closure}} (utils.rs:116)
13 std 0x5ba2ee [inlined] std::future::GenFuture<T>::poll (future.rs:43)
14 std 0x5ba2ee std::future::poll_with_tls_context (future.rs:98)
15 fly_proxy 0x5b62a6 [inlined] fly_proxy::handlers::tcp::local::handle::{{closure}} (local.rs:38)
16 std 0x5b62a6 [inlined] std::future::GenFuture<T>::poll (future.rs:43)
17 std 0x5b62a6 std::future::poll_with_tls_context (future.rs:98)
18 fly_proxy 0x634b94 fly_proxy::tcp::backhaul::handle_accept::{{closure}} (backhaul.rs:81)
19 std 0x604459 std::future::GenFuture<T>::poll (future.rs:43)
20 futures_util 0x4b37ce futures_util::future::future::inspect::Inspect<T>::poll (inspect.rs:42)
21 tokio 0x49de5c tokio::task::core::Core<T>::poll (core.rs:128)
22 tokio 0x6fa258 [inlined] tokio::task::harness::Harness<T>::poll::{{closure}}::{{closure}} (harness.rs:119)
23 core 0x6fa258 [inlined] core::ops::function::FnOnce::call_once (function.rs:232)
24 std 0x6fa258 [inlined] std::panic::AssertUnwindSafe<T>::call_once (panic.rs:318)
25 std 0x6fa258 std::panicking::try::do_call (panicking.rs:305)
26 <unknown> 0xcbbe37 __rust_maybe_catch_panic (lib.rs:86)
27 std 0x42085e [inlined] std::panicking::try (panicking.rs:281)
28 std 0x42085e [inlined] std::panic::catch_unwind (panic.rs:394)
29 tokio 0x42085e [inlined] tokio::task::harness::Harness<T>::poll::{{closure}} (harness.rs:100)
30 tokio 0x42085e [inlined] tokio::loom::std::causal_cell::CausalCell<T>::with_mut (causal_cell.rs:41)
31 tokio 0x42085e tokio::task::harness::Harness<T>::poll (harness.rs:99)
32 tokio 0xb444cf [inlined] tokio::task::raw::RawTask::poll (raw.rs:113)
33 tokio 0xb444cf [inlined] tokio::task::Task<T>::run (mod.rs:381)
34 tokio 0xb444cf tokio::runtime::thread_pool::worker::GenerationGuard::run_task (worker.rs:455)
35 tokio 0xb43b31 [inlined] tokio::runtime::thread_pool::worker::GenerationGuard::process_available_work (worker.rs:313)
36 tokio 0xb43b31 tokio::runtime::thread_pool::worker::GenerationGuard::run (worker.rs:278)
37 tokio 0xb461d2 [inlined] tokio::runtime::thread_pool::worker::Worker::run::{{closure}}::{{closure}} (worker.rs:160)
38 std 0xb461d2 [inlined] std::thread::local::LocalKey<T>::try_with (local.rs:262)
39 std 0xb461d2 [inlined] std::thread::local::LocalKey<T>::with (local.rs:239)
40 tokio 0xb461d2 [inlined] tokio::runtime::thread_pool::worker::Worker::run::{{closure}} (worker.rs:136)
41 tokio 0xb461d2 [inlined] tokio::runtime::thread_pool::current::set::{{closure}} (current.rs:47)
42 std 0xb461d2 [inlined] std::thread::local::LocalKey<T>::try_with (local.rs:262)
43 std 0xb461d2 std::thread::local::LocalKey<T>::with (local.rs:239)
44 tokio 0xb438f7 [inlined] tokio::runtime::thread_pool::current::set (current.rs:29)
45 tokio 0xb438f7 tokio::runtime::thread_pool::worker::Worker::run (worker.rs:132)
46 tokio 0xb5d1fa [inlined] tokio::runtime::thread_pool::Workers::spawn::{{closure}}::{{closure}} (mod.rs:113)
47 tokio 0xb5d1fa [inlined] tokio::runtime::blocking::task::BlockingTask<T>::poll (task.rs:30)
48 tokio 0xb5d1fa tokio::task::core::Core<T>::poll (core.rs:128)
49 tokio 0xb3be63 [inlined] tokio::task::harness::Harness<T>::poll::{{closure}}::{{closure}} (harness.rs:119)
50 core 0xb3be63 [inlined] core::ops::function::FnOnce::call_once (function.rs:232)
51 std 0xb3be63 [inlined] std::panic::AssertUnwindSafe<T>::call_once (panic.rs:318)
52 std 0xb3be63 std::panicking::try::do_call (panicking.rs:305)
53 <unknown> 0xcbbe37 __rust_maybe_catch_panic (lib.rs:86)
54 std 0xb5d904 [inlined] std::panicking::try (panicking.rs:281)
55 std 0xb5d904 [inlined] std::panic::catch_unwind (panic.rs:394)
56 tokio 0xb5d904 [inlined] tokio::task::harness::Harness<T>::poll::{{closure}} (harness.rs:100)
57 tokio 0xb5d904 [inlined] tokio::loom::std::causal_cell::CausalCell<T>::with_mut (causal_cell.rs:41)
58 tokio 0xb5d904 tokio::task::harness::Harness<T>::poll (harness.rs:99)
59 tokio 0xb4ac2f [inlined] tokio::task::raw::RawTask::poll (raw.rs:113)
60 tokio 0xb4ac2f [inlined] tokio::task::Task<T>::run (mod.rs:381)
61 tokio 0xb4ac2f [inlined] tokio::runtime::blocking::pool::run_task (pool.rs:311)
62 tokio 0xb4ac2f tokio::runtime::blocking::pool::Inner::run (pool.rs:230)
63 tokio 0xb471df [inlined] tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}::{{closure}} (pool.rs:210)
64 tokio 0xb471df tokio::runtime::context::enter (context.rs:72)
65 tokio 0xb3df5e [inlined] tokio::runtime::handle::Handle::enter (handle.rs:34)
66 tokio 0xb3df5e [inlined] tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} (pool.rs:209)
67 std 0xb3df5e std::sys_common::backtrace::__rust_begin_short_backtrace (backtrace.rs:129)
68 std 0xb3beb5 [inlined] std::thread::Builder::spawn_unchecked::{{closure}}::{{closure}} (mod.rs:475)
69 std 0xb3beb5 [inlined] std::panic::AssertUnwindSafe<T>::call_once (panic.rs:318)
70 std 0xb3beb5 std::panicking::try::do_call (panicking.rs:305)
71 <unknown> 0xcbbe37 __rust_maybe_catch_panic (lib.rs:86)
72 std 0xb5000a [inlined] std::panicking::try (panicking.rs:281)
73 std 0xb5000a [inlined] std::panic::catch_unwind (panic.rs:394)
74 std 0xb5000a [inlined] std::thread::Builder::spawn_unchecked::{{closure}} (mod.rs:474)
75 core 0xb5000a core::ops::function::FnOnce::call_once{{vtable.shim}} (function.rs:232)
76 alloc 0xca76df alloc::boxed::Box<T>::call_once (boxed.rs:1015)
77 alloc 0xcbb550 [inlined] alloc::boxed::Box<T>::call_once (boxed.rs:1015)
78 std 0xcbb550 [inlined] std::sys_common::thread::start_thread (thread.rs:13)
79 std 0xcbb550 std::sys::unix::thread::Thread::new::thread_start (thread.rs:80)
Some context
I'm using a Lazy (from the once_cell crate) static mut like:
pub static mut SEMAPHORES: Lazy<HashMap<String, Semaphore>> = Lazy::new(|| HashMap::new());
I'm using an actor, processing messages one at a time that creates a new Semaphore in this static HashMap (using unsafe). I figure if it's one at a time, this should be relatively safe. All the String keys are unique.
I'm using an unsafe static mut because I need acquired SemaphorePermits to have a 'static lifetime associated with them. I'm roughly assigning them to TcpStreams so they drop with the connections and allow more permits (and therefore, connections) to be acquired.
I'm never removing or replacing any Semaphore from this static mut HashMap. It's only additive.
Acquiring permits can happen at any time. They can also be dropped at any time.
I assume this panic is probably not a big problem since I'm likely done with the connection, but I think that unwrap could either go away or use an expect or match and log a warning.
jeromegn
All 9 comments
There is a Pr to rewrite the semaphore. Can you see if the issue happens with that? https://github.com/tokio-rs/tokio/pull/2325
carllerche
on 22 Mar 2020
So far so good, but this issue did not happen often. Hard to tell if it's better. I see the code changed significantly (removed and added a few unwraps).
In terms of performance it seems to be very similar.
jeromegn
on 22 Mar 2020
👍1
I see the code changed significantly (removed and added a few unwraps).
@jeromegn FWIW, https://github.com/tokio-rs/tokio/pull/2325/commits/d20a93aba62e80b122ad614a684eaf40029ec2e6 changes the one Option::unwrap in the new semaphore to an expectas you suggested in the issue โ all the remaining unwraps are on Mutex::lock calls that will fail if a lock is poisoned, and in that case, there's not a whole lot of additional context an expect can add.
hawkw
on 22 Mar 2020
Thanks! I didnโt look too closely at all the code TBH, it looked good
though.
I think this expect will help if it ever happens again :)
On Sun, Mar 22, 2020 at 16:41 Eliza Weisman notifications@github.com
wrote:
I see the code changed significantly (removed and added a few unwraps).
@jeromegn https://github.com/jeromegn FWIW, d20a93a
https://github.com/tokio-rs/tokio/commit/d20a93aba62e80b122ad614a684eaf40029ec2e6
changes the one Option::unwrap in the new semaphore to an expectas you
suggested in the issue โ all the remaining unwraps are on Mutex::lock
calls that will fail if a lock is poisoned, and in that case, there's not a
whole lot of additional context an expect can add.โ
You are receiving this because you were mentioned.Reply to this email directly, view it on GitHub
https://github.com/tokio-rs/tokio/issues/2336#issuecomment-602269835,
or unsubscribe
https://github.com/notifications/unsubscribe-auth/AAAKSPIEOG3O2HAJP3DLVMDRIZZV3ANCNFSM4LRIPQQQ
.
jeromegn
on 22 Mar 2020
❤1
Just got the following poisoned lock error! That's clearer now at least :) Not sure what I might be doing wrong though:
called `Result::unwrap()` on an `Err` value: "PoisonError { inner: .. }"
Thread 0
0 core 0xce2213 core::option::expect_none_failed (option.rs:1199)
1 core 0xb40ad7 [inlined] core::result::Result<T>::unwrap (result.rs:963)
2 tokio 0xb40ad7 tokio::sync::batch_semaphore::Semaphore::release (batch_semaphore.rs:125)
3 core 0x705424 [inlined] core::ptr::drop_in_place (mod.rs:174)
4 core 0x705424 [inlined] core::ptr::drop_in_place (mod.rs:174)
5 core 0x705424 core::ptr::drop_in_place (mod.rs:174)
6 core 0x71a6a0 [inlined] core::ptr::drop_in_place (mod.rs:174)
7 core 0x71a6a0 [inlined] core::ptr::drop_in_place (mod.rs:174)
8 alloc 0x71a6a0 alloc::sync::Arc<T>::drop_slow (sync.rs:739)
9 alloc 0x60a343 [inlined] alloc::sync::Arc<T>::drop (sync.rs:1249)
10 core 0x60a343 [inlined] core::ptr::drop_in_place (mod.rs:174)
11 core 0x60a343 [inlined] core::ptr::drop_in_place (mod.rs:174)
12 core 0x60a343 [inlined] core::ptr::drop_in_place (mod.rs:174)
13 fly_proxy 0x60a343 [inlined] fly_proxy::handlers::utils::transfer::{{closure}} (utils.rs:116)
14 std 0x60a343 std::future::GenFuture<T>::poll (future.rs:43)
15 std 0x5c0cd0 [inlined] std::future::poll_with_tls_context (future.rs:98)
16 fly_proxy 0x5c0cd0 [inlined] fly_proxy::handlers::tcp::local::handle::{{closure}} (local.rs:38)
17 std 0x5c0cd0 [inlined] std::future::GenFuture<T>::poll (future.rs:43)
18 std 0x5c0cd0 std::future::poll_with_tls_context (future.rs:98)
19 fly_proxy 0x642856 fly_proxy::tcp::backhaul::handle_accept::{{closure}} (backhaul.rs:81)
20 std 0x614129 std::future::GenFuture<T>::poll (future.rs:43)
21 futures_util 0x458ebe futures_util::future::future::inspect::Inspect<T>::poll (inspect.rs:42)
22 tokio 0x5183f0 [inlined] tokio::runtime::task::core::Core<T>::poll::{{closure}}::{{closure}} (core.rs:164)
23 tokio 0x5183f0 [inlined] tokio::coop::budget::{{closure}} (coop.rs:97)
24 std 0x5183f0 [inlined] std::thread::local::LocalKey<T>::try_with (local.rs:262)
25 std 0x5183f0 std::thread::local::LocalKey<T>::with (local.rs:239)
26 tokio 0x557454 [inlined] tokio::coop::budget (coop.rs:79)
27 tokio 0x557454 [inlined] tokio::runtime::task::core::Core<T>::poll::{{closure}} (core.rs:164)
28 tokio 0x557454 [inlined] tokio::loom::std::causal_cell::CausalCell<T>::with_mut (causal_cell.rs:32)
29 tokio 0x557454 tokio::runtime::task::core::Core<T>::poll (core.rs:149)
30 tokio 0x5a965f [inlined] tokio::runtime::task::harness::Harness<T>::poll::{{closure}} (harness.rs:108)
31 core 0x5a965f [inlined] core::ops::function::FnOnce::call_once (function.rs:232)
32 std 0x5a965f std::panic::AssertUnwindSafe<T>::call_once (panic.rs:318)
33 std 0x6c1ae9 std::panicking::try::do_call (panicking.rs:305)
34 <unknown> 0xcb7f27 __rust_maybe_catch_panic (lib.rs:86)
35 std 0x442c89 [inlined] std::panicking::try (panicking.rs:281)
36 std 0x442c89 [inlined] std::panic::catch_unwind (panic.rs:394)
37 tokio 0x442c89 tokio::runtime::task::harness::Harness<T>::poll (harness.rs:84)
38 tokio 0xb54710 [inlined] tokio::runtime::task::raw::RawTask::poll (raw.rs:66)
39 tokio 0xb54710 [inlined] tokio::runtime::task::Notified<T>::run (mod.rs:169)
40 tokio 0xb54710 tokio::runtime::thread_pool::worker::Context::run_task (worker.rs:299)
41 tokio 0xb53c9e tokio::runtime::thread_pool::worker::Context::run (worker.rs:278)
42 tokio 0xb35b53 [inlined] tokio::runtime::thread_pool::worker::run::{{closure}} (worker.rs:256)
43 tokio 0xb35b53 tokio::macros::scoped_tls::ScopedKey<T>::set (scoped_tls.rs:64)
44 tokio 0xb5385b tokio::runtime::thread_pool::worker::run (worker.rs:253)
45 tokio 0xb44318 [inlined] tokio::runtime::thread_pool::worker::Launch::launch::{{closure}} (worker.rs:232)
46 tokio 0xb44318 [inlined] tokio::runtime::blocking::task::BlockingTask<T>::poll (task.rs:38)
47 tokio 0xb44318 [inlined] tokio::runtime::task::core::Core<T>::poll::{{closure}}::{{closure}} (core.rs:164)
48 tokio 0xb44318 [inlined] tokio::coop::budget::{{closure}} (coop.rs:97)
49 std 0xb44318 [inlined] std::thread::local::LocalKey<T>::try_with (local.rs:262)
50 std 0xb44318 std::thread::local::LocalKey<T>::with (local.rs:239)
51 tokio 0xb4f8a1 [inlined] tokio::coop::budget (coop.rs:79)
52 tokio 0xb4f8a1 [inlined] tokio::runtime::task::core::Core<T>::poll::{{closure}} (core.rs:164)
53 tokio 0xb4f8a1 [inlined] tokio::loom::std::causal_cell::CausalCell<T>::with_mut (causal_cell.rs:32)
54 tokio 0xb4f8a1 [inlined] tokio::runtime::task::core::Core<T>::poll (core.rs:149)
55 tokio 0xb4f8a1 [inlined] tokio::runtime::task::harness::Harness<T>::poll::{{closure}} (harness.rs:108)
56 core 0xb4f8a1 [inlined] core::ops::function::FnOnce::call_once (function.rs:232)
57 std 0xb4f8a1 std::panic::AssertUnwindSafe<T>::call_once (panic.rs:318)
58 std 0xb43139 std::panicking::try::do_call (panicking.rs:305)
59 <unknown> 0xcb7f27 __rust_maybe_catch_panic (lib.rs:86)
60 std 0xb3a7de [inlined] std::panicking::try (panicking.rs:281)
61 std 0xb3a7de [inlined] std::panic::catch_unwind (panic.rs:394)
62 tokio 0xb3a7de tokio::runtime::task::harness::Harness<T>::poll (harness.rs:84)
63 tokio 0xb38784 [inlined] tokio::runtime::task::raw::RawTask::poll (raw.rs:66)
64 tokio 0xb38784 [inlined] tokio::runtime::task::Notified<T>::run (mod.rs:169)
65 tokio 0xb38784 tokio::runtime::blocking::pool::Inner::run (pool.rs:230)
66 tokio 0xb43c58 [inlined] tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}}::{{closure}} (pool.rs:210)
67 tokio 0xb43c58 tokio::runtime::context::enter (context.rs:72)
68 tokio 0xb3d5af [inlined] tokio::runtime::handle::Handle::enter (handle.rs:39)
69 tokio 0xb3d5af [inlined] tokio::runtime::blocking::pool::Spawner::spawn_thread::{{closure}} (pool.rs:209)
70 std 0xb3d5af std::sys_common::backtrace::__rust_begin_short_backtrace (backtrace.rs:129)
71 std 0xb43075 [inlined] std::thread::Builder::spawn_unchecked::{{closure}}::{{closure}} (mod.rs:475)
72 std 0xb43075 [inlined] std::panic::AssertUnwindSafe<T>::call_once (panic.rs:318)
73 std 0xb43075 std::panicking::try::do_call (panicking.rs:305)
74 <unknown> 0xcb7f27 __rust_maybe_catch_panic (lib.rs:86)
75 std 0xb4c74a [inlined] std::panicking::try (panicking.rs:281)
76 std 0xb4c74a [inlined] std::panic::catch_unwind (panic.rs:394)
77 std 0xb4c74a [inlined] std::thread::Builder::spawn_unchecked::{{closure}} (mod.rs:474)
78 core 0xb4c74a core::ops::function::FnOnce::call_once{{vtable.shim}} (function.rs:232)
79 alloc 0xca37cf alloc::boxed::Box<T>::call_once (boxed.rs:1015)
80 alloc 0xcb7640 [inlined] alloc::boxed::Box<T>::call_once (boxed.rs:1015)
81 std 0xcb7640 [inlined] std::sys_common::thread::start_thread (thread.rs:13)
82 std 0xcb7640 std::sys::unix::thread::Thread::new::thread_start (thread.rs:80)
And this segfault:
* thread #1, name = 'fly-proxy', stop reason = signal SIGSEGV
* frame #0: 0x0000000000cd7b87 fly-proxy`__pthread_mutex_lock
frame #1: 0x0000000000b40921 fly-proxy`tokio::sync::batch_semaphore::Semaphore::release::h710209b6c526c8a3 [inlined] std::sys::unix::mutex::Mutex::lock::h54bf18c46e1ef510(self=<unavailable>) at mutex.rs:57:16
frame #2: 0x0000000000b4091b fly-proxy`tokio::sync::batch_semaphore::Semaphore::release::h710209b6c526c8a3 [inlined] std::sys_common::mutex::Mutex::raw_lock::h1644f33eb71dbc02 at mutex.rs:41
frame #3: 0x0000000000b4091b fly-proxy`tokio::sync::batch_semaphore::Semaphore::release::h710209b6c526c8a3 [inlined] std::sync::mutex::Mutex$LT$T$GT$::lock::hda6ec7ceda6077d6(self=0x000000000277b738) at mutex.rs:218
frame #4: 0x0000000000b40918 fly-proxy`tokio::sync::batch_semaphore::Semaphore::release::h710209b6c526c8a3(self=0x000000000277b738, added=1) at batch_semaphore.rs:125
frame #5: 0x0000000000705424 fly-proxy`core::ptr::drop_in_place::h411945349488f137 [inlined] core::ptr::drop_in_place::hd6157251532c8d57((null)=<unavailable>) at mod.rs:174
frame #6: 0x000000000070541e fly-proxy`core::ptr::drop_in_place::h411945349488f137 [inlined] core::ptr::drop_in_place::hff0e4fee50f317a3((null)=<unavailable>) at mod.rs:174
frame #7: 0x000000000070541e fly-proxy`core::ptr::drop_in_place::h411945349488f137((null)=0x0000000012443fd0) at mod.rs:174
frame #8: 0x000000000071a6a0 fly-proxy`alloc::sync::Arc$LT$T$GT$::drop_slow::h1c9bf917a417faf5 [inlined] core::ptr::drop_in_place::h4e5eae5e71317d20((null)=<unavailable>) at mod.rs:174
frame #9: 0x000000000071a69b fly-proxy`alloc::sync::Arc$LT$T$GT$::drop_slow::h1c9bf917a417faf5 [inlined] core::ptr::drop_in_place::h917f9a9fd85390ec((null)=<unavailable>) at mod.rs:174
frame #10: 0x000000000071a69b fly-proxy`alloc::sync::Arc$LT$T$GT$::drop_slow::h1c9bf917a417faf5(self=0x0000000013a3b4b8) at sync.rs:739
frame #11: 0x000000000060a343 fly-proxy`_$LT$std..future..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h8cbdf22d66b57469 [inlined] _$LT$alloc..sync..Arc$LT$T$GT$$u20$as$u20$core..ops..drop..Drop$GT$::drop::h15a93354651d76f0(self=<unavailable>) at sync.rs:1249:12
frame #12: 0x000000000060a33e fly-proxy`_$LT$std..future..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h8cbdf22d66b57469 [inlined] core::ptr::drop_in_place::hda681aa425605120((null)=<unavailable>) at mod.rs:174
frame #13: 0x000000000060a33e fly-proxy`_$LT$std..future..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h8cbdf22d66b57469 [inlined] core::ptr::drop_in_place::ha912d58f8f31eaa7((null)=<unavailable>) at mod.rs:174
frame #14: 0x000000000060a33e fly-proxy`_$LT$std..future..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h8cbdf22d66b57469 [inlined] core::ptr::drop_in_place::h3b4f4277042ffe2e((null)=<unavailable>) at mod.rs:174
frame #15: 0x000000000060a33e fly-proxy`_$LT$std..future..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h8cbdf22d66b57469 at utils.rs:116
frame #16: 0x0000000000609364 fly-proxy`_$LT$std..future..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h8cbdf22d66b57469(self=Pin<&mut std::future::GenFuture<generator-0>> @ r12, cx=<unavailable>) at future.rs:43
frame #17: 0x00000000005c0cd0 fly-proxy`std::future::poll_with_tls_context::hbbe126cd3c0ac61c at future.rs:98:13
frame #18: 0x00000000005c0c96 fly-proxy`std::future::poll_with_tls_context::hbbe126cd3c0ac61c at local.rs:38
frame #19: 0x00000000005c00ef fly-proxy`std::future::poll_with_tls_context::hbbe126cd3c0ac61c at future.rs:43
frame #20: 0x00000000005c00e4 fly-proxy`std::future::poll_with_tls_context::hbbe126cd3c0ac61c(f=Pin<&mut std::future::GenFuture<generator-0>> @ r13) at future.rs:98
frame #21: 0x0000000000642856 fly-proxy`fly_proxy::tcp::backhaul::handle_accept::_$u7b$$u7b$closure$u7d$$u7d$::h31d265bcc05d8893 at backhaul.rs:81:12
frame #22: 0x0000000000614129 fly-proxy`_$LT$std..future..GenFuture$LT$T$GT$$u20$as$u20$core..future..future..Future$GT$::poll::ha635e976c6b2bbbc(self=<unavailable>, cx=<unavailable>) at future.rs:43:14
frame #23: 0x0000000000458ebe fly-proxy`_$LT$futures_util..future..future..inspect..Inspect$LT$Fut$C$F$GT$$u20$as$u20$core..future..future..Future$GT$::poll::h4ea9e03f9e53deea(self=Pin<&mut futures_util::future::future::inspect::Inspect<std::future::GenFuture<generator-0>, closure-2>> @ rbx, cx=<unavailable>) at inspect.rs:42:23
frame #24: 0x00000000005183f0 fly-proxy`std::thread::local::LocalKey$LT$T$GT$::with::h2e5a950e14f31047 [inlined] tokio::runtime::task::core::Core$LT$T$C$S$GT$::poll::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::heee169d4611f6f68 at core.rs:164:39
frame #25: 0x00000000005183e5 fly-proxy`std::thread::local::LocalKey$LT$T$GT$::with::h2e5a950e14f31047 at coop.rs:97
frame #26: 0x00000000005183bd fly-proxy`std::thread::local::LocalKey$LT$T$GT$::with::h2e5a950e14f31047 at local.rs:262
frame #27: 0x00000000005183b2 fly-proxy`std::thread::local::LocalKey$LT$T$GT$::with::h2e5a950e14f31047(self=<unavailable>, f=closure-0 @ 0x000055a08b569be0) at local.rs:239
frame #28: 0x0000000000557454 fly-proxy`tokio::runtime::task::core::Core$LT$T$C$S$GT$::poll::hc16427bcfa91b43d [inlined] tokio::coop::budget::h31d7cb82dc6d120e(f=closure-0 @ 0x000055a08b6a5040) at coop.rs:79:4
frame #29: 0x000000000055743b fly-proxy`tokio::runtime::task::core::Core$LT$T$C$S$GT$::poll::hc16427bcfa91b43d at core.rs:164
frame #30: 0x00000000005573f0 fly-proxy`tokio::runtime::task::core::Core$LT$T$C$S$GT$::poll::hc16427bcfa91b43d [inlined] tokio::loom::std::causal_cell::CausalCell$LT$T$GT$::with_mut::hc607228aa0807b3b(self=<unavailable>, f=<unavailable>) at causal_cell.rs:32
frame #31: 0x00000000005573f0 fly-proxy`tokio::runtime::task::core::Core$LT$T$C$S$GT$::poll::hc16427bcfa91b43d(self=0x0000000013a38f70, header=<unavailable>) at core.rs:149
frame #32: 0x00000000005a965f fly-proxy`_$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::hb827205bda5e68d5 at harness.rs:108:26
frame #33: 0x00000000005a95c5 fly-proxy`_$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::hb827205bda5e68d5 [inlined] core::ops::function::FnOnce::call_once::h4748731cf9fa894e at function.rs:232
frame #34: 0x00000000005a95c5 fly-proxy`_$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::hb827205bda5e68d5(self=<unavailable>, _args=<unavailable>) at panic.rs:318
frame #35: 0x00000000006c1ae9 fly-proxy`std::panicking::try::do_call::hdb99d79343488281(data="ศฌ\x0f\\n\x7f") at panicking.rs:305:39
frame #36: 0x0000000000cb7f27 fly-proxy`__rust_maybe_catch_panic at lib.rs:86:7
frame #37: 0x0000000000442c89 fly-proxy`tokio::runtime::task::harness::Harness$LT$T$C$S$GT$::poll::h6dbca89b9134c442 at panicking.rs:281:12
frame #38: 0x0000000000442c47 fly-proxy`tokio::runtime::task::harness::Harness$LT$T$C$S$GT$::poll::h6dbca89b9134c442 [inlined] std::panic::catch_unwind::hb0972889d4c0d5df(f=AssertUnwindSafe<closure-0> @ 0x000055a08b596e80) at panic.rs:394
frame #39: 0x0000000000442c47 fly-proxy`tokio::runtime::task::harness::Harness$LT$T$C$S$GT$::poll::h6dbca89b9134c442(self=Harness<futures_util::future::future::inspect::Inspect<std::future::GenFuture<generator-0>, closure-2>, alloc::sync::Arc<tokio::runtime::thread_pool::worker::Worker>> @ 0x00007f6e5c0facc8) at harness.rs:84
frame #40: 0x0000000000b54710 fly-proxy`tokio::runtime::thread_pool::worker::Context::run_task::hb146772e6c97ad2f [inlined] tokio::runtime::task::raw::RawTask::poll::h1d636a4c275db238(self=<unavailable>) at raw.rs:66:17
frame #41: 0x0000000000b5470a fly-proxy`tokio::runtime::thread_pool::worker::Context::run_task::hb146772e6c97ad2f [inlined] tokio::runtime::task::Notified$LT$S$GT$::run::h5748e3852f635f1f(self=<unavailable>) at mod.rs:169
frame #42: 0x0000000000b5470a fly-proxy`tokio::runtime::thread_pool::worker::Context::run_task::hb146772e6c97ad2f(self=<unavailable>, task=<unavailable>, core=0x0000000001f71400) at worker.rs:299
frame #43: 0x0000000000b53c65 fly-proxy`tokio::runtime::thread_pool::worker::Context::run::h030d8a73566fa069(self=0x00007f6e5c0faf20, core=0x0000000001f71400) at worker.rs:271:23
frame #44: 0x0000000000b35b53 fly-proxy`tokio::macros::scoped_tls::ScopedKey$LT$T$GT$::set::h1b55c182a40e3acd [inlined] tokio::runtime::thread_pool::worker::run::_$u7b$$u7b$closure$u7d$$u7d$::hb7e8e385aac350ed at worker.rs:256:16
frame #45: 0x0000000000b35b48 fly-proxy`tokio::macros::scoped_tls::ScopedKey$LT$T$GT$::set::h1b55c182a40e3acd(self=<unavailable>, t=0x00007f6e5c0faf20, f=<unavailable>) at scoped_tls.rs:64
frame #46: 0x0000000000b5385b fly-proxy`tokio::runtime::thread_pool::worker::run::hae4f235356504d46(worker=<unavailable>) at worker.rs:253:4
frame #47: 0x0000000000b44318 fly-proxy`std::thread::local::LocalKey$LT$T$GT$::with::had63485ba879b7b2 [inlined] tokio::runtime::thread_pool::worker::Launch::launch::_$u7b$$u7b$closure$u7d$$u7d$::hb290cd145de12d2a at worker.rs:232:44
frame #48: 0x0000000000b44312 fly-proxy`std::thread::local::LocalKey$LT$T$GT$::with::had63485ba879b7b2 at task.rs:38
frame #49: 0x0000000000b442da fly-proxy`std::thread::local::LocalKey$LT$T$GT$::with::had63485ba879b7b2 [inlined] tokio::runtime::task::core::Core$LT$T$C$S$GT$::poll::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::h5f079d8a806627f8 at core.rs:164
frame #50: 0x0000000000b442da fly-proxy`std::thread::local::LocalKey$LT$T$GT$::with::had63485ba879b7b2 at coop.rs:97
frame #51: 0x0000000000b44288 fly-proxy`std::thread::local::LocalKey$LT$T$GT$::with::had63485ba879b7b2 at local.rs:262
frame #52: 0x0000000000b4427a fly-proxy`std::thread::local::LocalKey$LT$T$GT$::with::had63485ba879b7b2(self=<unavailable>, f=closure-0 @ 0x000055a08b64f8d0) at local.rs:239
frame #53: 0x0000000000b4f8a1 fly-proxy`_$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h2c521fc41c022a81 [inlined] tokio::coop::budget::h045ee7286fca1499(f=<unavailable>) at coop.rs:79:4
frame #54: 0x0000000000b4f88f fly-proxy`_$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h2c521fc41c022a81 at core.rs:164
frame #55: 0x0000000000b4f862 fly-proxy`_$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h2c521fc41c022a81 [inlined] tokio::loom::std::causal_cell::CausalCell$LT$T$GT$::with_mut::h479c270800da5214(self=<unavailable>, f=<unavailable>) at causal_cell.rs:32
frame #56: 0x0000000000b4f862 fly-proxy`_$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h2c521fc41c022a81 [inlined] tokio::runtime::task::core::Core$LT$T$C$S$GT$::poll::h893248a0097bff26(self=0x0000000001f728d0, header=<unavailable>) at core.rs:149
frame #57: 0x0000000000b4f862 fly-proxy`_$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h2c521fc41c022a81 at harness.rs:108
frame #58: 0x0000000000b4f84c fly-proxy`_$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h2c521fc41c022a81 [inlined] core::ops::function::FnOnce::call_once::h3752923893a90d35 at function.rs:232
frame #59: 0x0000000000b4f84c fly-proxy`_$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::h2c521fc41c022a81(self=<unavailable>, _args=<unavailable>) at panic.rs:318
frame #60: 0x0000000000b43139 fly-proxy`std::panicking::try::do_call::h28f86723edec11eb(data="p\xb0\x0f\\n\x7f") at panicking.rs:305:39
frame #61: 0x0000000000cb7f27 fly-proxy`__rust_maybe_catch_panic at lib.rs:86:7
frame #62: 0x0000000000b3a7de fly-proxy`tokio::runtime::task::harness::Harness$LT$T$C$S$GT$::poll::h23ca5bfe4238bb3b at panicking.rs:281:12
frame #63: 0x0000000000b3a79e fly-proxy`tokio::runtime::task::harness::Harness$LT$T$C$S$GT$::poll::h23ca5bfe4238bb3b [inlined] std::panic::catch_unwind::he327a9c2984843c6(f=AssertUnwindSafe<closure-0> @ 0x000055a08ba74480) at panic.rs:394
frame #64: 0x0000000000b3a79e fly-proxy`tokio::runtime::task::harness::Harness$LT$T$C$S$GT$::poll::h23ca5bfe4238bb3b(self=Harness<tokio::runtime::blocking::task::BlockingTask<closure-0>, tokio::runtime::blocking::schedule::NoopSchedule> @ 0x00007f6e5c0fb070) at harness.rs:84
frame #65: 0x0000000000b38784 fly-proxy`tokio::runtime::blocking::pool::Inner::run::hfa30a529ed30807b [inlined] tokio::runtime::task::raw::RawTask::poll::h1d636a4c275db238(self=<unavailable>) at raw.rs:66:17
frame #66: 0x0000000000b3877b fly-proxy`tokio::runtime::blocking::pool::Inner::run::hfa30a529ed30807b [inlined] tokio::runtime::task::Notified$LT$S$GT$::run::h42a39d2ec6bd9ede(self=<unavailable>) at mod.rs:169
frame #67: 0x0000000000b3877b fly-proxy`tokio::runtime::blocking::pool::Inner::run::hfa30a529ed30807b(self=0x0000000001f72250) at pool.rs:230
frame #68: 0x0000000000b43c58 fly-proxy`tokio::runtime::context::enter::h2544655a56ddac74 [inlined] tokio::runtime::blocking::pool::Spawner::spawn_thread::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::h67336a04ae833b88 at pool.rs:210:20
frame #69: 0x0000000000b43c4b fly-proxy`tokio::runtime::context::enter::h2544655a56ddac74(new=<unavailable>, f=<unavailable>) at context.rs:72
frame #70: 0x0000000000b3d5af fly-proxy`std::sys_common::backtrace::__rust_begin_short_backtrace::hd559b6fd86b78946 [inlined] tokio::runtime::handle::Handle::enter::hec1500daef07d722(self=<unavailable>, f=<unavailable>) at handle.rs:39:8
frame #71: 0x0000000000b3d597 fly-proxy`std::sys_common::backtrace::__rust_begin_short_backtrace::hd559b6fd86b78946 at pool.rs:209
frame #72: 0x0000000000b3d57f fly-proxy`std::sys_common::backtrace::__rust_begin_short_backtrace::hd559b6fd86b78946(f=<unavailable>) at backtrace.rs:129
frame #73: 0x0000000000b43075 fly-proxy`std::panicking::try::do_call::h0ef40cf1fba3d8aa [inlined] std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::_$u7b$$u7b$closure$u7d$$u7d$::h713f12a2826a9d7a at mod.rs:475:16
frame #74: 0x0000000000b43054 fly-proxy`std::panicking::try::do_call::h0ef40cf1fba3d8aa [inlined] _$LT$std..panic..AssertUnwindSafe$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$$LP$$RP$$GT$$GT$::call_once::hfab282d81ee3de4d at panic.rs:318
frame #75: 0x0000000000b43054 fly-proxy`std::panicking::try::do_call::h0ef40cf1fba3d8aa(data=<unavailable>) at panicking.rs:305
frame #76: 0x0000000000cb7f27 fly-proxy`__rust_maybe_catch_panic at lib.rs:86:7
frame #77: 0x0000000000b4c74a fly-proxy`core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h7d53a2daab12dddb at panicking.rs:281:12
frame #78: 0x0000000000b4c701 fly-proxy`core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h7d53a2daab12dddb [inlined] std::panic::catch_unwind::hdab643d3faa6443f at panic.rs:394
frame #79: 0x0000000000b4c701 fly-proxy`core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h7d53a2daab12dddb [inlined] std::thread::Builder::spawn_unchecked::_$u7b$$u7b$closure$u7d$$u7d$::h18e5f0c8b2104fc0 at mod.rs:474
frame #80: 0x0000000000b4c6d0 fly-proxy`core::ops::function::FnOnce::call_once$u7b$$u7b$vtable.shim$u7d$$u7d$::h7d53a2daab12dddb((null)=0x00007f6e5c0fb3a0, (null)=<unavailable>) at function.rs:232
frame #81: 0x0000000000ca37cf fly-proxy`_$LT$alloc..boxed..Box$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$::call_once::h5c209cc2f51c734a at boxed.rs:1015:8
frame #82: 0x0000000000cb7640 fly-proxy`std::sys::unix::thread::Thread::new::thread_start::h0344e3a791d8fe52 [inlined] _$LT$alloc..boxed..Box$LT$F$GT$$u20$as$u20$core..ops..function..FnOnce$LT$A$GT$$GT$::call_once::ha982e2070915d4a3 at boxed.rs:1015:8
frame #83: 0x0000000000cb7634 fly-proxy`std::sys::unix::thread::Thread::new::thread_start::h0344e3a791d8fe52 [inlined] std::sys_common::thread::start_thread::hc8a38454df69b8b9 at thread.rs:13
frame #84: 0x0000000000cb75ba fly-proxy`std::sys::unix::thread::Thread::new::thread_start::h0344e3a791d8fe52 at thread.rs:80
frame #85: 0x0000000000cd7388 fly-proxy`start + 147
frame #86: 0x0000000000cd837f fly-proxy`__clone + 47
I bet that's a problem:
(lldb) f 5
frame #5: 0x0000000000705424 fly-proxy`core::ptr::drop_in_place::h411945349488f137 [inlined] core::ptr::drop_in_place::hd6157251532c8d57((null)=<unavailable>) at mod.rs:174
(lldb) fr v
(tokio::sync::semaphore::SemaphorePermit *) = <variable not available>
It's worth reminding you I'm using an unsafe static mut to store my Semaphores! Which is probably the problem here.
jeromegn
on 23 Mar 2020
@jeromegn do you have the backtrace from the thread that was holding the lock when it was poisoned?
hawkw
on 23 Mar 2020
Pretty sure now that this was all due to my unsafe code. Don't mind me!
I refactored to make it all safe and it's been handling all the permit acquisitions smoothly and fast for the last 6 hours.
:+1:
jeromegn
on 23 Mar 2020
👍1
Was this page helpful?
0 / 5 - 0 ratings
Related issues
hawkw
ยท
5Comments
djc
ยท
4Comments
abonander
ยท
5Comments
zhuxiujia
ยท
3Comments
carllerche
ยท
4Comments