Hi,
Validator node stacked at specific height, reboot does not help.
====================
Version: 1.0.0-unknown-x86_64-linux-gnu
0: sp_panic_handler::set::{{closure}}
1: <alloc::boxed::Box<F,A> as core::ops::function::Fn>::call
at rustc/c0983a9aac889d16722a12602ac678051e62c3fb/library/alloc/src/boxed.rs:1952:9
std::panicking::rust_panic_with_hook
at rustc/c0983a9aac889d16722a12602ac678051e62c3fb/library/std/src/panicking.rs:692:13
2: std::panicking::begin_panic_handler::{{closure}}
at rustc/c0983a9aac889d16722a12602ac678051e62c3fb/library/std/src/panicking.rs:579:13
3: std::sys_common::backtrace::__rust_end_short_backtrace
at rustc/c0983a9aac889d16722a12602ac678051e62c3fb/library/std/src/sys_common/backtrace.rs:138:18
4: rust_begin_unwind
at rustc/c0983a9aac889d16722a12602ac678051e62c3fb/library/std/src/panicking.rs:575:5
5: core::panicking::panic_fmt
at rustc/c0983a9aac889d16722a12602ac678051e62c3fb/library/core/src/panicking.rs:65:14
6: core::result::unwrap_failed
at rustc/c0983a9aac889d16722a12602ac678051e62c3fb/library/core/src/result.rs:1791:5
7: <(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize
8: <(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize
9: <(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize
10: <(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize
11: <(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize
12: <(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize
13: <(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize
14: <(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize
15: <(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize
16: <(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize
17: <(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize
18: <(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize
19: <(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize
20: frame_executive::Executive<System,Block,Context,UnsignedValidator,AllPalletsWithSystem,COnRuntimeUpgrade>::initialize_block
21: tracing::span::Span::in_scope
22: frame_executive::Executive<System,Block,Context,UnsignedValidator,AllPalletsWithSystem,COnRuntimeUpgrade>::execute_block
23: <bifrost_testnet_runtime::Runtime as sp_api::runtime_decl_for_Core::Core<sp_runtime::generic::block::Block<sp_runtime::generic::header::Header<u32,sp_runtime::traits::BlakeTwo256>,fp_self_contained::unchecked_extrinsic::UncheckedExtrinsic<<<account::EthereumSignature as sp_runtime::traits::Verify>::Signer as sp_runtime::traits::IdentifyAccount>::AccountId,bifrost_testnet_runtime::Call,account::EthereumSignature,(frame_system::extensions::check_non_zero_sender::CheckNonZeroSender<bifrost_testnet_runtime::Runtime>,frame_system::extensions::check_spec_version::CheckSpecVersion<bifrost_testnet_runtime::Runtime>,frame_system::extensions::check_tx_version::CheckTxVersion<bifrost_testnet_runtime::Runtime>,frame_system::extensions::check_genesis::CheckGenesis<bifrost_testnet_runtime::Runtime>,frame_system::extensions::check_mortality::CheckMortality<bifrost_testnet_runtime::Runtime>,frame_system::extensions::check_nonce::CheckNonce<bifrost_testnet_runtime::Runtime>,frame_system::extensions::check_weight::CheckWeight<bifrost_testnet_runtime::Runtime>,pallet_transaction_payment::ChargeTransactionPayment<bifrost_testnet_runtime::Runtime>)>>>>::execute_block
24: std::panicking::try
25: std:
:local::LocalKey::with
26: sc_executor::native_executor::WasmExecutor::with_instance::{{closure}}
27: sc_executor::wasm_runtime::RuntimeCache::with_instance
28: sp_state_machine::execution::StateMachine<B,H,Exec>::execute_aux
29: sp_state_machine::execution::StateMachine<B,H,Exec>::execute_using_consensus_failure_handler
30: <sc_service::client::call_executor::LocalCallExecutor<Block,B,E> as sc_client_api::call_executor::CallExecutor>::contextual_call
31: <sc_service::client::client::Client<B,E,Block,RA> as sp_api::CallApiAt>::call_api_at
32: sp_api::runtime_decl_for_Core::execute_block_call_api_at
33: <bifrost_testnet_runtime::RuntimeApiImpl<SR_API_BLOCK,RuntimeApiImplCall> as sp_api::Core<SR_API_BLOCK>>::Core_execute_block_runtime_api_impl
34: sp_api::Core::execute_block_with_context
35: sc_service::client::client::Client<B,E,Block,RA>::prepare_block_storage_changes
36: <core::future::from_generator::GenFuture as core::future::future::Future>::poll
37: <core::future::from_generator::GenFuture as core::future::future::Future>::poll
38: <core::future::from_generator::GenFuture as core::future::future::Future>::poll
39: <core::future::from_generator::GenFuture as core::future::future::Future>::poll
40: <core::future::from_generator::GenFuture as core::future::future::Future>::poll
41: <futures_util::future::future:
:Map<Fut,F> as core::future::future::Future>::poll
42: <sc_service::task_manager::prometheus_future::PrometheusFuture as core::future::future::Future>::poll
43: <futures_util::future::select::Select<A,B> as core::future::future::Future>::poll
44: <core::future::from_generator::GenFuture as core::future::future::Future>::poll
45: std::local::LocalKey::with
46: tokio:
:CachedParkThread::block_on
47: tokio::runtime::handle::Handle::block_on
48: <tokio::runtime::blocking::task::BlockingTask as core::future::future::Future>::poll
49: tokio::runtime::task::harness::Harness<T,S>::poll
50: tokio::runtime::blocking::pool::Inner::run
51: std::sys_common::backtrace::__rust_begin_short_backtrace
52: core::ops::function::FnOnce::call_once{{vtable.shim}}
53: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce>::call_once
at rustc/c0983a9aac889d16722a12602ac678051e62c3fb/library/alloc/src/boxed.rs:1938:9
<alloc::boxed::Box<F,A> as core::ops::function::FnOnce>::call_once
at rustc/c0983a9aac889d16722a12602ac678051e62c3fb/library/alloc/src/boxed.rs:1938:9
std::sys::unix:
:thread_start
at rustc/c0983a9aac889d16722a12602ac678051e62c3fb/library/std/src/sys/unix/thread.rs:108:17
54:
55: __clone
Thread ‘tokio-runtime-worker’ panicked at ‘Graceful validator reward auto-compound: Module(ModuleError { index: 10, error: 2, message: Some(“InsufficientBalance”) })’, /bifrost/pallets/bfc-staking/src/pallet/impls.rs:362
This is a bug. Please report it at:
support.anonymous.an
2022-10-24 10:03:58 Block prepare storage changes error: Error at calling runtime api: Execution failed: Execution aborted due to trap: wasm trap: wasm unreachable instruction executed
WASM backtrace:
0: 0x24bfd9 - <unknown>!rust_panic
1: 0x24bfd2 - <unknown>!std::panicking::rust_panic_with_hook::hd20dbb32ccf2960c
2: 0x24bc3d - <unknown>!std::panicking::begin_panic_handler::{{closure}}::h6f71c047f903cff6
3: 0x24bb87 - <unknown>!std::sys_common::backtrace::__rust_end_short_backtrace::hbb6ee37c9cb141a4
4: 0x24bc85 - <unknown>!rust_begin_unwind
5: 0x1b2a32 - <unknown>!core::panicking::panic_fmt::ha6d5b6466ab38e11
6: 0x1b2e05 - <unknown>!core::result::unwrap_failed::h7aa4a6c45486df8b
7: 0xaa870 - <unknown>!<(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize<BlockNumber>>::on_initialize::h93502e603e15f2d8
8: 0x11fe6d - <unknown>!<(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize<BlockNumber>>::on_initialize::had430e6d91ca4c07
9: 0x41118 - <unknown>!<(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize<BlockNumber>>::on_initialize::h58cc151aae952149
10: 0xe7221 - !<(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize::hedbe5eed270208e6
11: 0x182c1c - !frame_executive::Executive<System,Block,Context,UnsignedValidator,AllPalletsWithSystem,COnRuntimeUpgrade>::initialize_block::h8d01681df137b3e8
12: 0x1819c3 - !frame_executive::Executive<System,Block,Context,UnsignedValidator,AllPalletsWithSystem,COnRuntimeUpgrade>::execute_block::h474c089bf8022a80
13: 0x1028ff - !Core_execute_block
2022-10-24 10:03:58 
Error importing block 0xf8be738c253504d542acb1d1708a1d5b4549845d93d5a1a6752757f1e2defaa5: consensus error: Import failed: Import failed: Error at calling runtime api: Execution failed: Execution aborted due to trap: wasm trap: wasm unreachable instruction executed
WASM backtrace:
0: 0x24bfd9 - <unknown>!rust_panic
1: 0x24bfd2 - <unknown>!std::panicking::rust_panic_with_hook::hd20dbb32ccf2960c
2: 0x24bc3d - <unknown>!std::panicking::begin_panic_handler::{{closure}}::h6f71c047f903cff6
3: 0x24bb87 - <unknown>!std::sys_common::backtrace::__rust_end_short_backtrace::hbb6ee37c9cb141a4
4: 0x24bc85 - <unknown>!rust_begin_unwind
5: 0x1b2a32 - <unknown>!core::panicking::panic_fmt::ha6d5b6466ab38e11
6: 0x1b2e05 - <unknown>!core::result::unwrap_failed::h7aa4a6c45486df8b
7: 0xaa870 - <unknown>!<(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize<BlockNumber>>::on_initialize::h93502e603e15f2d8
8: 0x11fe6d - <unknown>!<(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize<BlockNumber>>::on_initialize::had430e6d91ca4c07
9: 0x41118 - <unknown>!<(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize<BlockNumber>>::on_initialize::h58cc151aae952149
10: 0xe7221 - !<(TupleElement0,TupleElement1) as frame_support::traits::hooks::OnInitialize>::on_initialize::hedbe5eed270208e6
11: 0x182c1c - !frame_executive::Executive<System,Block,Context,UnsignedValidator,AllPalletsWithSystem,COnRuntimeUpgrade>::initialize_block::h8d01681df137b3e8
12: 0x1819c3 - !frame_executive::Executive<System,Block,Context,UnsignedValidator,AllPalletsWithSystem,COnRuntimeUpgrade>::execute_block::h474c089bf8022a80
13: 0x1028ff - !Core_execute_block
2022-10-24 10:03:59 
Idle (19 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 
2.4MiB/s 
219.0kiB/s
2022-10-24 10:04:04 Idle (19 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 437.5kiB/s 149.2kiB/s
2022-10-24 10:04:09 Idle (20 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 164.5kiB/s 191.4kiB/s
2022-10-24 10:04:14 Idle (20 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 61.2kiB/s 64.3kiB/s
2022-10-24 10:04:19 Idle (23 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 186.8kiB/s 186.5kiB/s
2022-10-24 10:04:24 Idle (23 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 88.3kiB/s 77.3kiB/s
2022-10-24 10:04:29 Idle (24 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 237.7kiB/s 272.8kiB/s
2022-10-24 10:04:34 Idle (24 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 288.5kiB/s 282.9kiB/s
2022-10-24 10:04:39 Idle (25 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 134.4kiB/s 184.6kiB/s
2022-10-24 10:04:44 Idle (25 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 84.5kiB/s 2.6kiB/s
2022-10-24 10:04:49 Idle (25 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 138.9kiB/s 160.3kiB/s
2022-10-24 10:04:54 Idle (26 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 199.8kiB/s 219.8kiB/s
2022-10-24 10:04:59 Idle (26 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 190.9kiB/s 222.6kiB/s
2022-10-24 10:05:04 Idle (26 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 215.2kiB/s 218.8kiB/s
2022-10-24 10:05:09 Idle (27 peers), best: #336000 (0xd5ad…c5fa), finalized #335998 (0x8459…5f71), 198.5kiB/s 175.8kiB/s