crash on rollback about aeplugin_dev_mode HOT 5 CLOSED

contract-aeproject_node-1      | 11:57:41.350 [debug] init: GCHeight = 3
contract-aeproject_node-1      | 11:57:41.350 [error] gen_server aeplugin_dev_mode_emitter terminated with reason: {{{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,...}]},...},...} in aec_conductor:consensus_request/1 line 217
contract-aeproject_node-1      | 11:57:41.350 [error] CRASH REPORT Process aeplugin_dev_mode_emitter with 0 neighbours crashed with reason: {{{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,...}]},...},...} in aec_conductor:consensus_request/1 line 217
contract-aeproject_node-1      | 11:57:41.350 [error] CRASH REPORT Process <0.2106.0> with 0 neighbours exited with reason: {{{{{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,...}]},...},...},...},...} in gen_server:call/2 line 215 in gen_server:call/2 line 215
contract-aeproject_node-1      | 11:57:41.350 [error] Supervisor aeplugin_dev_mode_sup had child aeplugin_dev_mode_emitter started with aeplugin_dev_mode_emitter:start_link() at <0.1713.0> exit with reason {{{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,...}]},...},...} in aec_conductor:consensus_request/1 line 217 in context child_terminated
contract-aeproject_node-1      | 11:57:41.350 [error] Cowboy stream 1 with ranch listener devmode_listener and connection process <0.2105.0> had its request process exit with reason: {{{{{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,...}]},...},...},...},...} in gen_server:call/2 line 215

test1-aeproject_node-1      | 11:08:02.056 [debug] Req = #{bindings => #{},body_length => 0,cert => undefined,has_body => false,headers => #{<<"connection">> => <<"close">>,<<"host">> => <<"node:3313">>},host => <<"node">>,host_info => undefined,method => <<"GET">>,path => <<"/emit_kb">>,path_info => undefined,peer => {{172,20,0,3},40648},pid => <0.2258.0>,port => 3313,qs => <<"n=1">>,ref => devmode_listener,scheme => <<"http">>,sock => {{172,20,0,4},3313},streamid => 1,version => 'HTTP/1.0'}, Opts = []
test1-aeproject_node-1      | 11:08:02.057 [debug] insert_block -> {ok,true,{key_header,3,<<38,250,95,12,189,196,174,108,101,51,198,203,168,72,180,132,154,5,195,215,183,71,131,110,101,251,255,113,154,226,159,5>>,<<17,158,166,69,231,14,255,223,99,151,11,147,253,196,156,128,62,15,4,96,152,97,135,223,85,15,120,32,12,29,16,89>>,<<151,148,195,127,249,149,160,163,250,184,28,63,211,120,148,255,101,205,2,11,1,128,71,78,44,16,25,176,40,124,227,195>>,1338,0,1658747270836,5,no_value,<<184,173,206,115,90,87,82,169,197,169,242,251,158,58,175,61,87,40,20,155,116,185,64,191,187,173,124,51,80,7,44,8>>,<<55,239,123,83,136,117,193,86,122,60,31,91,246,65,232,223,183,40,98,14,4,83,38,133,228,37,233,151,193,29,10,84>>,<<0,0,2,140>>,#{consensus => aec_consensus_on_demand}},[]}
test1-aeproject_node-1      | 11:08:02.058 [error] gen_server aec_tx_pool terminated with reason: no match of right hand value {error,unknown_hash} in aec_tx_pool:do_top_change/2 line 820
test1-aeproject_node-1      | 11:08:02.058 [error] CRASH REPORT Process aec_tx_pool with 0 neighbours crashed with reason: no match of right hand value {error,unknown_hash} in aec_tx_pool:do_top_change/2 line 820
test1-aeproject_node-1      | 11:08:02.058 [error] Supervisor aecore_sup had child aec_tx_pool started with aec_tx_pool:start_link() at <0.1621.0> exit with reason no match of right hand value {error,unknown_hash} in aec_tx_pool:do_top_change/2 line 820 in context child_terminated
test1-aeproject_node-1      | 11:08:02.058 [debug] Reason = {{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,249}]}]},{gen_server,call,[aec_tx_pool,{top_change,#{new_hash => <<154,109,4,4,220,221,9,142,213,126,94,255,181,9,116,150,116,173,105,73,118,113,92,19,150,28,54,174,231,193,48,133>>,new_height => 4,old_hash => <<114,11,12,255,216,72,234,169,203,147,135,212,96,32,154,178,112,194,92,221,138,94,89,82,91,148,178,57,111,152,77,120>>,old_height => 4,prev_new_hash => <<115,102,79,204,251,221,187,124,123,168,37,22,147,133,17,6,116,111,28,110,151,32,153,253,188,158,168,218,248,152,22,112>>,type => key}},10000]}}
test1-aeproject_node-1      | 11:08:02.058 [error] gen_server aec_conductor terminated with reason: {{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,...}]},...} in gen_server:call/3 line 223
test1-aeproject_node-1      | 11:08:02.058 [debug] Options = []
test1-aeproject_node-1      | 11:08:02.058 [error] CRASH REPORT Process aec_conductor with 0 neighbours exited with reason: {{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,...}]},...} in gen_server:call/3 line 223
test1-aeproject_node-1      | 11:08:02.058 [debug] init: GCHeight = 4
test1-aeproject_node-1      | 11:08:02.058 [debug] consensus_request({mine_blocks,1,key}) Failed: exit {{{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,249}]}]},{gen_server,call,[aec_tx_pool,{top_change,#{new_hash => <<154,109,4,4,220,221,9,142,213,126,94,255,181,9,116,150,116,173,105,73,118,113,92,19,150,28,54,174,231,193,48,133>>,new_height => 4,old_hash => <<114,11,12,255,216,72,234,169,203,147,135,212,96,32,154,178,112,194,92,221,138,94,89,82,91,148,178,57,111,152,77,120>>,old_height => 4,prev_new_hash => <<115,102,79,204,251,221,187,124,123,168,37,22,147,133,17,6,116,111,28,110,151,32,153,253,188,158,168,218,248,152,22,112>>,type => key}},10000]}},{gen_server,call,[aec_conductor,{add_synced_block,{key_block,{key_header,4,<<115,102,79,204,251,221,187,124,123,168,37,22,147,133,17,6,116,111,28,110,151,32,153,253,188,158,168,218,248,152,22,112>>,<<115,102,79,204,251,221,187,124,123,168,37,22,147,133,17,6,116,111,28,110,151,32,153,253,188,158,168,218,248,152,22,112>>,<<242,234,34,72,169,20,36,200,191,228,233,155,175,203,108,42,40,6,170,60,123,162,103,134,73,180,247,141,54,9,251,70>>,1338,0,1658747282056,5,no_value,<<67,48,83,3,213,241,82,158,103,104,92,4,42,35,168,242,181,249,129,143,56,179,24,136,55,33,110,248,223,78,39,157>>,<<55,239,123,83,136,117,193,86,122,60,31,91,246,65,232,223,183,40,98,14,4,83,38,133,228,37,233,151,193,29,10,84>>,<<0,0,2,140>>,#{consensus => aec_consensus_on_demand}}}},30000]}} [{gen_server,call,3,[{file,"gen_server.erl"},{line,223}]},{aec_consensus_on_demand,client_request,1,[{file,"/app/apps/aecore/src/aec_consensus_on_demand.erl"},{line,77}]},{aec_consensus_on_demand,'-client_request/1-lc$^2/1-2-',1,[{file,"/app/apps/aecore/src/aec_consensus_on_demand.erl"},{line,97}]},{aec_consensus_on_demand,client_request,1,[{file,"/app/apps/aecore/src/aec_consensus_on_demand.erl"},{line,97}]},{aec_conductor,consensus_request,1,[{file,"/app/apps/aecore/src/aec_conductor.erl"},{line,213}]},{aeplugin_dev_mode_emitter,emit_keyblocks_,2,[{file,"/aeplugin_dev_mode/src/aeplugin_dev_mode_emitter.erl"},{line,186}]},{aeplugin_dev_mode_emitter,handle_call,3,[{file,"/aeplugin_dev_mode/src/aeplugin_dev_mode_emitter.erl"},{line,110}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]}]
test1-aeproject_node-1      | 11:08:02.058 [error] Supervisor aec_conductor_sup had child aec_conductor started with aec_conductor:start_link() at <0.1628.0> exit with reason {{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,...}]},...} in context child_terminated
test1-aeproject_node-1      | 11:08:02.058 [error] gen_server aeplugin_dev_mode_emitter terminated with reason: {{{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,...}]},...},...} in aec_conductor:consensus_request/1 line 217
test1-aeproject_node-1      | 11:08:02.059 [error] CRASH REPORT Process aeplugin_dev_mode_emitter with 0 neighbours crashed with reason: {{{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,...}]},...},...} in aec_conductor:consensus_request/1 line 217
test1-aeproject_node-1      | 11:08:02.059 [error] Supervisor aeplugin_dev_mode_sup had child aeplugin_dev_mode_emitter started with aeplugin_dev_mode_emitter:start_link() at <0.1649.0> exit with reason {{{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,...}]},...},...} in aec_conductor:consensus_request/1 line 217 in context child_terminated
test1-aeproject_node-1      | 11:08:02.059 [error] CRASH REPORT Process <0.2259.0> with 0 neighbours exited with reason: {{{{{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,...}]},...},...},...},...} in gen_server:call/2 line 215 in gen_server:call/2 line 215
test1-aeproject_node-1      | 11:08:02.059 [error] Cowboy stream 1 with ranch listener devmode_listener and connection process <0.2258.0> had its request process exit with reason: {{{{{{badmatch,{error,unknown_hash}},[{aec_tx_pool,do_top_change,2,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,820}]},{aec_tx_pool,handle_call_,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,402}]},{timer,tc,1,[{file,"timer.erl"},{line,166}]},{aec_tx_pool,handle_call,3,[{file,"/app/apps/aecore/src/aec_tx_pool.erl"},{line,394}]},{gen_server,try_handle_call,4,[{file,"gen_server.erl"},{line,661}]},{gen_server,handle_msg,6,[{file,"gen_server.erl"},{line,690}]},{proc_lib,init_p_do_apply,...}]},...},...},...},...} in gen_server:call/2 line 215
test1-aeproject_node-1      | 11:08:17.286 [debug] [593] Msg = #{}
test1-aeproject_node-1      | 11:08:47.358 [debug] [409] Msg = #{}