Running this script against the latest main commit I got:

./tigerbeetle start --addresses=3000 0_0.tigerbeetle
info(io): opening "0_0.tigerbeetle"...
info(main): 0: cluster=0: listening on 127.0.0.1:3000
info(message_bus): connection from client 206272961074970794762714954581878172245
thread 2523663 panic: reached unreachable code
/home/phil/tb/tigerbeetle/zig/lib/std/debug.zig:225:14: 0x26920b in std.debug.assert (tigerbeetle)
    if (!ok) unreachable; // assertion failure
             ^
/home/phil/tb/tigerbeetle/src/lsm/manifest.zig:317:23: 0x42a7ee in lsm.manifest.ManifestType(lsm.table.TableType(u128,lsm.groove.IdTreeValue,lsm.groove.IdTreeValue.compare_keys,lsm.groove.IdTreeValue.key_from_value,554112867134706473364364839029663282043,lsm.groove.IdTreeValue.tombstone,lsm.groove.IdTreeValue.tombstone_from_key),storage.Storage).assert_no_invisible_tables (tigerbeetle)
                assert(it.next() == null);
                      ^
/home/phil/tb/tigerbeetle/src/lsm/tree.zig:873:57: 0x404f46 in lsm.tree.TreeType(lsm.table.TableType(u128,lsm.groove.IdTreeValue,lsm.groove.IdTreeValue.compare_keys,lsm.groove.IdTreeValue.key_from_value,554112867134706473364364839029663282043,lsm.groove.IdTreeValue.tombstone,lsm.groove.IdTreeValue.tombstone_from_key),storage.Storage,[]const u8{84,114,97,110,115,102,101,114,46,105,100}).checkpoint (tigerbeetle)
                tree.manifest.assert_no_invisible_tables(snapshot);
                                                        ^
/home/phil/tb/tigerbeetle/src/lsm/groove.zig:890:34: 0x3e99cb in lsm.groove.GrooveType(storage.Storage,tigerbeetle.Transfer,(struct state_machine.struct:40:14 constant)).checkpoint (tigerbeetle)
            groove.ids.checkpoint(Join.tree_callback(.ids));
                                 ^
/home/phil/tb/tigerbeetle/src/lsm/forest.zig:200:62: 0x3c202c in lsm.forest.ForestType(storage.Storage,(struct state_machine.struct:47:49 constant)).checkpoint (tigerbeetle)
                @field(forest.grooves, field.name).checkpoint(Join.groove_callback(field.name));
                                                             ^
/home/phil/tb/tigerbeetle/src/state_machine.zig:382:35: 0x3a3117 in state_machine.StateMachineType(storage.Storage).checkpoint (tigerbeetle)
            self.forest.checkpoint(checkpoint_finish);
                                  ^
/home/phil/tb/tigerbeetle/src/vsr/replica.zig:2531:46: 0x386e72 in vsr.replica.ReplicaType(state_machine.StateMachineType(storage.Storage),message_bus.MessageBusType(vsr.ProcessType.replica),storage.Storage,time.Time).commit_op_compact_callback (tigerbeetle)
                self.state_machine.checkpoint(commit_op_checkpoint_state_machine_callback);
                                             ^
/home/phil/tb/tigerbeetle/src/state_machine.zig:374:21: 0x3a305a in state_machine.StateMachineType(storage.Storage).compact_finish (tigerbeetle)
            callback(self);
                    ^
/home/phil/tb/tigerbeetle/src/lsm/forest.zig:169:37: 0x3e94df in lsm.forest.ForestType(storage.Storage,(struct state_machine.struct:47:49 constant)).JoinType(lsm.forest.JoinOp.compacting).groove_callback().groove_cb (tigerbeetle)
                            callback(forest);
                                    ^
/home/phil/tb/tigerbeetle/src/lsm/groove.zig:848:37: 0x40350d in lsm.groove.GrooveType(storage.Storage,tigerbeetle.Transfer,(struct state_machine.struct:40:14 constant)).JoinType(lsm.groove.JoinOp.compacting).tree_callback().tree_cb (tigerbeetle)
                            callback(groove);
                                    ^
/home/phil/tb/tigerbeetle/src/lsm/tree.zig:848:21: 0x464318 in lsm.tree.TreeType(lsm.table.TableType(lsm.composite_key.CompositeKey(u64),lsm.composite_key.Value,lsm.composite_key.CompositeKey(u64).compare_keys,lsm.composite_key.CompositeKey(u64).key_from_value,(struct lsm.composite_key.CompositeKey(u64) constant),lsm.composite_key.CompositeKey(u64).tombstone,lsm.composite_key.CompositeKey(u64).tombstone_from_key),storage.Storage,[]const u8{84,114,97,110,115,102,101,114,46,97,109,111,117,110,116}).compact_manifest_callback (tigerbeetle)
            callback(tree);
                    ^
/home/phil/tb/tigerbeetle/src/lsm/manifest.zig:539:21: 0x47eab3 in lsm.manifest.ManifestType(lsm.table.TableType(lsm.composite_key.CompositeKey(u64),lsm.composite_key.Value,lsm.composite_key.CompositeKey(u64).compare_keys,lsm.composite_key.CompositeKey(u64).key_from_value,(struct lsm.composite_key.CompositeKey(u64) constant),lsm.composite_key.CompositeKey(u64).tombstone,lsm.composite_key.CompositeKey(u64).tombstone_from_key),storage.Storage).manifest_log_compact_callback (tigerbeetle)
            callback(manifest);
                    ^
/home/phil/tb/tigerbeetle/src/lsm/manifest_log.zig:451:25: 0x4c233c in lsm.manifest_log.ManifestLogType(storage.Storage,lsm.manifest.TableInfoType(lsm.table.TableType(lsm.composite_key.CompositeKey(u64),lsm.composite_key.Value,lsm.composite_key.CompositeKey(u64).compare_keys,lsm.composite_key.CompositeKey(u64).key_from_value,(struct lsm.composite_key.CompositeKey(u64) constant),lsm.composite_key.CompositeKey(u64).tombstone,lsm.composite_key.CompositeKey(u64).tombstone_from_key))).compact_flush_callback (tigerbeetle)
                callback(manifest_log);
                        ^
/home/phil/tb/tigerbeetle/src/lsm/manifest_log.zig:358:25: 0x46b9a4 in lsm.manifest_log.ManifestLogType(storage.Storage,lsm.manifest.TableInfoType(lsm.table.TableType(lsm.composite_key.CompositeKey(u64),lsm.composite_key.Value,lsm.composite_key.CompositeKey(u64).compare_keys,lsm.composite_key.CompositeKey(u64).key_from_value,(struct lsm.composite_key.CompositeKey(u64) constant),lsm.composite_key.CompositeKey(u64).tombstone,lsm.composite_key.CompositeKey(u64).tombstone_from_key))).write_block (tigerbeetle)
                callback(manifest_log);
                        ^
/home/phil/tb/tigerbeetle/src/lsm/manifest_log.zig:338:37: 0x4c210e in lsm.manifest_log.ManifestLogType(storage.Storage,lsm.manifest.TableInfoType(lsm.table.TableType(lsm.composite_key.CompositeKey(u64),lsm.composite_key.Value,lsm.composite_key.CompositeKey(u64).compare_keys,lsm.composite_key.CompositeKey(u64).key_from_value,(struct lsm.composite_key.CompositeKey(u64) constant),lsm.composite_key.CompositeKey(u64).tombstone,lsm.composite_key.CompositeKey(u64).tombstone_from_key))).flush (tigerbeetle)
            manifest_log.write_block();
                                    ^
/home/phil/tb/tigerbeetle/src/lsm/manifest_log.zig:421:31: 0x47ea26 in lsm.manifest_log.ManifestLogType(storage.Storage,lsm.manifest.TableInfoType(lsm.table.TableType(lsm.composite_key.CompositeKey(u64),lsm.composite_key.Value,lsm.composite_key.CompositeKey(u64).compare_keys,lsm.composite_key.CompositeKey(u64).key_from_value,(struct lsm.composite_key.CompositeKey(u64) constant),lsm.composite_key.CompositeKey(u64).tombstone,lsm.composite_key.CompositeKey(u64).tombstone_from_key))).compact (tigerbeetle)
            manifest_log.flush(compact_flush_callback);
                              ^
/home/phil/tb/tigerbeetle/src/lsm/manifest.zig:530:42: 0x45e901 in lsm.manifest.ManifestType(lsm.table.TableType(lsm.composite_key.CompositeKey(u64),lsm.composite_key.Value,lsm.composite_key.CompositeKey(u64).compare_keys,lsm.composite_key.CompositeKey(u64).key_from_value,(struct lsm.composite_key.CompositeKey(u64) constant),lsm.composite_key.CompositeKey(u64).tombstone,lsm.composite_key.CompositeKey(u64).tombstone_from_key),storage.Storage).compact (tigerbeetle)
            manifest.manifest_log.compact(manifest_log_compact_callback);
                                         ^
/home/phil/tb/tigerbeetle/src/lsm/tree.zig:819:34: 0x447c29 in lsm.tree.TreeType(lsm.table.TableType(lsm.composite_key.CompositeKey(u64),lsm.composite_key.Value,lsm.composite_key.CompositeKey(u64).compare_keys,lsm.composite_key.CompositeKey(u64).key_from_value,(struct lsm.composite_key.CompositeKey(u64) constant),lsm.composite_key.CompositeKey(u64).tombstone,lsm.composite_key.CompositeKey(u64).tombstone_from_key),storage.Storage,[]const u8{84,114,97,110,115,102,101,114,46,97,109,111,117,110,116}).compact_done (tigerbeetle)
            tree.manifest.compact(compact_manifest_callback);
                                 ^
/home/phil/tb/tigerbeetle/src/lsm/tree.zig:732:67: 0x4293aa in lsm.tree.TreeType(lsm.table.TableType(lsm.composite_key.CompositeKey(u64),lsm.composite_key.Value,lsm.composite_key.CompositeKey(u64).compare_keys,lsm.composite_key.CompositeKey(u64).key_from_value,(struct lsm.composite_key.CompositeKey(u64) constant),lsm.composite_key.CompositeKey(u64).tombstone,lsm.composite_key.CompositeKey(u64).tombstone_from_key),storage.Storage,[]const u8{84,114,97,110,115,102,101,114,46,97,109,111,117,110,116}).compact_tick_done (tigerbeetle)
            if (tree.compaction_io_pending == 0) tree.compact_done();
                                                                  ^
/home/phil/tb/tigerbeetle/src/lsm/tree.zig:702:35: 0x447701 in lsm.tree.TreeType(lsm.table.TableType(lsm.composite_key.CompositeKey(u64),lsm.composite_key.Value,lsm.composite_key.CompositeKey(u64).compare_keys,lsm.composite_key.CompositeKey(u64).key_from_value,(struct lsm.composite_key.CompositeKey(u64) constant),lsm.composite_key.CompositeKey(u64).tombstone,lsm.composite_key.CompositeKey(u64).tombstone_from_key),storage.Storage,[]const u8{84,114,97,110,115,102,101,114,46,97,109,111,117,110,116}).compact_tick_callback_table_immutable (tigerbeetle)
            tree.compact_tick_done();
                                  ^
/home/phil/tb/tigerbeetle/src/lsm/compaction.zig:389:21: 0x47d732 in lsm.compaction.CompactionType(lsm.table.TableType(lsm.composite_key.CompositeKey(u64),lsm.composite_key.Value,lsm.composite_key.CompositeKey(u64).compare_keys,lsm.composite_key.CompositeKey(u64).key_from_value,(struct lsm.composite_key.CompositeKey(u64) constant),lsm.composite_key.CompositeKey(u64).tombstone,lsm.composite_key.CompositeKey(u64).tombstone_from_key),storage.Storage,lsm.table_immutable.TableImmutableIteratorType).cpu_merge_start (tigerbeetle)
            callback(compaction);
                    ^
/home/phil/tb/tigerbeetle/src/lsm/compaction.zig:358:71: 0x45daa5 in lsm.compaction.CompactionType(lsm.table.TableType(lsm.composite_key.CompositeKey(u64),lsm.composite_key.Value,lsm.composite_key.CompositeKey(u64).compare_keys,lsm.composite_key.CompositeKey(u64).key_from_value,(struct lsm.composite_key.CompositeKey(u64) constant),lsm.composite_key.CompositeKey(u64).tombstone,lsm.composite_key.CompositeKey(u64).tombstone_from_key),storage.Storage,lsm.table_immutable.TableImmutableIteratorType).io_finish (tigerbeetle)
            if (compaction.io_pending == 0) compaction.cpu_merge_start();
                                                                      ^
/home/phil/tb/tigerbeetle/src/lsm/compaction.zig:325:42: 0x47d540 in lsm.compaction.write_callback.callback (tigerbeetle)
                    _compaction.io_finish();
                                         ^
/home/phil/tb/tigerbeetle/src/lsm/grid.zig:330:37: 0x4e7a2c in lsm.grid.GridType(storage.Storage).write_block_callback (tigerbeetle)
            completed_write.callback(completed_write);
                                    ^
/home/phil/tb/tigerbeetle/src/storage.zig:310:27: 0x32c90c in storage.Storage.on_write (tigerbeetle)
            write.callback(write);
                          ^
/home/phil/tb/tigerbeetle/src/io/linux.zig:860:29: 0x32cb40 in io.linux.struct:858:25.wrapper (tigerbeetle)
                    callback(
                            ^
/home/phil/tb/tigerbeetle/src/io/linux.zig:460:40: 0x30ff58 in io.linux.Completion.complete (tigerbeetle)
                    completion.callback(completion.context, completion, &result);
                                       ^
/home/phil/tb/tigerbeetle/src/io/linux.zig:110:64: 0x30dae8 in io.linux.IO.flush (tigerbeetle)
            while (copy.pop()) |completion| completion.complete();
                                                               ^
/home/phil/tb/tigerbeetle/src/io/linux.zig:90:27: 0x2b2972 in io.linux.IO.run_for_ns (tigerbeetle)
            try self.flush(1, &timeouts, &etime);
                          ^
/home/phil/tb/tigerbeetle/src/main.zig:152:38: 0x29bd44 in Command.start (tigerbeetle)
            try command.io.run_for_ns(config.tick_ms * std.time.ns_per_ms);
                                     ^
/home/phil/tb/tigerbeetle/src/main.zig:45:44: 0x293708 in main (tigerbeetle)
        .start => |*args| try Command.start(allocator, args.addresses, args.memory, args.path),
                                           ^
/home/phil/tb/tigerbeetle/zig/lib/std/start.zig:561:37: 0x28cb4a in std.start.callMain (tigerbeetle)
            const result = root.main() catch |err| {
                                    ^
/home/phil/tb/tigerbeetle/zig/lib/std/start.zig:495:12: 0x26d89e in std.start.callMainWithArgs (tigerbeetle)
    return @call(.{ .modifier = .always_inline }, callMain, .{});
           ^
/home/phil/tb/tigerbeetle/zig/lib/std/start.zig:409:17: 0x26c936 in std.start.posixCallMainAndExit (tigerbeetle)
    std.os.exit(@call(.{ .modifier = .always_inline }, callMainWithArgs, .{ argc, argv, envp }));
                ^
/home/phil/tb/tigerbeetle/zig/lib/std/start.zig:322:5: 0x26c742 in std.start._start (tigerbeetle)
    @call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{});
    ^
[1]    2523663 IOT instruction (core dumped)  ./tigerbeetle start --addresses=3000 0_0.tigerbeetle

CommitTransferError.already_auto_committed doesn't appear to used.

Is it correct to assume that transfer_not_two_phase_commit means it was already auto-committed?

Bug Type:
correctness

Seed:
11493803597036870122

Commit Hash:
01494c5

Parameters:
[info] (state_checker):
SEED=11493803597036870122

replicas=3
clients=14
request_probability=72%
idle_on_probability=4%
idle_off_probability=16%
one_way_delay_mean=10 ticks
one_way_delay_min=2 ticks
packet_loss_probability=23%
path_maximum_capacity=8 messages
path_clog_duration_mean=207 ticks
path_clog_probability=1%
packet_replay_probability=42%
partition_mode=PartitionMode.uniform_partition
partition_probability=1%
unpartition_probability=5%
partition_stability=158 ticks
unpartition_stability=5 ticks
read_latency_min=0
read_latency_mean=10
write_latency_min=0
write_latency_mean=95
read_fault_probability=6%
write_fault_probability=1%
crash_probability=0.01%
crash_stability=255 ticks
restart_probability=1%
restart_stability=561 ticks

Stack Trace:

exit with code: 129


Tail of Debug Logs:
mulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=9 clogged for ticks=77
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=10 clogged for ticks=229
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=14 clogged for ticks=251
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=16 clogged for ticks=324
[err] (packet_simulator): dropped packet from=9 to=1.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=4 clogged for ticks=300
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=0 clogged for ticks=285
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=12 clogged for ticks=559
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=3 clogged for ticks=173
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=13 clogged for ticks=877
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=14 clogged for ticks=44
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=2 clogged for ticks=2
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=4 clogged for ticks=296
[debug] (vsr): 0: repair_timeout fired
[debug] (vsr): 0: repair_timeout reset
[debug] (replica): 0: repair_prepare: op=51 checksum=21088028739097856302406738726717091379 (already writing)
[debug] (packet_simulator): replayed packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=66 status=Status.view_change Header{ .checksum = 91424483127492698814768571823184706174, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 66, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=10 clogged for ticks=48
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=3 clogged for ticks=86
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=8 clogged for ticks=293
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=10 clogged for ticks=534
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=3 clogged for ticks=114
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=66 status=Status.view_change Header{ .checksum = 91424483127492698814768571823184706174, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 66, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=0 clogged for ticks=28
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=9 clogged for ticks=158
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=14 clogged for ticks=301
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=15 clogged for ticks=617
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=15 clogged for ticks=69
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=7 clogged for ticks=521
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=8 clogged for ticks=235
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=14 clogged for ticks=136
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=15 clogged for ticks=0
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=15 clogged for ticks=66
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=13 clogged for ticks=734
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=1 clogged for ticks=229
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=14 clogged for ticks=109
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=0 clogged for ticks=484
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=13 clogged for ticks=26
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=14 clogged for ticks=156
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=3 clogged for ticks=42
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=10 clogged for ticks=547
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=2 clogged for ticks=190
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=9 clogged for ticks=369
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=7 clogged for ticks=535
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=8 clogged for ticks=10
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=15 clogged for ticks=192
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=13 clogged for ticks=13
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=0 clogged for ticks=211
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=15 clogged for ticks=147
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=6 clogged for ticks=199
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=16 clogged for ticks=207
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=16 clogged for ticks=73
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=16 clogged for ticks=241
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=1 clogged for ticks=91
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=11 clogged for ticks=599
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=12 clogged for ticks=115
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=14 clogged for ticks=394
[debug] (vsr): 2: view_change_status_timeout fired
[debug] (replica): 2: transition_to_view_change_status: view=66..67
[debug] (vsr): 2: ping_timeout stopped
[debug] (vsr): 2: commit_timeout stopped
[debug] (vsr): 2: normal_status_timeout stopped
[debug] (vsr): 2: view_change_status_timeout started
[debug] (vsr): 2: view_change_message_timeout started
[debug] (vsr): 2: repair_timeout stopped
[debug] (vsr): 2: prepare_timeout stopped
[debug] (replica): 2: reset 0 do_view_change message(s) from view=null
[debug] (replica): 2: sending start_view_change to replica 0: Header{ .checksum = 148788271288469827079392522094434529010, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[err] (packet_simulator): submit_packet: Path{ .source = 2, .target = 0 } reached capacity, dropped packet=4
[debug] (replica): 2: sending start_view_change to replica 1: Header{ .checksum = 148788271288469827079392522094434529010, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=2 clogged for ticks=353
[debug] (journal): 0: write_sectors: offset=53493760 len=978944 unlocked
[debug] (journal): 0: write_header: op=51 sectors[4096..8192]
[debug] (journal): 0: write_sectors: offset=4096 len=4096 locked
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=1 clogged for ticks=234
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=8 clogged for ticks=33
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=10 clogged for ticks=167
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=13 clogged for ticks=1365
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=3 clogged for ticks=405
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=15 clogged for ticks=14
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=3 clogged for ticks=62
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=8 clogged for ticks=269
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=11 clogged for ticks=165
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=16 clogged for ticks=592
[debug] (vsr): 1: view_change_status_timeout fired
[debug] (replica): 1: transition_to_view_change_status: view=66..67
[debug] (vsr): 1: ping_timeout stopped
[debug] (vsr): 1: commit_timeout stopped
[debug] (vsr): 1: normal_status_timeout stopped
[debug] (vsr): 1: view_change_status_timeout started
[debug] (vsr): 1: view_change_message_timeout started
[debug] (vsr): 1: repair_timeout stopped
[debug] (vsr): 1: prepare_timeout stopped
[debug] (replica): 1: reset 0 do_view_change message(s) from view=null
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 34925897259489638383297767014559971525, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 34925897259489638383297767014559971525, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=3 clogged for ticks=232
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=3 clogged for ticks=206
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=6 clogged for ticks=770
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=11 clogged for ticks=38
[debug] (vsr): 211007803378812328229853630647012732826: request_timeout fired
[debug] (vsr): 211007803378812328229853630647012732826: request_timeout backing off
[debug] (vsr): 211007803378812328229853630647012732826: request_timeout after=326..474 (rtt=30 min=10 max=1000 attempts=17)
[debug] (client): 211007803378812328229853630647012732826: on_request_timeout: resending request=2 checksum=317772138329866820221796767385335562778
[debug] (client): 211007803378812328229853630647012732826: sending request to replica 2: Header{ .checksum = 317772138329866820221796767385335562778, .checksum_body = 98081761841810822994326224503485043631, .parent = 155365525320536877751013994650567452311, .client = 211007803378812328229853630647012732826, .context = 25, .request = 2, .cluster = 0, .epoch = 0, .view = 51, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (network): send_message: Process{ .client = 211007803378812328229853630647012732826 } > Process{ .replica = 2 }: Command.request
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=0 clogged for ticks=274
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=10 clogged for ticks=95
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=67 status=Status.view_change Header{ .checksum = 34925897259489638383297767014559971525, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: 1 message(s)
[debug] (replica): 2: on_start_view_change: view=67 quorum received
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=52 dest.len=32 copied=32
[debug] (replica): 2: sending do_view_change to replica 1: Header{ .checksum = 143191943885305665980256935230075102131, .checksum_body = 198426207204972815804614059460232725549, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 52, .commit = 49, .timestamp = 63, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.do_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=3 clogged for ticks=90
[debug] (vsr): 0: view_change_status_timeout fired
[debug] (replica): 0: transition_to_view_change_status: view=66..67
[debug] (vsr): 0: ping_timeout stopped
[debug] (vsr): 0: commit_timeout stopped
[debug] (vsr): 0: normal_status_timeout stopped
[debug] (vsr): 0: view_change_status_timeout started
[debug] (vsr): 0: view_change_message_timeout started
[debug] (vsr): 0: repair_timeout stopped
[debug] (vsr): 0: prepare_timeout stopped
[debug] (replica): 0: reset 3 do_view_change message(s) from view=66
[debug] (replica): 0: sending start_view_change to replica 1: Header{ .checksum = 248383076221056861483801297275895435280, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 2: Header{ .checksum = 248383076221056861483801297275895435280, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=1 clogged for ticks=158
[debug] (packet_simulator): replayed packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=67 status=Status.view_change Header{ .checksum = 248383076221056861483801297275895435280, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: 1 message(s)
[debug] (replica): 1: on_start_view_change: view=67 quorum received
[debug] (journal): 1: copy_latest_headers_between: op_min=0 op_max=52 dest.len=32 copied=32
[debug] (replica): 1: sending do_view_change to replica 1: Header{ .checksum = 25339260051478379642186510769861207298, .checksum_body = 198426207204972815804614059460232725549, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 52, .commit = 49, .timestamp = 64, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_message: view=67 status=Status.view_change Header{ .checksum = 25339260051478379642186510769861207298, .checksum_body = 198426207204972815804614059460232725549, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 52, .commit = 49, .timestamp = 64, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_do_view_change: 1 message(s)
[debug] (replica): 1: on_do_view_change: waiting for quorum
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=8 clogged for ticks=40
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=13 clogged for ticks=284
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=9 clogged for ticks=37
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=15 clogged for ticks=389
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=1 clogged for ticks=88
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=9 clogged for ticks=66
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=14 clogged for ticks=588
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=8 clogged for ticks=171
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=12 clogged for ticks=19
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=13 clogged for ticks=15
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=9 clogged for ticks=253
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=9 clogged for ticks=110
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=15 clogged for ticks=67
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=8 clogged for ticks=54
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=0 clogged for ticks=25
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=11 clogged for ticks=277
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=3 clogged for ticks=56
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=10 clogged for ticks=16
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=13 clogged for ticks=449
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=13 clogged for ticks=505
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=1 clogged for ticks=192
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=10 clogged for ticks=252
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=11 clogged for ticks=444
[err] (packet_simulator): dropped packet from=6 to=2.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=15 clogged for ticks=568
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=11 clogged for ticks=537
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=16 clogged for ticks=296
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=0 clogged for ticks=292
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=4 clogged for ticks=76
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=16 clogged for ticks=179
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=7 clogged for ticks=121
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=12 clogged for ticks=693
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=5 clogged for ticks=12
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=8 clogged for ticks=9
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=16 clogged for ticks=328
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=8 clogged for ticks=35
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=3 clogged for ticks=8
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=4 clogged for ticks=421
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=67 status=Status.view_change Header{ .checksum = 248383076221056861483801297275895435280, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: 2 message(s)
[debug] (replica): 2: on_start_view_change: ignoring (quorum received already)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=2 clogged for ticks=75
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=10 clogged for ticks=262
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=15 clogged for ticks=192
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=1 clogged for ticks=195
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=3 clogged for ticks=121
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=4 clogged for ticks=50
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=14 clogged for ticks=88
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=16 clogged for ticks=371
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=9 clogged for ticks=24
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=8 clogged for ticks=242
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=4 clogged for ticks=39
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=10 clogged for ticks=206
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=1 clogged for ticks=65
[err] (packet_simulator): dropped packet from=0 to=1.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=8 clogged for ticks=31
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=8 clogged for ticks=502
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=3 clogged for ticks=502
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=6 clogged for ticks=285
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=9 clogged for ticks=39
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=8 clogged for ticks=1
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=14 clogged for ticks=239
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=9 clogged for ticks=731
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=13 clogged for ticks=184
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=1 clogged for ticks=59
[debug] (vsr): 295053281728242544783420554551301260516: request_timeout fired
[debug] (vsr): 295053281728242544783420554551301260516: request_timeout backing off
[debug] (vsr): 295053281728242544783420554551301260516: request_timeout after=317..1044 (rtt=30 min=10 max=1000 attempts=8)
[debug] (client): 295053281728242544783420554551301260516: on_request_timeout: resending request=3 checksum=113192342753930750258885197839572941237
[debug] (client): 295053281728242544783420554551301260516: sending request to replica 2: Header{ .checksum = 113192342753930750258885197839572941237, .checksum_body = 252674135858478617928046913993741778797, .parent = 170704462561137599274424428789553518351, .client = 295053281728242544783420554551301260516, .context = 24, .request = 3, .cluster = 0, .epoch = 0, .view = 54, .op = 0, .commit = 0, .timestamp = 0, .size = 975737, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (network): send_message: Process{ .client = 295053281728242544783420554551301260516 } > Process{ .replica = 2 }: Command.request
[debug] (vsr): 2: view_change_message_timeout fired
[debug] (vsr): 2: view_change_message_timeout reset
[debug] (replica): 2: sending start_view_change to replica 0: Header{ .checksum = 148788271288469827079392522094434529010, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[err] (packet_simulator): submit_packet: Path{ .source = 2, .target = 0 } reached capacity, dropped packet=4
[debug] (replica): 2: sending start_view_change to replica 1: Header{ .checksum = 148788271288469827079392522094434529010, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=52 dest.len=32 copied=32
[debug] (replica): 2: sending do_view_change to replica 1: Header{ .checksum = 143191943885305665980256935230075102131, .checksum_body = 198426207204972815804614059460232725549, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 52, .commit = 49, .timestamp = 63, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.do_view_change
[err] (packet_simulator): submit_packet: Path{ .source = 2, .target = 1 } reached capacity, dropped packet=5
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=13 clogged for ticks=329
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=6 clogged for ticks=502
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=9 clogged for ticks=438
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=10 clogged for ticks=19
[err] (packet_simulator): dropped packet from=2 to=0.
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.nack_prepare
[debug] (replica): 0: on_message: view=67 status=Status.view_change Header{ .checksum = 11121579069666451371095422121659725280, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 21088028739097856302406738726717091379, .request = 0, .cluster = 0, .epoch = 0, .view = 66, .op = 51, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.nack_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_nack_prepare: ignoring (older view)
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.nack_prepare
[debug] (replica): 0: on_message: view=67 status=Status.view_change Header{ .checksum = 11121579069666451371095422121659725280, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 21088028739097856302406738726717091379, .request = 0, .cluster = 0, .epoch = 0, .view = 66, .op = 51, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.nack_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_nack_prepare: ignoring (older view)
[debug] (packet_simulator): replayed packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (replica): 0: on_message: view=67 status=Status.view_change Header{ .checksum = 85157876442447231688618019474858175203, .checksum_body = 198426207204972815804614059460232725549, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 66, .op = 52, .commit = 49, .timestamp = 63, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: ignoring (older view)
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=67 status=Status.view_change Header{ .checksum = 281059202467510103615709092701845063271, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 66, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (older view)
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=67 status=Status.view_change Header{ .checksum = 281059202467510103615709092701845063271, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 66, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (older view)
[debug] (packet_simulator): replayed packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=67 status=Status.view_change Header{ .checksum = 148788271288469827079392522094434529010, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: 1 message(s)
[debug] (replica): 0: on_start_view_change: view=67 quorum received
[debug] (journal): 0: copy_latest_headers_between: op_min=0 op_max=52 dest.len=32 copied=32
[debug] (replica): 0: sending do_view_change to replica 1: Header{ .checksum = 297311158262262137619469466934577221264, .checksum_body = 198426207204972815804614059460232725549, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 52, .commit = 51, .timestamp = 63, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.do_view_change
[debug] (packet_simulator): replayed packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=67 status=Status.view_change Header{ .checksum = 148788271288469827079392522094434529010, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=2)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=15 clogged for ticks=153
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=11 clogged for ticks=265
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=4 clogged for ticks=13
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=9 clogged for ticks=56
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=14 clogged for ticks=287
[debug] (vsr): 1: view_change_message_timeout fired
[debug] (vsr): 1: view_change_message_timeout reset
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 34925897259489638383297767014559971525, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 34925897259489638383297767014559971525, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=67 status=Status.view_change Header{ .checksum = 148788271288469827079392522094434529010, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=2)
[debug] (packet_simulator): replayed packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=67 status=Status.view_change Header{ .checksum = 34925897259489638383297767014559971525, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=3 clogged for ticks=507
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=5 clogged for ticks=60
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=6 clogged for ticks=290
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=12 clogged for ticks=294
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (replica): 0: on_message: view=67 status=Status.view_change Header{ .checksum = 85157876442447231688618019474858175203, .checksum_body = 198426207204972815804614059460232725549, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 66, .op = 52, .commit = 49, .timestamp = 63, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: ignoring (older view)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=3 clogged for ticks=91
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=11 clogged for ticks=41
[debug] (vsr): 0: view_change_message_timeout fired
[debug] (vsr): 0: view_change_message_timeout reset
[debug] (replica): 0: sending start_view_change to replica 1: Header{ .checksum = 248383076221056861483801297275895435280, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 2: Header{ .checksum = 248383076221056861483801297275895435280, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (journal): 0: copy_latest_headers_between: op_min=0 op_max=52 dest.len=32 copied=32
[debug] (replica): 0: sending do_view_change to replica 1: Header{ .checksum = 297311158262262137619469466934577221264, .checksum_body = 198426207204972815804614059460232725549, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 52, .commit = 51, .timestamp = 63, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.do_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=3 clogged for ticks=356
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=7 clogged for ticks=0
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=9 clogged for ticks=641
[debug] (packet_simulator): delivering packet from=10 to=2
[debug] (network): deliver_message: Process{ .client = 295053281728242544783420554551301260516 } > Process{ .replica = 2 }: Command.request
[debug] (replica): 2: on_message: view=67 status=Status.view_change Header{ .checksum = 113192342753930750258885197839572941237, .checksum_body = 252674135858478617928046913993741778797, .parent = 170704462561137599274424428789553518351, .client = 295053281728242544783420554551301260516, .context = 24, .request = 3, .cluster = 0, .epoch = 0, .view = 54, .op = 0, .commit = 0, .timestamp = 0, .size = 975737, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (replica): 2: on_request: ignoring (Status.view_change)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=16 clogged for ticks=85
[err] (packet_simulator): dropped packet from=0 to=1.
[debug] (packet_simulator): replayed packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.do_view_change
[debug] (replica): 1: on_message: view=67 status=Status.view_change Header{ .checksum = 297311158262262137619469466934577221264, .checksum_body = 198426207204972815804614059460232725549, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 52, .commit = 51, .timestamp = 63, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_do_view_change: 2 message(s)
[debug] (replica): 1: on_do_view_change: view=67 quorum received
[debug] (replica): 1: on_do_view_change: replica=0 v'=63 op=52 commit=51 latest=Header{ .checksum = 138121538964620398974032417385976652336, .checksum_body = 172455124435379985576757410909512342430, .parent = 21088028739097856302406738726717091379, .client = 114587172820684642095745604157145874771, .context = 227399499304835206892657516681538539333, .request = 4, .cluster = 0, .epoch = 0, .view = 64, .op = 52, .commit = 49, .timestamp = 468010000000, .size = 768318, .replica = 1, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_do_view_change: replica=1 v'=64 op=52 commit=49 latest=Header{ .checksum = 138121538964620398974032417385976652336, .checksum_body = 172455124435379985576757410909512342430, .parent = 21088028739097856302406738726717091379, .client = 114587172820684642095745604157145874771, .context = 227399499304835206892657516681538539333, .request = 4, .cluster = 0, .epoch = 0, .view = 64, .op = 52, .commit = 49, .timestamp = 468010000000, .size = 768318, .replica = 1, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_do_view_change: view=67 op=52..52 commit=49..51 checksum=138121538964620398974032417385976652336
[debug] (replica): 1: on_do_view_change: latest op exists exactly
[debug] (journal): 1: remove_entries_from: op_min=53
[debug] (replica): 1: repair_header: op=52 false (checksum clean)
[debug] (replica): 1: repair_header: op=51 false (checksum clean)
[debug] (replica): 1: repair_header: op=50 false (checksum clean)
[debug] (replica): 1: repair_header: op=49 false (checksum clean)
[debug] (replica): 1: repair_header: op=48 false (checksum clean)
[debug] (replica): 1: repair_header: op=47 false (checksum clean)
[debug] (replica): 1: repair_header: op=46 false (checksum clean)
[debug] (replica): 1: repair_header: op=45 false (checksum clean)
[debug] (replica): 1: repair_header: op=44 false (checksum clean)
[debug] (replica): 1: repair_header: op=43 false (checksum clean)
[debug] (replica): 1: repair_header: op=42 false (checksum clean)
[debug] (replica): 1: repair_header: op=41 false (checksum clean)
[debug] (replica): 1: repair_header: op=40 false (checksum clean)
[debug] (replica): 1: repair_header: op=39 false (checksum clean)
[debug] (replica): 1: repair_header: op=38 false (checksum clean)
[debug] (replica): 1: repair_header: op=37 false (checksum clean)
[debug] (replica): 1: repair_header: op=36 false (checksum clean)
[debug] (replica): 1: repair_header: op=35 false (checksum clean)
[debug] (replica): 1: repair_header: op=34 false (checksum clean)
[debug] (replica): 1: repair_header: op=33 false (checksum clean)
[debug] (replica): 1: repair_header: op=32 false (checksum clean)
[debug] (replica): 1: repair_header: op=31 false (checksum clean)
[debug] (replica): 1: repair_header: op=30 false (checksum clean)
[debug] (replica): 1: repair_header: op=29 false (checksum clean)
[debug] (replica): 1: repair_header: op=28 false (checksum clean)
[debug] (replica): 1: repair_header: op=27 false (checksum clean)
[debug] (replica): 1: repair_header: op=26 false (checksum clean)
[debug] (replica): 1: repair_header: op=25 false (checksum clean)
[debug] (replica): 1: repair_header: op=24 false (checksum clean)
[debug] (replica): 1: repair_header: op=23 false (checksum clean)
[debug] (replica): 1: repair_header: op=22 false (checksum clean)
[debug] (replica): 1: repair_header: op=21 false (checksum clean)
[debug] (replica): 1: repair_header: op=52 false (checksum clean)
[debug] (replica): 1: repair_header: op=51 false (checksum clean)
[debug] (replica): 1: repair_header: op=50 false (checksum clean)
[debug] (replica): 1: repair_header: op=49 false (checksum clean)
[debug] (replica): 1: repair_header: op=48 false (checksum clean)
[debug] (replica): 1: repair_header: op=47 false (checksum clean)
[debug] (replica): 1: repair_header: op=46 false (checksum clean)
[debug] (replica): 1: repair_header: op=45 false (checksum clean)
[debug] (replica): 1: repair_header: op=44 false (checksum clean)
[debug] (replica): 1: repair_header: op=43 false (checksum clean)
[debug] (replica): 1: repair_header: op=42 false (checksum clean)
[debug] (replica): 1: repair_header: op=41 false (checksum clean)
[debug] (replica): 1: repair_header: op=40 false (checksum clean)
[debug] (replica): 1: repair_header: op=39 false (checksum clean)
[debug] (replica): 1: repair_header: op=38 false (checksum clean)
[debug] (replica): 1: repair_header: op=37 false (checksum clean)
[debug] (replica): 1: repair_header: op=36 false (checksum clean)
[debug] (replica): 1: repair_header: op=35 false (checksum clean)
[debug] (replica): 1: repair_header: op=34 false (checksum clean)
[debug] (replica): 1: repair_header: op=33 false (checksum clean)
[debug] (replica): 1: repair_header: op=32 false (checksum clean)
[debug] (replica): 1: repair_header: op=31 false (checksum clean)
[debug] (replica): 1: repair_header: op=30 false (checksum clean)
[debug] (replica): 1: repair_header: op=29 false (checksum clean)
[debug] (replica): 1: repair_header: op=28 false (checksum clean)
[debug] (replica): 1: repair_header: op=27 false (checksum clean)
[debug] (replica): 1: repair_header: op=26 false (checksum clean)
[debug] (replica): 1: repair_header: op=25 false (checksum clean)
[debug] (replica): 1: repair_header: op=24 false (checksum clean)
[debug] (replica): 1: repair_header: op=23 false (checksum clean)
[debug] (replica): 1: repair_header: op=22 false (checksum clean)
[debug] (replica): 1: repair_header: op=21 false (checksum clean)
[debug] (vsr): 1: repair_timeout started
[debug] (vsr): 1: repair_timeout reset
[debug] (journal): 1: read_sectors: offset=52445184 len=1048576
[debug] (packet_simulator): replayed packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=67 status=Status.view_change Header{ .checksum = 34925897259489638383297767014559971525, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=11 clogged for ticks=203
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=15 clogged for ticks=13
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=4 clogged for ticks=178
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=12 clogged for ticks=32
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=12 clogged for ticks=107
[err] (packet_simulator): dropped packet from=0 to=2.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=2 clogged for ticks=156
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=16 clogged for ticks=81
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=0 clogged for ticks=56
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=67 status=Status.view_change Header{ .checksum = 34925897259489638383297767014559971525, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.do_view_change
[debug] (replica): 1: on_message: view=67 status=Status.view_change Header{ .checksum = 297311158262262137619469466934577221264, .checksum_body = 198426207204972815804614059460232725549, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 67, .op = 52, .commit = 51, .timestamp = 63, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_do_view_change: ignoring (duplicate message)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=8 clogged for ticks=138
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=12 clogged for ticks=401
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=9 clogged for ticks=735
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=12 clogged for ticks=30
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=13 clogged for ticks=356
[debug] (replica): 1: commit_op: executing view=67 true op=50 checksum=202079458619031925105824147341074188676 (hash)
[debug] (state_machine): state=f59e343c8b92f0dd3e8aa613f6c01b55 input=21dd49726e007aab4f296825b41b8d40 input.len=526947 new state=6675bf96004df9487d484ec19a15cea6
[info] (state_checker): 0036/0037 f59e343c8b92f0dd3e8aa613f6c01b55 > 6675bf96004df9487d484ec19a15cea6 1
[debug] (replica): 1: update_client_table_entry: client=6983078754327587608766091583351809616 session=14 request=3
[debug] (replica): 1: commit_op: replying to client: Header{ .checksum = 2887673327437171247428745035314840118, .checksum_body = 210434455080340872032307404475635975035, .parent = 125797494482566173252750752275374413505, .client = 6983078754327587608766091583351809616, .context = 0, .request = 3, .cluster = 0, .epoch = 0, .view = 57, .op = 50, .commit = 50, .timestamp = 0, .size = 144, .replica = 0, .command = Command.reply, .operation = Operation(3), .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .client = 6983078754327587608766091583351809616 }: Command.reply
[debug] (journal): 1: read_sectors: offset=53493760 len=1048576
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=9 clogged for ticks=645
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=0 clogged for ticks=66
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=2 clogged for ticks=38
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=16 clogged for ticks=319
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=16 clogged for ticks=150
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=7 clogged for ticks=512
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=8 clogged for ticks=125
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=12 clogged for ticks=85
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=13 clogged for ticks=24
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=14 clogged for ticks=496
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=11 clogged for ticks=251
[debug] (replica): 1: commit_op: executing view=67 true op=51 checksum=21088028739097856302406738726717091379 (hash)
[debug] (state_machine): state=6675bf96004df9487d484ec19a15cea6 input=b22879210f467fe978980058a55d9b8 input.len=975609 new state=cc602ec55913d9e349fc19580eaff73d
[err] (state_checker): state checker error: error.ReplicaTransitionedToInvalidState

Time:
2022-07-12 23:55:20.317320284 +0000 UTC

I understand TigerBeetle has been designed from ground up for financial transactions but it would be great If somehow we can also store/query user defined custom types/entities (users, products etc.) as If entities Ids and other data is stored in any other database then we can loose that data because of disk failure or any other issue as other databases are not as fault tolerant as TigerBeetle is, In that case whole system will become useless. with the ability to store user define entities/types TigerBeetle will become a complete database and a game changer with all other innovative and novel features. I request you to please consider this proposal.

Bug Type:
crash

Seed:
8770239035259319625

Commit Hash:
01494c5

Parameters:
[info] (state_checker):
SEED=8770239035259319625

replicas=3
clients=17
request_probability=56%
idle_on_probability=18%
idle_off_probability=13%
one_way_delay_mean=3 ticks
one_way_delay_min=1 ticks
packet_loss_probability=29%
path_maximum_capacity=7 messages
path_clog_duration_mean=51 ticks
path_clog_probability=0%
packet_replay_probability=17%
partition_mode=PartitionMode.uniform_size
partition_probability=0%
unpartition_probability=6%
partition_stability=189 ticks
unpartition_stability=16 ticks
read_latency_min=0
read_latency_mean=11
write_latency_min=2
write_latency_mean=82
read_fault_probability=5%
write_fault_probability=3%
crash_probability=0.01%
crash_stability=176 ticks
restart_probability=1%
restart_stability=545 ticks

Stack Trace:

thread panic: reached unreachable code
zig/lib/std/debug.zig:225:14: in std.debug.assert (simulator)
if (!ok) unreachable; // assertion failure
^
src/vsr/replica.zig:3562:27: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).repair_header (simulator)
assert(existing.op != header.op);
^
src/vsr/replica.zig:919:35: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).on_repair (simulator)
if (self.repair_header(message.header)) {
^
src/vsr/replica.zig:714:31: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).on_prepare (simulator)
self.on_repair(message);
^
src/vsr/replica.zig:525:44: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).on_message (simulator)
.prepare => self.on_prepare(message),
^
src/test/message_bus.zig:57:27: in test.message_bus.struct:55:35.wrapper (simulator)
on_message(@intToPtr(Context, @ptrToInt(_context)), message);
^
src/test/network.zig:167:41: in test.network.Network.deliver_message (simulator)
target_bus.on_message_callback.?(target_bus.on_message_context, message);
^
src/test/packet_simulator.zig:342:42: in test.packet_simulator.PacketSimulator(test.network.Packet).tick (simulator)
data.callback(data.packet, path);
^
src/simulator.zig:277:46: in main (simulator)
cluster.network.packet_simulator.tick(cluster.health);
^
zig/lib/std/start.zig:561:37: in std.start.callMain (simulator)
const result = root.main() catch |err| {
^
zig/lib/std/start.zig:495:12: in std.start.callMainWithArgs (simulator)
return @call(.{ .modifier = .always_inline }, callMain, .{});
^
zig/lib/std/start.zig:409:17: in std.start.posixCallMainAndExit (simulator)
std.os.exit(@call(.{ .modifier = .always_inline }, callMainWithArgs, .{ argc, argv, envp }));
^
zig/lib/std/start.zig:322:5: in std.start._start (simulator)
@call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{});
^
exit with signal: 6. Indicates a crash bug.


Tail of Debug Logs:
rsion = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 1: sending ping to replica 2: Header{ .checksum = 25000917487665338764926468982847220375, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 33090000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.ping
[debug] (vsr): 1: repair_timeout fired
[debug] (vsr): 1: repair_timeout reset
[err] (packet_simulator): dropped packet from=1 to=0.
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.commit
[debug] (replica): 0: on_message: view=0 status=Status.recovering Header{ .checksum = 82479382986955905386004704953578278698, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 280607819162479564554656084090657428767, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 3, .timestamp = 0, .size = 128, .replica = 2, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_message: waiting for journal to recover
[debug] (packet_simulator): delivering packet from=2 to=1
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.ping
[debug] (replica): 1: on_message: view=2 status=Status.normal Header{ .checksum = 335802669918276638328180047463461405437, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 33060000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: sending pong to replica 2: Header{ .checksum = 230489681182442948565241483801603984099, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 33060000000, .commit = 0, .timestamp = 33090000000, .size = 128, .replica = 1, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.pong
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=2 status=Status.normal Header{ .checksum = 25000917487665338764926468982847220375, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 33090000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: sending pong to replica 1: Header{ .checksum = 33420681393442512731934106888319541338, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 33090000000, .commit = 0, .timestamp = 33100000000, .size = 128, .replica = 2, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.pong
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.pong
[debug] (replica): 2: on_message: view=2 status=Status.normal Header{ .checksum = 230489681182442948565241483801603984099, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 33060000000, .commit = 0, .timestamp = 33090000000, .size = 128, .replica = 1, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (clock): 2: learn: replica=1 m0=33060000000 t1=33090000000 m2=33100000000 t2=33100000000 one_way_delay=20000000 asymmetric_delay=0 clock_offset=10000000
[err] (packet_simulator): dropped packet from=2 to=0.
[err] (packet_simulator): dropped packet from=2 to=1.
[debug] (journal): 0: recover_prepares: recovering slot=118
[debug] (journal): 0: recover_prepares: recovering slot=119
[debug] (journal): 0: recover_prepares: recovering slot=120
[debug] (journal): 0: recover_prepares: recovering slot=121
[debug] (vsr): 72925076024182917858532624641142997607: request_timeout fired
[debug] (vsr): 72925076024182917858532624641142997607: request_timeout backing off
[debug] (vsr): 72925076024182917858532624641142997607: request_timeout after=654..805 (rtt=30 min=10 max=1000 attempts=10)
[debug] (client): 72925076024182917858532624641142997607: on_request_timeout: resending request=0 checksum=17714069835035191794731308703332901407
[debug] (client): 72925076024182917858532624641142997607: sending request to replica 1: Header{ .checksum = 17714069835035191794731308703332901407, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 72925076024182917858532624641142997607, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .client = 72925076024182917858532624641142997607 } > Process{ .replica = 1 }: Command.request
[debug] (packet_simulator): delivering packet from=11 to=1
[debug] (network): deliver_message: Process{ .client = 72925076024182917858532624641142997607 } > Process{ .replica = 1 }: Command.request
[debug] (replica): 1: on_message: view=2 status=Status.normal Header{ .checksum = 17714069835035191794731308703332901407, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 72925076024182917858532624641142997607, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_request: ignoring (follower, register)
[debug] (journal): 0: recover_prepares: recovering slot=122
[debug] (journal): 0: recover_prepares: recovering slot=123
[debug] (vsr): 2: repair_timeout fired
[debug] (vsr): 2: repair_timeout reset
[debug] (vsr): 1: repair_timeout fired
[debug] (vsr): 1: repair_timeout reset
[debug] (journal): 0: recover_prepares: recovering slot=124
[debug] (journal): 0: recover_prepares: recovering slot=125
[debug] (journal): 0: recover_prepares: recovering slot=126
[debug] (journal): 0: recover_prepares: recovering slot=127
[debug] (journal): 0: recover_slot: recovered slot=0 label=@n decision=eql
[debug] (journal): 0: recover_slot: recovered slot=1 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=2 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=3 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=4 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=5 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=6 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=7 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=8 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=9 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=10 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=11 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=12 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=13 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=14 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=15 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=16 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=17 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=18 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=19 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=20 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=21 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=22 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=23 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=24 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=25 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=26 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=27 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=28 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=29 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=30 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=31 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=32 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=33 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=34 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=35 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=36 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=37 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=38 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=39 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=40 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=41 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=42 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=43 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=44 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=45 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=46 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=47 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=48 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=49 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=50 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=51 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=52 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=53 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=54 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=55 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=56 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=57 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=58 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=59 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=60 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=61 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=62 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=63 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=64 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=65 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=66 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=67 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=68 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=69 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=70 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=71 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=72 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=73 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=74 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=75 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=76 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=77 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=78 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=79 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=80 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=81 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=82 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=83 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=84 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=85 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=86 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=87 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=88 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=89 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=90 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=91 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=92 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=93 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=94 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=95 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=96 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=97 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=98 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=99 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=100 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=101 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=102 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=103 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=104 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=105 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=106 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=107 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=108 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=109 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=110 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=111 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=112 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=113 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=114 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=115 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=116 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=117 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=118 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=119 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=120 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=121 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=122 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=123 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=124 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=125 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=126 label=@j decision=nil
[debug] (journal): 0: recover_slot: recovered slot=127 label=@j decision=nil
[debug] (journal): 0: recover_slots: dirty=0 faulty=0
[debug] (replica): 0: transition_to_normal_from_recovering_status: view=0 leader
[debug] (vsr): 0: ping_timeout started
[debug] (vsr): 0: commit_timeout started
[debug] (vsr): 0: repair_timeout started
[debug] (vsr): 0: recovery_timeout stopped
[debug] (vsr): 2: ping_timeout fired
[debug] (vsr): 2: ping_timeout reset
[debug] (replica): 2: sending ping to replica 0: Header{ .checksum = 245929035965067344276764355088057875149, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 34060000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 2: sending ping to replica 1: Header{ .checksum = 245929035965067344276764355088057875149, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 34060000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.ping
[debug] (vsr): 2: commit_timeout fired
[debug] (vsr): 2: commit_timeout reset
[debug] (replica): 2: sending commit to replica 0: Header{ .checksum = 82479382986955905386004704953578278698, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 280607819162479564554656084090657428767, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 3, .timestamp = 0, .size = 128, .replica = 2, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.commit
[debug] (replica): 2: sending commit to replica 1: Header{ .checksum = 82479382986955905386004704953578278698, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 280607819162479564554656084090657428767, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 3, .timestamp = 0, .size = 128, .replica = 2, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.commit
[debug] (vsr): 2: repair_timeout fired
[debug] (vsr): 2: repair_timeout reset
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.commit
[debug] (replica): 0: on_message: view=0 status=Status.normal Header{ .checksum = 82479382986955905386004704953578278698, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 280607819162479564554656084090657428767, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 3, .timestamp = 0, .size = 128, .replica = 2, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: view_jump: waiting to jump to newer view
[debug] (replica): 0: view_jump: requesting start_view message
[debug] (replica): 0: sending request_start_view to replica 2: Header{ .checksum = 79686325546817063055692460609047931228, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request_start_view, .operation = Operation.reserved, .version = 0 }
[info] (replica): 0: send_message_to_replica: TODO request_start_view
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request_start_view
[debug] (replica): 0: on_commit: ignoring (newer view)
[err] (packet_simulator): dropped packet from=2 to=1.
[err] (packet_simulator): dropped packet from=2 to=1.
[debug] (vsr): 1: ping_timeout fired
[debug] (vsr): 1: ping_timeout reset
[debug] (replica): 1: sending ping to replica 0: Header{ .checksum = 267756780059743919244616819783956089580, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 34090000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 1: sending ping to replica 2: Header{ .checksum = 267756780059743919244616819783956089580, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 34090000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.ping
[debug] (vsr): 1: repair_timeout fired
[debug] (vsr): 1: repair_timeout reset
[debug] (clock): 2: synchronized: truechimers=2/3 clock_offset=0ns..0ns accuracy=0ns
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request_start_view
[debug] (replica): 2: on_message: view=2 status=Status.normal Header{ .checksum = 79686325546817063055692460609047931228, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request_start_view, .operation = Operation.reserved, .version = 0 }
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=3 dest.len=32 copied=4
[debug] (replica): 2: sending start_view to replica 0: Header{ .checksum = 28279923589702944203449731667594605074, .checksum_body = 55491794307953600588024889805835455022, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 3, .commit = 3, .timestamp = 0, .size = 640, .replica = 2, .command = Command.start_view, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view
[err] (packet_simulator): dropped packet from=1 to=0.
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=2 status=Status.normal Header{ .checksum = 267756780059743919244616819783956089580, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 34090000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: sending pong to replica 1: Header{ .checksum = 225964967945852066741807995453322428088, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 34090000000, .commit = 0, .timestamp = 34100000000, .size = 128, .replica = 2, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.pong
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 0: on_message: view=0 status=Status.normal Header{ .checksum = 245929035965067344276764355088057875149, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 34060000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: sending pong to replica 2: Header{ .checksum = 48618720842963728146389215788125294561, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 34060000000, .commit = 0, .timestamp = 34100000000, .size = 128, .replica = 0, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.pong
[err] (packet_simulator): dropped packet from=2 to=0.
[debug] (packet_simulator): delivering packet from=2 to=1
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.pong
[debug] (replica): 1: on_message: view=2 status=Status.normal Header{ .checksum = 225964967945852066741807995453322428088, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 34090000000, .commit = 0, .timestamp = 34100000000, .size = 128, .replica = 2, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (clock): 1: learn: replica=2 m0=34090000000 t1=34100000000 m2=34110000000 t2=34110000000 one_way_delay=10000000 asymmetric_delay=0 clock_offset=0
[debug] (clock): 1: synchronized: truechimers=2/3 clock_offset=0ns..0ns accuracy=0ns
[err] (packet_simulator): dropped packet from=0 to=2.
[debug] (vsr): 136230913011779207600103268024900035136: request_timeout fired
[debug] (vsr): 136230913011779207600103268024900035136: request_timeout backing off
[debug] (vsr): 136230913011779207600103268024900035136: request_timeout after=283..371 (rtt=30 min=10 max=1000 attempts=10)
[debug] (client): 136230913011779207600103268024900035136: on_request_timeout: resending request=0 checksum=141169034413528437886527447173478256344
[debug] (client): 136230913011779207600103268024900035136: sending request to replica 1: Header{ .checksum = 141169034413528437886527447173478256344, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 136230913011779207600103268024900035136, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .client = 136230913011779207600103268024900035136 } > Process{ .replica = 1 }: Command.request
[err] (packet_simulator): dropped packet from=19 to=1.
[debug] (vsr): 172990577333784703836240163392524942135: request_timeout fired
[debug] (vsr): 172990577333784703836240163392524942135: request_timeout backing off
[debug] (vsr): 172990577333784703836240163392524942135: request_timeout after=770..899 (rtt=30 min=10 max=1000 attempts=11)
[debug] (client): 172990577333784703836240163392524942135: on_request_timeout: resending request=0 checksum=149197670650058519946663842157426746886
[debug] (client): 172990577333784703836240163392524942135: sending request to replica 2: Header{ .checksum = 149197670650058519946663842157426746886, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 172990577333784703836240163392524942135, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .client = 172990577333784703836240163392524942135 } > Process{ .replica = 2 }: Command.request
[debug] (packet_simulator): delivering packet from=10 to=2
[debug] (network): deliver_message: Process{ .client = 172990577333784703836240163392524942135 } > Process{ .replica = 2 }: Command.request
[debug] (replica): 2: on_message: view=2 status=Status.normal Header{ .checksum = 149197670650058519946663842157426746886, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 172990577333784703836240163392524942135, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (replica): 2: on_request: replying to duplicate request
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .client = 172990577333784703836240163392524942135 }: Command.reply
[err] (packet_simulator): dropped packet from=2 to=10.
[debug] (vsr): 122183239364546111596049911814033996109: request_timeout fired
[debug] (vsr): 122183239364546111596049911814033996109: request_timeout backing off
[debug] (vsr): 122183239364546111596049911814033996109: request_timeout after=739..248 (rtt=30 min=10 max=1000 attempts=12)
[debug] (client): 122183239364546111596049911814033996109: on_request_timeout: resending request=0 checksum=127735885776624070072672702280333006124
[debug] (client): 122183239364546111596049911814033996109: sending request to replica 0: Header{ .checksum = 127735885776624070072672702280333006124, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 122183239364546111596049911814033996109, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .client = 122183239364546111596049911814033996109 } > Process{ .replica = 0 }: Command.request
[debug] (packet_simulator): delivering packet from=16 to=0
[debug] (network): deliver_message: Process{ .client = 122183239364546111596049911814033996109 } > Process{ .replica = 0 }: Command.request
[debug] (replica): 0: on_message: view=0 status=Status.normal Header{ .checksum = 127735885776624070072672702280333006124, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 122183239364546111596049911814033996109, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (replica): 0: on_request: new session
[err] (replica): 0: on_request: dropping (clock not synchronized)
[debug] (vsr): 0: repair_timeout fired
[debug] (vsr): 0: repair_timeout reset
[debug] (vsr): 2: repair_timeout fired
[debug] (vsr): 2: repair_timeout reset
[debug] (vsr): 1: repair_timeout fired
[debug] (vsr): 1: repair_timeout reset
[debug] (vsr): 0: ping_timeout fired
[debug] (vsr): 0: ping_timeout reset
[debug] (replica): 0: sending ping to replica 1: Header{ .checksum = 326074057305105820858780145699743327580, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 34920000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.ping
[debug] (replica): 0: sending ping to replica 2: Header{ .checksum = 326074057305105820858780145699743327580, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 34920000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.ping
[debug] (vsr): 0: commit_timeout fired
[debug] (vsr): 0: commit_timeout reset
[debug] (replica): 0: sending commit to replica 1: Header{ .checksum = 96499579904055674132432461151983520881, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 41826076010321932376190093941186211105, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.commit
[debug] (replica): 0: sending commit to replica 2: Header{ .checksum = 96499579904055674132432461151983520881, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 41826076010321932376190093941186211105, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.commit
[debug] (vsr): 0: repair_timeout fired
[debug] (vsr): 0: repair_timeout reset
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.ping
[debug] (replica): 1: on_message: view=2 status=Status.normal Header{ .checksum = 326074057305105820858780145699743327580, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 34920000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: sending pong to replica 0: Header{ .checksum = 15729035882880979765855877513123419008, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 34920000000, .commit = 0, .timestamp = 34920000000, .size = 128, .replica = 1, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.pong
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.commit
[debug] (replica): 1: on_message: view=2 status=Status.normal Header{ .checksum = 96499579904055674132432461151983520881, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 41826076010321932376190093941186211105, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_commit: ignoring (older view)
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=2 status=Status.normal Header{ .checksum = 326074057305105820858780145699743327580, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 34920000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: sending pong to replica 0: Header{ .checksum = 5489297355563553136042466653915209005, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 34920000000, .commit = 0, .timestamp = 34930000000, .size = 128, .replica = 2, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.pong
[debug] (packet_simulator): replayed packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.pong
[debug] (replica): 0: on_message: view=0 status=Status.normal Header{ .checksum = 15729035882880979765855877513123419008, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 34920000000, .commit = 0, .timestamp = 34920000000, .size = 128, .replica = 1, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (clock): 0: learn: replica=1 m0=34920000000 t1=34920000000 m2=34930000000 t2=34930000000 one_way_delay=5000000 asymmetric_delay=0 clock_offset=-5000000
[debug] (clock): 0: synchronized: truechimers=2/3 clock_offset=0ns..0ns accuracy=0ns
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.pong
[debug] (replica): 0: on_message: view=0 status=Status.normal Header{ .checksum = 5489297355563553136042466653915209005, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 34920000000, .commit = 0, .timestamp = 34930000000, .size = 128, .replica = 2, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[warn] (clock): 0: learn: m0=34920000000 < window.monotonic=34940000000
[debug] (packet_simulator): replayed packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.pong
[debug] (replica): 0: on_message: view=0 status=Status.normal Header{ .checksum = 15729035882880979765855877513123419008, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 34920000000, .commit = 0, .timestamp = 34920000000, .size = 128, .replica = 1, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[warn] (clock): 0: learn: m0=34920000000 < window.monotonic=34940000000
[debug] (packet_simulator): replayed packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.pong
[debug] (replica): 0: on_message: view=0 status=Status.normal Header{ .checksum = 15729035882880979765855877513123419008, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 34920000000, .commit = 0, .timestamp = 34920000000, .size = 128, .replica = 1, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[warn] (clock): 0: learn: m0=34920000000 < window.monotonic=34940000000
[err] (packet_simulator): dropped packet from=1 to=0.
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.commit
[debug] (replica): 2: on_message: view=2 status=Status.normal Header{ .checksum = 96499579904055674132432461151983520881, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 41826076010321932376190093941186211105, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_commit: ignoring (older view)
[debug] (vsr): 144975797817200414350582868343606763607: request_timeout fired
[debug] (vsr): 144975797817200414350582868343606763607: request_timeout backing off
[debug] (vsr): 144975797817200414350582868343606763607: request_timeout after=1002..978 (rtt=30 min=10 max=1000 attempts=10)
[debug] (client): 144975797817200414350582868343606763607: on_request_timeout: resending request=0 checksum=309298773475437319325703147009506610946
[debug] (client): 144975797817200414350582868343606763607: sending request to replica 1: Header{ .checksum = 309298773475437319325703147009506610946, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 144975797817200414350582868343606763607, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .client = 144975797817200414350582868343606763607 } > Process{ .replica = 1 }: Command.request
[err] (packet_simulator): dropped packet from=12 to=1.
[debug] (vsr): 2: ping_timeout fired
[debug] (vsr): 2: ping_timeout reset
[debug] (replica): 2: sending ping to replica 0: Header{ .checksum = 293397322717516068103316035459996903512, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 35060000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 2: sending ping to replica 1: Header{ .checksum = 293397322717516068103316035459996903512, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 35060000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.ping
[debug] (vsr): 2: commit_timeout fired
[debug] (vsr): 2: commit_timeout reset
[debug] (replica): 2: sending commit to replica 0: Header{ .checksum = 82479382986955905386004704953578278698, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 280607819162479564554656084090657428767, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 3, .timestamp = 0, .size = 128, .replica = 2, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.commit
[debug] (replica): 2: sending commit to replica 1: Header{ .checksum = 82479382986955905386004704953578278698, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 280607819162479564554656084090657428767, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 3, .timestamp = 0, .size = 128, .replica = 2, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.commit
[debug] (vsr): 2: repair_timeout fired
[debug] (vsr): 2: repair_timeout reset
[err] (packet_simulator): dropped packet from=2 to=0.
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.commit
[debug] (replica): 0: on_message: view=0 status=Status.normal Header{ .checksum = 82479382986955905386004704953578278698, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 280607819162479564554656084090657428767, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 3, .timestamp = 0, .size = 128, .replica = 2, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: view_jump: waiting to jump to newer view
[debug] (replica): 0: view_jump: requesting start_view message
[debug] (replica): 0: sending request_start_view to replica 2: Header{ .checksum = 79686325546817063055692460609047931228, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request_start_view, .operation = Operation.reserved, .version = 0 }
[info] (replica): 0: send_message_to_replica: TODO request_start_view
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request_start_view
[debug] (replica): 0: on_commit: ignoring (newer view)
[debug] (packet_simulator): delivering packet from=2 to=1
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.commit
[debug] (replica): 1: on_message: view=2 status=Status.normal Header{ .checksum = 82479382986955905386004704953578278698, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 280607819162479564554656084090657428767, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 3, .timestamp = 0, .size = 128, .replica = 2, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_commit: checksum verified
[debug] (vsr): 1: normal_status_timeout reset
[debug] (vsr): 1: ping_timeout fired
[debug] (vsr): 1: ping_timeout reset
[debug] (replica): 1: sending ping to replica 0: Header{ .checksum = 298718331310626527851561052066245309240, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 35090000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 1: sending ping to replica 2: Header{ .checksum = 298718331310626527851561052066245309240, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 35090000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.ping
[debug] (vsr): 1: repair_timeout fired
[debug] (vsr): 1: repair_timeout reset
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 0: on_message: view=0 status=Status.normal Header{ .checksum = 298718331310626527851561052066245309240, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 35090000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: sending pong to replica 1: Header{ .checksum = 168084015796447100025732410724114163180, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 35090000000, .commit = 0, .timestamp = 35090000000, .size = 128, .replica = 0, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.pong
[debug] (packet_simulator): delivering packet from=2 to=1
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.ping
[debug] (replica): 1: on_message: view=2 status=Status.normal Header{ .checksum = 293397322717516068103316035459996903512, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 35060000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: sending pong to replica 2: Header{ .checksum = 197075378857474831813776453281755279849, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 35060000000, .commit = 0, .timestamp = 35090000000, .size = 128, .replica = 1, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.pong
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=2 status=Status.normal Header{ .checksum = 298718331310626527851561052066245309240, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 35090000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: sending pong to replica 1: Header{ .checksum = 75018047071671311702737768879895118921, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 35090000000, .commit = 0, .timestamp = 35100000000, .size = 128, .replica = 2, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.pong
[err] (packet_simulator): dropped packet from=0 to=2.
[err] (packet_simulator): dropped packet from=2 to=1.
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.pong
[debug] (replica): 1: on_message: view=2 status=Status.normal Header{ .checksum = 168084015796447100025732410724114163180, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 35090000000, .commit = 0, .timestamp = 35090000000, .size = 128, .replica = 0, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (clock): 1: learn: replica=0 m0=35090000000 t1=35090000000 m2=35150000000 t2=35150000000 one_way_delay=30000000 asymmetric_delay=0 clock_offset=-30000000
[debug] (packet_simulator): replayed packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.pong
[debug] (replica): 2: on_message: view=2 status=Status.normal Header{ .checksum = 197075378857474831813776453281755279849, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 35060000000, .commit = 0, .timestamp = 35090000000, .size = 128, .replica = 1, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (clock): 2: learn: replica=1 m0=35060000000 t1=35090000000 m2=35150000000 t2=35150000000 one_way_delay=45000000 asymmetric_delay=25000000 clock_offset=10000000
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.pong
[debug] (replica): 2: on_message: view=2 status=Status.normal Header{ .checksum = 197075378857474831813776453281755279849, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 35060000000, .commit = 0, .timestamp = 35090000000, .size = 128, .replica = 1, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (clock): 2: learn: replica=1 m0=35060000000 t1=35090000000 m2=35170000000 t2=35170000000 one_way_delay=55000000 asymmetric_delay=35000000 clock_offset=10000000
[debug] (vsr): 3287866257359059985775056949272098863: request_timeout fired
[debug] (vsr): 3287866257359059985775056949272098863: request_timeout backing off
[debug] (vsr): 3287866257359059985775056949272098863: request_timeout after=474..207 (rtt=30 min=10 max=1000 attempts=12)
[debug] (client): 3287866257359059985775056949272098863: on_request_timeout: resending request=0 checksum=303437202224966091401053667077016530339
[debug] (client): 3287866257359059985775056949272098863: sending request to replica 0: Header{ .checksum = 303437202224966091401053667077016530339, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 3287866257359059985775056949272098863, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .client = 3287866257359059985775056949272098863 } > Process{ .replica = 0 }: Command.request
[debug] (packet_simulator): delivering packet from=8 to=0
[debug] (network): deliver_message: Process{ .client = 3287866257359059985775056949272098863 } > Process{ .replica = 0 }: Command.request
[debug] (replica): 0: on_message: view=0 status=Status.normal Header{ .checksum = 303437202224966091401053667077016530339, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 3287866257359059985775056949272098863, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (replica): 0: on_request: new session
[debug] (replica): 0: on_request: request 303437202224966091401053667077016530339
[debug] (replica): 0: on_request: prepare 95113813869640709928281012229104923518
[debug] (vsr): 0: prepare_timeout started
[debug] (replica): 0: on_prepare: advancing: op=0..1 checksum=41826076010321932376190093941186211105..95113813869640709928281012229104923518
[debug] (journal): 0: set_header_as_dirty: op=1 checksum=95113813869640709928281012229104923518
[debug] (replica): 0: replicate: replicating to replica 1
[debug] (replica): 0: sending prepare to replica 1: Header{ .checksum = 95113813869640709928281012229104923518, .checksum_body = 98081761841810822994326224503485043631, .parent = 41826076010321932376190093941186211105, .client = 3287866257359059985775056949272098863, .context = 303437202224966091401053667077016530339, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 1, .commit = 0, .timestamp = 35310000000, .size = 128, .replica = 0, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.prepare
[debug] (replica): 0: append: appending to journal
[debug] (journal): 0: write: view=0 op=1 len=128: 95113813869640709928281012229104923518 starting
[debug] (journal): 0: write_sectors: offset=1064960 len=4096 locked
[debug] (packet_simulator): replayed packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.prepare
[debug] (replica): 1: on_message: view=2 status=Status.normal Header{ .checksum = 95113813869640709928281012229104923518, .checksum_body = 98081761841810822994326224503485043631, .parent = 41826076010321932376190093941186211105, .client = 3287866257359059985775056949272098863, .context = 303437202224966091401053667077016530339, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 1, .commit = 0, .timestamp = 35310000000, .size = 128, .replica = 0, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_prepare: ignoring (repair)

Time:
2022-07-13 12:47:30.821866645 +0000 UTC

when i open tigerbeetle.com with iran ip it respond with 406 code. loads correctly when i use VPN

TigerBeetle currently supports the following operations;

• create_accounts
• create_transfers
• commit_transfers
• lookup_accounts

There is a requirement to lookup an existing transfer via id (lookup_transfers(u128).
Although the state_machine already supports transfer lookups via get_transfer, the operation is not exposed.

Furthermore the https://github.com/coilhq/tigerbeetle-node repo needs to be updated to support transfer retrieval.

Bug Type:
correctness

Seed:
3406191993184487393

Commit Hash:
01494c5

Parameters:
[info] (state_checker):
SEED=3406191993184487393

replicas=3
clients=6
request_probability=41%
idle_on_probability=7%
idle_off_probability=14%
one_way_delay_mean=11 ticks
one_way_delay_min=1 ticks
packet_loss_probability=19%
path_maximum_capacity=17 messages
path_clog_duration_mean=419 ticks
path_clog_probability=1%
packet_replay_probability=33%
partition_mode=PartitionMode.uniform_size
partition_probability=1%
unpartition_probability=1%
partition_stability=152 ticks
unpartition_stability=0 ticks
read_latency_min=2
read_latency_mean=8
write_latency_min=0
write_latency_mean=7
read_fault_probability=6%
write_fault_probability=2%
crash_probability=0.01%
crash_stability=806 ticks
restart_probability=1%
restart_stability=143 ticks

Stack Trace:

exit with code: 129


Tail of Debug Logs:
luster = 0, .epoch = 0, .view = 92, .op = 0, .commit = 0, .timestamp = 0, .size = 770141, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (network): send_message: Process{ .client = 27344200203525815038850809559365994372 } > Process{ .replica = 1 }: Command.request
[debug] (packet_simulator): delivering packet from=5 to=1
[debug] (network): deliver_message: Process{ .client = 27344200203525815038850809559365994372 } > Process{ .replica = 1 }: Command.request
[debug] (replica): 1: on_message: view=93 status=Status.view_change Header{ .checksum = 220204636111232530118292884133430645948, .checksum_body = 50245981828837326439766196754815785902, .parent = 168277661143012477177271820117339398903, .client = 27344200203525815038850809559365994372, .context = 2, .request = 10, .cluster = 0, .epoch = 0, .view = 92, .op = 0, .commit = 0, .timestamp = 0, .size = 770141, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_request: ignoring (Status.view_change)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=6 clogged for ticks=219
[err] (packet_simulator): dropped packet from=2 to=1.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=7 clogged for ticks=348
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=3 clogged for ticks=21
[err] (packet_simulator): dropped packet from=1 to=2.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=6 clogged for ticks=206
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=6 clogged for ticks=246
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=1 clogged for ticks=406
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=1 clogged for ticks=202
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=6 clogged for ticks=242
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=3 clogged for ticks=552
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=0 clogged for ticks=118
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=7 clogged for ticks=493
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=3 clogged for ticks=668
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=3 clogged for ticks=185
[debug] (vsr): 1: view_change_status_timeout fired
[debug] (replica): 1: transition_to_view_change_status: view=93..94
[debug] (vsr): 1: ping_timeout stopped
[debug] (vsr): 1: commit_timeout stopped
[debug] (vsr): 1: normal_status_timeout stopped
[debug] (vsr): 1: view_change_status_timeout started
[debug] (vsr): 1: view_change_message_timeout started
[debug] (vsr): 1: repair_timeout stopped
[debug] (vsr): 1: prepare_timeout stopped
[debug] (replica): 1: reset 0 do_view_change message(s) from view=null
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 192301657158625697281087330944801877776, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 192301657158625697281087330944801877776, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (vsr): 2: view_change_message_timeout fired
[debug] (vsr): 2: view_change_message_timeout reset
[debug] (replica): 2: sending start_view_change to replica 0: Header{ .checksum = 16745914531903793378893340110805526981, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 93, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 1: Header{ .checksum = 16745914531903793378893340110805526981, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 93, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=61 dest.len=32 copied=32
[debug] (replica): 2: sending do_view_change to replica 0: Header{ .checksum = 301583685648147837809554740241321055654, .checksum_body = 294810028318817366549588706410439499791, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 93, .op = 61, .commit = 58, .timestamp = 92, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (vsr): 0: view_change_message_timeout fired
[debug] (vsr): 0: view_change_message_timeout reset
[debug] (replica): 0: sending start_view_change to replica 1: Header{ .checksum = 273704761360968411045177967179755370274, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 93, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 2: Header{ .checksum = 273704761360968411045177967179755370274, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 93, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=3 clogged for ticks=702
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=8 clogged for ticks=774
[err] (packet_simulator): unpartitioned network: partition={ false, false, false }
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=93 status=Status.view_change Header{ .checksum = 192301657158625697281087330944801877776, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: view_jump: jumping to view change
[debug] (replica): 2: transition_to_view_change_status: view=93..94
[debug] (vsr): 2: ping_timeout stopped
[debug] (vsr): 2: commit_timeout stopped
[debug] (vsr): 2: normal_status_timeout stopped
[debug] (vsr): 2: view_change_status_timeout started
[debug] (vsr): 2: view_change_message_timeout started
[debug] (vsr): 2: repair_timeout stopped
[debug] (vsr): 2: prepare_timeout stopped
[debug] (replica): 2: reset 0 do_view_change message(s) from view=null
[debug] (replica): 2: sending start_view_change to replica 0: Header{ .checksum = 101388122876582760640688170375567333392, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 1: Header{ .checksum = 101388122876582760640688170375567333392, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 2: on_start_view_change: 1 message(s)
[debug] (replica): 2: on_start_view_change: view=94 quorum received
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=61 dest.len=32 copied=32
[debug] (replica): 2: sending do_view_change to replica 1: Header{ .checksum = 339694816374994560849231454220807307598, .checksum_body = 294810028318817366549588706410439499791, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 61, .commit = 58, .timestamp = 92, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.do_view_change
[err] (packet_simulator): dropped packet from=2 to=0.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=6 clogged for ticks=685
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=8 clogged for ticks=234
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=6 clogged for ticks=277
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=94 status=Status.view_change Header{ .checksum = 273704761360968411045177967179755370274, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 93, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: ignoring (older view)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=5 clogged for ticks=558
[debug] (packet_simulator): replayed packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=93 status=Status.view_change Header{ .checksum = 16745914531903793378893340110805526981, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 93, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: 1 message(s)
[debug] (replica): 0: on_start_view_change: view=93 quorum received
[debug] (journal): 0: copy_latest_headers_between: op_min=0 op_max=61 dest.len=32 copied=32
[debug] (replica): 0: sending do_view_change to replica 0: Header{ .checksum = 138187932096741066634130295156851938113, .checksum_body = 177310321252202099528854464168395478546, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 93, .op = 61, .commit = 58, .timestamp = 92, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_message: view=93 status=Status.view_change Header{ .checksum = 138187932096741066634130295156851938113, .checksum_body = 177310321252202099528854464168395478546, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 93, .op = 61, .commit = 58, .timestamp = 92, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: 1 message(s)
[debug] (replica): 0: on_do_view_change: waiting for quorum
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=6 clogged for ticks=1606
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=93 status=Status.view_change Header{ .checksum = 16745914531903793378893340110805526981, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 93, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=2)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=3 clogged for ticks=136
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=7 clogged for ticks=27
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=3 clogged for ticks=917
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=8 clogged for ticks=123
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=4 clogged for ticks=723
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=3 clogged for ticks=189
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=7 clogged for ticks=1134
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=0 clogged for ticks=78
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=3 clogged for ticks=489
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=7 clogged for ticks=68
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=8 clogged for ticks=473
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=6 clogged for ticks=767
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=93 status=Status.view_change Header{ .checksum = 101388122876582760640688170375567333392, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: view_jump: jumping to view change
[debug] (replica): 0: transition_to_view_change_status: view=93..94
[debug] (vsr): 0: ping_timeout stopped
[debug] (vsr): 0: commit_timeout stopped
[debug] (vsr): 0: normal_status_timeout stopped
[debug] (vsr): 0: view_change_status_timeout started
[debug] (vsr): 0: view_change_message_timeout started
[debug] (vsr): 0: repair_timeout stopped
[debug] (vsr): 0: prepare_timeout stopped
[debug] (replica): 0: reset 1 do_view_change message(s) from view=93
[debug] (replica): 0: sending start_view_change to replica 1: Header{ .checksum = 281362512409244457737336051959983193538, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 2: Header{ .checksum = 281362512409244457737336051959983193538, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 0: on_start_view_change: 1 message(s)
[debug] (replica): 0: on_start_view_change: view=94 quorum received
[debug] (journal): 0: copy_latest_headers_between: op_min=0 op_max=61 dest.len=32 copied=32
[debug] (replica): 0: sending do_view_change to replica 1: Header{ .checksum = 124924144947466025800620679407211028394, .checksum_body = 177310321252202099528854464168395478546, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 61, .commit = 58, .timestamp = 92, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.do_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=3 clogged for ticks=41
[err] (packet_simulator): dropped packet from=0 to=1.
[debug] (packet_simulator): replayed packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=94 status=Status.view_change Header{ .checksum = 281362512409244457737336051959983193538, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: 1 message(s)
[debug] (replica): 1: on_start_view_change: view=94 quorum received
[debug] (journal): 1: copy_latest_headers_between: op_min=0 op_max=59 dest.len=32 copied=32
[debug] (replica): 1: sending do_view_change to replica 1: Header{ .checksum = 127713588745448440086015362436472397367, .checksum_body = 89583151691299355962746970047031894109, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 59, .commit = 57, .timestamp = 91, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_message: view=94 status=Status.view_change Header{ .checksum = 127713588745448440086015362436472397367, .checksum_body = 89583151691299355962746970047031894109, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 59, .commit = 57, .timestamp = 91, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_do_view_change: 1 message(s)
[debug] (replica): 1: on_do_view_change: waiting for quorum
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=94 status=Status.view_change Header{ .checksum = 281362512409244457737336051959983193538, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: ignoring (duplicate message replica=0)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=0 clogged for ticks=207
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=8 clogged for ticks=506
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=6 clogged for ticks=47
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=5 clogged for ticks=527
[debug] (vsr): 1: view_change_message_timeout fired
[debug] (vsr): 1: view_change_message_timeout reset
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 192301657158625697281087330944801877776, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 192301657158625697281087330944801877776, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=4 clogged for ticks=220
[debug] (vsr): 2: view_change_message_timeout fired
[debug] (vsr): 2: view_change_message_timeout reset
[debug] (replica): 2: sending start_view_change to replica 0: Header{ .checksum = 101388122876582760640688170375567333392, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 1: Header{ .checksum = 101388122876582760640688170375567333392, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=61 dest.len=32 copied=32
[debug] (replica): 2: sending do_view_change to replica 1: Header{ .checksum = 339694816374994560849231454220807307598, .checksum_body = 294810028318817366549588706410439499791, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 61, .commit = 58, .timestamp = 92, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.do_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=3 clogged for ticks=291
[err] (packet_simulator): dropped packet from=1 to=2.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=7 clogged for ticks=444
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=7 clogged for ticks=444
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=3 clogged for ticks=308
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=94 status=Status.view_change Header{ .checksum = 101388122876582760640688170375567333392, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=2)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=4 clogged for ticks=354
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=8 clogged for ticks=501
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=3 clogged for ticks=917
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=0 clogged for ticks=61
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=8 clogged for ticks=146
[debug] (vsr): 0: view_change_message_timeout fired
[debug] (vsr): 0: view_change_message_timeout reset
[debug] (replica): 0: sending start_view_change to replica 1: Header{ .checksum = 281362512409244457737336051959983193538, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 2: Header{ .checksum = 281362512409244457737336051959983193538, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (journal): 0: copy_latest_headers_between: op_min=0 op_max=61 dest.len=32 copied=32
[debug] (replica): 0: sending do_view_change to replica 1: Header{ .checksum = 124924144947466025800620679407211028394, .checksum_body = 177310321252202099528854464168395478546, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 61, .commit = 58, .timestamp = 92, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.do_view_change
[err] (packet_simulator): dropped packet from=0 to=2.
[err] (packet_simulator): dropped packet from=0 to=2.
[debug] (packet_simulator): replayed packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 110559030372813900348458033142393109091, .checksum_body = 98081761841810822994326224503485043631, .parent = 166981343971631773235589951944186428934, .client = 65459479704107910451097262802840693083, .context = 5, .request = 9, .cluster = 0, .epoch = 0, .view = 88, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (replica): 2: on_request: ignoring (Status.view_change)
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 145528379763305517215915483903133354529, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 92, .op = 552970000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: sending pong to replica 0: Header{ .checksum = 236246359108048086270733592869245114217, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 552970000000, .commit = 0, .timestamp = 558680000000, .size = 128, .replica = 2, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.pong
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 110559030372813900348458033142393109091, .checksum_body = 98081761841810822994326224503485043631, .parent = 166981343971631773235589951944186428934, .client = 65459479704107910451097262802840693083, .context = 5, .request = 9, .cluster = 0, .epoch = 0, .view = 88, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (replica): 2: on_request: ignoring (Status.view_change)
[debug] (packet_simulator): replayed packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request_headers
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 304612445240698770682799895737061095427, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 92, .op = 60, .commit = 59, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request_headers, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_request_headers: ignoring (older view)
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 147514158771500148669189970814806307260, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 92, .op = 553970000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: sending pong to replica 0: Header{ .checksum = 179957068742969649469834717697056140729, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 553970000000, .commit = 0, .timestamp = 558680000000, .size = 128, .replica = 2, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.pong
[debug] (packet_simulator): replayed packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request_headers
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 304612445240698770682799895737061095427, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 92, .op = 60, .commit = 59, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request_headers, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_request_headers: ignoring (older view)
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 152620718764307361834397912754702521998, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 92, .op = 554970000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: sending pong to replica 0: Header{ .checksum = 327141289120816198267031813264048684253, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 554970000000, .commit = 0, .timestamp = 558680000000, .size = 128, .replica = 2, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.pong
[err] (packet_simulator): dropped packet from=0 to=2.
[err] (packet_simulator): dropped packet from=0 to=2.
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 273704761360968411045177967179755370274, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 93, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (older view)
[err] (packet_simulator): dropped packet from=0 to=2.
[err] (packet_simulator): dropped packet from=0 to=2.
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 273704761360968411045177967179755370274, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 93, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (older view)
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 281362512409244457737336051959983193538, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: 2 message(s)
[debug] (replica): 2: on_start_view_change: ignoring (quorum received already)
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.pong
[debug] (replica): 0: on_message: view=94 status=Status.view_change Header{ .checksum = 179957068742969649469834717697056140729, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 553970000000, .commit = 0, .timestamp = 558680000000, .size = 128, .replica = 2, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (clock): 0: learn: replica=2 m0=553970000000 t1=558680000000 m2=558690000000 t2=558690000000 one_way_delay=2360000000 asymmetric_delay=-2250000000 clock_offset=100000000
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=7 clogged for ticks=114
[debug] (clock): 0: synchronized: truechimers=2/3 clock_offset=0ns..0ns accuracy=0ns
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=0 clogged for ticks=617
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=2 clogged for ticks=1729
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.pong
[debug] (replica): 0: on_message: view=94 status=Status.view_change Header{ .checksum = 236246359108048086270733592869245114217, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 552970000000, .commit = 0, .timestamp = 558680000000, .size = 128, .replica = 2, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[warn] (clock): 0: learn: m0=552970000000 < window.monotonic=558700000000
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=94 status=Status.view_change Header{ .checksum = 281362512409244457737336051959983193538, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: ignoring (duplicate message replica=0)
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.pong
[debug] (replica): 0: on_message: view=94 status=Status.view_change Header{ .checksum = 327141289120816198267031813264048684253, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 554970000000, .commit = 0, .timestamp = 558680000000, .size = 128, .replica = 2, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[warn] (clock): 0: learn: m0=554970000000 < window.monotonic=558700000000
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 281362512409244457737336051959983193538, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=0)
[err] (packet_simulator): dropped packet from=0 to=1.
[debug] (vsr): 1: view_change_message_timeout fired
[debug] (vsr): 1: view_change_message_timeout reset
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 192301657158625697281087330944801877776, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 192301657158625697281087330944801877776, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=0 clogged for ticks=107
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request_headers
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 304612445240698770682799895737061095427, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 92, .op = 60, .commit = 59, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request_headers, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_request_headers: ignoring (older view)
[debug] (packet_simulator): replayed packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 192301657158625697281087330944801877776, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=6 clogged for ticks=419
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 110559030372813900348458033142393109091, .checksum_body = 98081761841810822994326224503485043631, .parent = 166981343971631773235589951944186428934, .client = 65459479704107910451097262802840693083, .context = 5, .request = 9, .cluster = 0, .epoch = 0, .view = 88, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (replica): 2: on_request: ignoring (Status.view_change)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=4 clogged for ticks=375
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=5 clogged for ticks=275
[debug] (vsr): 2: view_change_message_timeout fired
[debug] (vsr): 2: view_change_message_timeout reset
[debug] (replica): 2: sending start_view_change to replica 0: Header{ .checksum = 101388122876582760640688170375567333392, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 1: Header{ .checksum = 101388122876582760640688170375567333392, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=61 dest.len=32 copied=32
[debug] (replica): 2: sending do_view_change to replica 1: Header{ .checksum = 339694816374994560849231454220807307598, .checksum_body = 294810028318817366549588706410439499791, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 61, .commit = 58, .timestamp = 92, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.do_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=3 clogged for ticks=1332
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=0 clogged for ticks=729
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=7 clogged for ticks=232
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=3 clogged for ticks=334
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=5 clogged for ticks=380
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=4 clogged for ticks=182
[debug] (packet_simulator): replayed packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 192301657158625697281087330944801877776, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=4 clogged for ticks=205
[debug] (packet_simulator): replayed packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request_headers
[debug] (replica): 2: on_message: view=94 status=Status.view_change Header{ .checksum = 304612445240698770682799895737061095427, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 92, .op = 60, .commit = 59, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request_headers, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_request_headers: ignoring (older view)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=1 clogged for ticks=15
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=0 clogged for ticks=333
[err] (packet_simulator): dropped packet from=0 to=2.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=6 clogged for ticks=265
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=8 clogged for ticks=100
[err] (packet_simulator): dropped packet from=1 to=2.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=8 clogged for ticks=153
[debug] (vsr): 0: view_change_message_timeout fired
[debug] (vsr): 0: view_change_message_timeout reset
[debug] (replica): 0: sending start_view_change to replica 1: Header{ .checksum = 281362512409244457737336051959983193538, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 2: Header{ .checksum = 281362512409244457737336051959983193538, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (journal): 0: copy_latest_headers_between: op_min=0 op_max=61 dest.len=32 copied=32
[debug] (replica): 0: sending do_view_change to replica 1: Header{ .checksum = 124924144947466025800620679407211028394, .checksum_body = 177310321252202099528854464168395478546, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 61, .commit = 58, .timestamp = 92, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.do_view_change
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.do_view_change
[debug] (replica): 1: on_message: view=94 status=Status.view_change Header{ .checksum = 124924144947466025800620679407211028394, .checksum_body = 177310321252202099528854464168395478546, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 94, .op = 61, .commit = 58, .timestamp = 92, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_do_view_change: 2 message(s)
[debug] (replica): 1: on_do_view_change: view=94 quorum received
[debug] (replica): 1: on_do_view_change: replica=0 v'=92 op=61 commit=58 latest=Header{ .checksum = 189344397916355031745147269347034725344, .checksum_body = 98081761841810822994326224503485043631, .parent = 289712674547617145100370517531510141349, .client = 65459479704107910451097262802840693083, .context = 110559030372813900348458033142393109091, .request = 9, .cluster = 0, .epoch = 0, .view = 92, .op = 61, .commit = 58, .timestamp = 552320000000, .size = 128, .replica = 2, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_do_view_change: replica=1 v'=91 op=59 commit=57 latest=Header{ .checksum = 5936299442387245859284104260459110620, .checksum_body = 255893307917939104760359114691809252225, .parent = 296160201216084397893060525195855222853, .client = 27344200203525815038850809559365994372, .context = 318135809867654346475832079429549481168, .request = 9, .cluster = 0, .epoch = 0, .view = 91, .op = 59, .commit = 57, .timestamp = 544950000000, .size = 944165, .replica = 1, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_do_view_change: view=94 op=59..61 commit=57..58 checksum=189344397916355031745147269347034725344
[debug] (journal): 1: set_header_as_dirty: op=61 checksum=189344397916355031745147269347034725344
[debug] (journal): 1: remove_entries_from: op_min=62
[debug] (replica): 1: repair_header: op=61 exists, checksum dirty
[debug] (journal): 1: set_header_as_dirty: op=61 checksum=189344397916355031745147269347034725344
[debug] (replica): 1: repair_header: op=58 false (exists)
[debug] (replica): 1: repair_header: op=57 false (checksum clean)
[debug] (replica): 1: repair_header: op=56 false (checksum clean)
[debug] (replica): 1: repair_header: op=55 false (checksum clean)
[debug] (replica): 1: repair_header: op=54 false (checksum clean)
[debug] (replica): 1: repair_header: op=53 false (checksum clean)
[debug] (replica): 1: repair_header: op=52 false (checksum clean)
[debug] (replica): 1: repair_header: op=51 false (checksum clean)
[debug] (replica): 1: repair_header: op=50 false (checksum clean)
[debug] (replica): 1: repair_header: op=49 false (checksum clean)
[debug] (replica): 1: repair_header: op=48 false (checksum clean)
[debug] (replica): 1: repair_header: op=47 false (checksum clean)
[debug] (replica): 1: repair_header: op=46 false (checksum clean)
[debug] (replica): 1: repair_header: op=45 false (checksum clean)
[debug] (replica): 1: repair_header: op=44 false (checksum clean)
[debug] (replica): 1: repair_header: op=43 false (checksum clean)
[debug] (replica): 1: repair_header: op=42 false (checksum clean)
[debug] (replica): 1: repair_header: op=41 false (checksum clean)
[debug] (replica): 1: repair_header: op=40 false (checksum clean)
[debug] (replica): 1: repair_header: op=39 false (checksum clean)
[debug] (replica): 1: repair_header: op=38 false (checksum clean)
[debug] (replica): 1: repair_header: op=37 false (checksum clean)
[debug] (replica): 1: repair_header: op=36 false (checksum clean)
[debug] (replica): 1: repair_header: op=35 false (checksum clean)
[debug] (replica): 1: repair_header: op=34 false (checksum clean)
[debug] (replica): 1: repair_header: op=33 false (checksum clean)
[debug] (replica): 1: repair_header: op=32 false (checksum clean)
[debug] (replica): 1: repair_header: op=31 false (checksum clean)
[debug] (replica): 1: repair_header: op=30 false (checksum clean)
[debug] (replica): 1: repair_header: op=29 false (checksum clean)
[debug] (replica): 1: repair_header: op=28 false (checksum clean)
[debug] (replica): 1: repair_header: op=59 false (checksum clean)
[debug] (replica): 1: repair_header: op=58 false (checksum clean)
[debug] (replica): 1: repair_header: op=57 false (checksum clean)
[debug] (replica): 1: repair_header: op=56 false (checksum clean)
[debug] (replica): 1: repair_header: op=55 false (checksum clean)
[debug] (replica): 1: repair_header: op=54 false (checksum clean)
[debug] (replica): 1: repair_header: op=53 false (checksum clean)
[debug] (replica): 1: repair_header: op=52 false (checksum clean)
[debug] (replica): 1: repair_header: op=51 false (checksum clean)
[debug] (replica): 1: repair_header: op=50 false (checksum clean)
[debug] (replica): 1: repair_header: op=49 false (checksum clean)
[debug] (replica): 1: repair_header: op=48 false (checksum clean)
[debug] (replica): 1: repair_header: op=47 false (checksum clean)
[debug] (replica): 1: repair_header: op=46 false (checksum clean)
[debug] (replica): 1: repair_header: op=45 false (checksum clean)
[debug] (replica): 1: repair_header: op=44 false (checksum clean)
[debug] (replica): 1: repair_header: op=43 false (checksum clean)
[debug] (replica): 1: repair_header: op=42 false (checksum clean)
[debug] (replica): 1: repair_header: op=41 false (checksum clean)
[debug] (replica): 1: repair_header: op=40 false (checksum clean)
[debug] (replica): 1: repair_header: op=39 false (checksum clean)
[debug] (replica): 1: repair_header: op=38 false (checksum clean)
[debug] (replica): 1: repair_header: op=37 false (checksum clean)
[debug] (replica): 1: repair_header: op=36 false (checksum clean)
[debug] (replica): 1: repair_header: op=35 false (checksum clean)
[debug] (replica): 1: repair_header: op=34 false (checksum clean)
[debug] (replica): 1: repair_header: op=33 false (checksum clean)
[debug] (replica): 1: repair_header: op=32 false (checksum clean)
[debug] (replica): 1: repair_header: op=31 false (checksum clean)
[debug] (replica): 1: repair_header: op=30 false (checksum clean)
[debug] (replica): 1: repair_header: op=29 false (checksum clean)
[debug] (replica): 1: repair_header: op=28 false (checksum clean)
[debug] (replica): 1: discard_uncommitted_headers: op=60 gap
[debug] (replica): 1: discard_uncommitted_headers: replica=0 op=60 nack=true
[debug] (replica): 1: discard_uncommitted_headers: op=60 nacks=1 threshold=1
[debug] (journal): 1: remove_entries_from: op_min=60
[debug] (vsr): 1: repair_timeout started
[debug] (vsr): 1: repair_timeout reset
[debug] (replica): 1: commit_op: executing view=94 true op=58 checksum=296160201216084397893060525195855222853 (hash)
[debug] (state_machine): state=257140ee68d34a5acba8fc7fb6d5172c input=27b078d03b26201813a363cec82be446 input.len=0 new state=60b1151266995f10af112734020ba002
[err] (state_checker): state checker error: error.ReplicaTransitionedToInvalidState

Time:
2022-07-12 14:47:11.741889355 +0000 UTC

Hi folks,
I got a Reached unreachable code panic, running benchmark.zig twice against the same server.

Steps to reproduce

1. Build tigerbeetle from sources, and access zig-output/bin
2. Run ./tigerbeetle init --cluster=0 --replica=0 --directory=.
3. Run ./tigerbeetle start --cluster=0 --replica=0 --directory=. --addresses=127.0.0.1:3001
4. In another terminal, run zig run -OReleaseSafe ./src/benchmark.zig twice.
   The first time it runs OK, but the second time the server crashes.

benchmark's output:

batiati@TSIBATIATI:~/tigerbeetle$ zig run -OReleaseSafe ./src/benchmark.zig
creating accounts...
batching transfers...
starting benchmark...
============================================
531914 transfers per second

create_transfers max p100 latency per 5000 transfers = 23ms
commit_transfers max p100 latency per 5000 transfers = 0ms
batiati@TSIBATIATI:~/tigerbeetle$ zig run -OReleaseSafe ./src/benchmark.zig
creating accounts...
batching transfers...
starting benchmark...
error(message_bus): error connecting to replica 0: error.ConnectionRefused
error(message_bus): error connecting to replica 0: error.ConnectionRefused
error(message_bus): error connecting to replica 0: error.ConnectionRefused
^C

Server's output (running Release-Safe)

info(storage): opening "cluster_0000000000_replica_000.tigerbeetle"...
info: cluster=0 replica=0: listening on 0.0.0.0:3001
info(message_bus): connection from client 55741322461856235128456485717824778813
info(message_bus): peer performed an orderly shutdown: message_bus.union:392:19{ .client = 55741322461856235128456485717824778813 }
info(message_bus): connection from client 305270733519922419579078604465720248410
thread 962 panic: reached unreachable code
Aborted

Server's output (running in Debug)

info(message_bus): connection from client 137357130890439269904175630134727826053
thread 911 panic: reached unreachable code
/home/batiati/zig/lib/std/debug.zig:226:14: 0x222ddb in std.debug.assert (tigerbeetle)
    if (!ok) unreachable; // assertion failure
             ^
/home/batiati/tigerbeetle/src/vsr/journal.zig:861:19: 0x29793c in vsr.journal.Journal(vsr.replica.Replica(state_machine.StateMachine,message_bus.MessageBusImpl(message_bus.ProcessType.replica),storage.Storage,time.Time),storage.Storage).write_prepare (tigerbeetle)
            assert(message.header.offset + sectors.len <= self.size_circular_buffer);
                  ^
/home/batiati/tigerbeetle/src/vsr/replica.zig:4226:39: 0x28825a in vsr.replica.Replica(state_machine.StateMachine,message_bus.MessageBusImpl(message_bus.ProcessType.replica),storage.Storage,time.Time).write_prepare (tigerbeetle)
            self.journal.write_prepare(write_prepare_on_write, message, trigger);
                                      ^
/home/batiati/tigerbeetle/src/vsr/replica.zig:1680:31: 0x278ebe in vsr.replica.Replica(state_machine.StateMachine,message_bus.MessageBusImpl(message_bus.ProcessType.replica),storage.Storage,time.Time).append (tigerbeetle)
            self.write_prepare(message, .append);
                              ^
/home/batiati/tigerbeetle/src/vsr/replica.zig:712:24: 0x268675 in vsr.replica.Replica(state_machine.StateMachine,message_bus.MessageBusImpl(message_bus.ProcessType.replica),storage.Storage,time.Time).on_prepare (tigerbeetle)
            self.append(message);
                       ^

Notes

• Reproduced using Linux Ubuntu 5.11.0-38-generic x86_64 GNU/Linux

• Running using scripts/benchmark.sh works fine because the cluster is initialized every execution.

I'm not sure if the benchmark is supposed to run multiple times against the same cluster. It's most likely I'm doing something wrong, but I didn't expect a server panic.

Bug Type:
crash

Seed:
11654694784361616086

Commit Hash:
01494c5

Parameters:
[info] (state_checker):
SEED=11654694784361616086

replicas=3
clients=4
request_probability=11%
idle_on_probability=18%
idle_off_probability=11%
one_way_delay_mean=9 ticks
one_way_delay_min=2 ticks
packet_loss_probability=25%
path_maximum_capacity=20 messages
path_clog_duration_mean=227 ticks
path_clog_probability=0%
packet_replay_probability=26%
partition_mode=PartitionMode.uniform_partition
partition_probability=0%
unpartition_probability=2%
partition_stability=176 ticks
unpartition_stability=9 ticks
read_latency_min=2
read_latency_mean=9
write_latency_min=0
write_latency_mean=66
read_fault_probability=6%
write_fault_probability=3%
crash_probability=0.01%
crash_stability=752 ticks
restart_probability=1%
restart_stability=966 ticks

Tail of Debug Logs:
556530337571515, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (vsr): 1: view_change_message_timeout fired
[debug] (vsr): 1: view_change_message_timeout reset
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 216442080330958390782706069435631874842, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 216442080330958390782706069435631874842, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (journal): 1: copy_latest_headers_between: op_min=0 op_max=68 dest.len=32 copied=32
[debug] (replica): 1: sending do_view_change to replica 2: Header{ .checksum = 314762811721899295135296439442619662016, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (vsr): 2: view_change_message_timeout fired
[debug] (vsr): 2: view_change_message_timeout reset
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=2381 status=Status.view_change Header{ .checksum = 265406233045811921053519574710870372982, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: ignoring (duplicate message replica=0)
[debug] (packet_simulator): replayed packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (replica): 2: on_message: view=2381 status=Status.view_change Header{ .checksum = 314762811721899295135296439442619662016, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_do_view_change: ignoring (duplicate message)
[debug] (packet_simulator): replayed packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2381 status=Status.view_change Header{ .checksum = 216442080330958390782706069435631874842, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=1)
[err] (packet_simulator): dropped packet from=1 to=0.
[err] (packet_simulator): dropped packet from=1 to=2.
[err] (packet_simulator): dropped packet from=1 to=2.
[debug] (vsr): 2: repair_timeout fired
[debug] (vsr): 2: repair_timeout reset
[debug] (replica): 2: repair_prepare: op=26 checksum=330296907629322072566147740092185258661 (committed, faulty)
[debug] (replica): 2: sending request_prepare to replica 1: Header{ .checksum = 141751061205600343772700865275573114638, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 330296907629322072566147740092185258661, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 26, .commit = 0, .timestamp = 1, .size = 128, .replica = 2, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.request_prepare
[err] (packet_simulator): dropped packet from=0 to=2.
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (replica): 2: on_message: view=2381 status=Status.view_change Header{ .checksum = 197544816351296270016908556530337571515, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_do_view_change: ignoring (duplicate message)
[err] (packet_simulator): dropped packet from=2 to=1.
[debug] (vsr): 0: view_change_message_timeout fired
[debug] (vsr): 0: view_change_message_timeout reset
[debug] (replica): 0: sending start_view_change to replica 1: Header{ .checksum = 265406233045811921053519574710870372982, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 2: Header{ .checksum = 265406233045811921053519574710870372982, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (journal): 0: copy_latest_headers_between: op_min=0 op_max=68 dest.len=32 copied=32
[debug] (replica): 0: sending do_view_change to replica 2: Header{ .checksum = 197544816351296270016908556530337571515, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (vsr): 1: view_change_message_timeout fired
[debug] (vsr): 1: view_change_message_timeout reset
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 216442080330958390782706069435631874842, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 216442080330958390782706069435631874842, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (journal): 1: copy_latest_headers_between: op_min=0 op_max=68 dest.len=32 copied=32
[debug] (replica): 1: sending do_view_change to replica 2: Header{ .checksum = 314762811721899295135296439442619662016, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (vsr): 2: view_change_message_timeout fired
[debug] (vsr): 2: view_change_message_timeout reset
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=2381 status=Status.view_change Header{ .checksum = 265406233045811921053519574710870372982, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: ignoring (duplicate message replica=0)
[debug] (packet_simulator): replayed packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=2381 status=Status.view_change Header{ .checksum = 216442080330958390782706069435631874842, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (replica): 2: on_message: view=2381 status=Status.view_change Header{ .checksum = 314762811721899295135296439442619662016, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_do_view_change: ignoring (duplicate message)
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=2381 status=Status.view_change Header{ .checksum = 265406233045811921053519574710870372982, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=0)
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2381 status=Status.view_change Header{ .checksum = 216442080330958390782706069435631874842, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (replica): 2: on_message: view=2381 status=Status.view_change Header{ .checksum = 197544816351296270016908556530337571515, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_do_view_change: ignoring (duplicate message)
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=2381 status=Status.view_change Header{ .checksum = 216442080330958390782706069435631874842, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (vsr): 2: repair_timeout fired
[debug] (vsr): 2: repair_timeout reset
[debug] (replica): 2: repair_prepare: op=26 checksum=330296907629322072566147740092185258661 (committed, faulty)
[debug] (replica): 2: sending request_prepare to replica 0: Header{ .checksum = 141751061205600343772700865275573114638, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 330296907629322072566147740092185258661, .request = 0, .cluster = 0, .epoch = 0, .view = 2381, .op = 26, .commit = 0, .timestamp = 1, .size = 128, .replica = 2, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.request_prepare
[err] (packet_simulator): dropped packet from=2 to=0.
[debug] (vsr): 0: view_change_status_timeout fired
[debug] (replica): 0: transition_to_view_change_status: view=2381..2382
[debug] (vsr): 0: ping_timeout stopped
[debug] (vsr): 0: commit_timeout stopped
[debug] (vsr): 0: normal_status_timeout stopped
[debug] (vsr): 0: view_change_status_timeout started
[debug] (vsr): 0: view_change_message_timeout started
[debug] (vsr): 0: repair_timeout stopped
[debug] (vsr): 0: prepare_timeout stopped
[debug] (replica): 0: reset 0 do_view_change message(s) from view=null
[debug] (replica): 0: sending start_view_change to replica 1: Header{ .checksum = 324959638660945898287986245671188119283, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 2: Header{ .checksum = 324959638660945898287986245671188119283, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (vsr): 1: view_change_status_timeout fired
[debug] (replica): 1: transition_to_view_change_status: view=2381..2382
[debug] (vsr): 1: ping_timeout stopped
[debug] (vsr): 1: commit_timeout stopped
[debug] (vsr): 1: normal_status_timeout stopped
[debug] (vsr): 1: view_change_status_timeout started
[debug] (vsr): 1: view_change_message_timeout started
[debug] (vsr): 1: repair_timeout stopped
[debug] (vsr): 1: prepare_timeout stopped
[debug] (replica): 1: reset 0 do_view_change message(s) from view=null
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (vsr): 2: view_change_status_timeout fired
[debug] (replica): 2: transition_to_view_change_status: view=2381..2382
[debug] (vsr): 2: ping_timeout stopped
[debug] (vsr): 2: commit_timeout stopped
[debug] (vsr): 2: normal_status_timeout stopped
[debug] (vsr): 2: view_change_status_timeout started
[debug] (vsr): 2: view_change_message_timeout started
[debug] (vsr): 2: repair_timeout stopped
[debug] (vsr): 2: prepare_timeout stopped
[debug] (replica): 2: reset 3 do_view_change message(s) from view=2381
[debug] (replica): 2: sending start_view_change to replica 0: Header{ .checksum = 295802827437696205256588745679722478451, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 1: Header{ .checksum = 295802827437696205256588745679722478451, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (packet_simulator): replayed packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 295802827437696205256588745679722478451, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: 1 message(s)
[debug] (replica): 0: on_start_view_change: view=2382 quorum received
[debug] (journal): 0: copy_latest_headers_between: op_min=0 op_max=68 dest.len=32 copied=32
[debug] (replica): 0: sending do_view_change to replica 0: Header{ .checksum = 235617427791570318995208597014748332261, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 235617427791570318995208597014748332261, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: 1 message(s)
[debug] (replica): 0: on_do_view_change: waiting for quorum
[err] (packet_simulator): dropped packet from=2 to=1.
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=2382 status=Status.view_change Header{ .checksum = 324959638660945898287986245671188119283, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: 1 message(s)
[debug] (replica): 1: on_start_view_change: view=2382 quorum received
[debug] (journal): 1: copy_latest_headers_between: op_min=0 op_max=68 dest.len=32 copied=32
[debug] (replica): 1: sending do_view_change to replica 0: Header{ .checksum = 298157826303812085379030874232707016109, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=2382 status=Status.view_change Header{ .checksum = 324959638660945898287986245671188119283, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: 1 message(s)
[debug] (replica): 2: on_start_view_change: view=2382 quorum received
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=68 dest.len=32 copied=32
[debug] (replica): 2: sending do_view_change to replica 0: Header{ .checksum = 209168693276895636277850210138742452951, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (packet_simulator): replayed packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: 2 message(s)
[debug] (replica): 0: on_start_view_change: ignoring (quorum received already)
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 298157826303812085379030874232707016109, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: 2 message(s)
[debug] (replica): 0: on_do_view_change: view=2382 quorum received
[debug] (replica): 0: on_do_view_change: replica=0 v'=2 op=68 commit=68 latest=Header{ .checksum = 271408327367023641725539222960565597137, .checksum_body = 200511362828524740113934185482600390872, .parent = 281142446631901820053319140298981567249, .client = 337275728709412499007489779383344250735, .context = 67099398369097624982858333466227228509, .request = 18, .cluster = 0, .epoch = 0, .view = 2, .op = 68, .commit = 67, .timestamp = 260630000000, .size = 354528, .replica = 2, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_do_view_change: replica=1 v'=2 op=68 commit=68 latest=Header{ .checksum = 271408327367023641725539222960565597137, .checksum_body = 200511362828524740113934185482600390872, .parent = 281142446631901820053319140298981567249, .client = 337275728709412499007489779383344250735, .context = 67099398369097624982858333466227228509, .request = 18, .cluster = 0, .epoch = 0, .view = 2, .op = 68, .commit = 67, .timestamp = 260630000000, .size = 354528, .replica = 2, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_do_view_change: view=2382 op=68..68 commit=68..68 checksum=271408327367023641725539222960565597137
[debug] (replica): 0: on_do_view_change: latest op exists exactly
[debug] (journal): 0: remove_entries_from: op_min=69
[debug] (replica): 0: repair_header: op=68 false (checksum clean)
[debug] (replica): 0: repair_header: op=67 false (checksum clean)
[debug] (replica): 0: repair_header: op=66 false (checksum clean)
[debug] (replica): 0: repair_header: op=65 false (checksum clean)
[debug] (replica): 0: repair_header: op=64 false (checksum clean)
[debug] (replica): 0: repair_header: op=63 false (checksum clean)
[debug] (replica): 0: repair_header: op=62 false (checksum clean)
[debug] (replica): 0: repair_header: op=61 false (checksum clean)
[debug] (replica): 0: repair_header: op=60 false (checksum clean)
[debug] (replica): 0: repair_header: op=59 false (checksum clean)
[debug] (replica): 0: repair_header: op=58 false (checksum clean)
[debug] (replica): 0: repair_header: op=57 false (checksum clean)
[debug] (replica): 0: repair_header: op=56 false (checksum clean)
[debug] (replica): 0: repair_header: op=55 false (checksum clean)
[debug] (replica): 0: repair_header: op=54 false (checksum clean)
[debug] (replica): 0: repair_header: op=53 false (checksum clean)
[debug] (replica): 0: repair_header: op=52 false (checksum clean)
[debug] (replica): 0: repair_header: op=51 false (checksum clean)
[debug] (replica): 0: repair_header: op=50 false (checksum clean)
[debug] (replica): 0: repair_header: op=49 false (checksum clean)
[debug] (replica): 0: repair_header: op=48 false (checksum clean)
[debug] (replica): 0: repair_header: op=47 false (checksum clean)
[debug] (replica): 0: repair_header: op=46 false (checksum clean)
[debug] (replica): 0: repair_header: op=45 false (checksum clean)
[debug] (replica): 0: repair_header: op=44 false (checksum clean)
[debug] (replica): 0: repair_header: op=43 false (checksum clean)
[debug] (replica): 0: repair_header: op=42 false (checksum clean)
[debug] (replica): 0: repair_header: op=41 false (checksum clean)
[debug] (replica): 0: repair_header: op=40 false (checksum clean)
[debug] (replica): 0: repair_header: op=39 false (checksum clean)
[debug] (replica): 0: repair_header: op=38 false (checksum clean)
[debug] (replica): 0: repair_header: op=37 false (checksum clean)
[debug] (replica): 0: repair_header: op=68 false (checksum clean)
[debug] (replica): 0: repair_header: op=67 false (checksum clean)
[debug] (replica): 0: repair_header: op=66 false (checksum clean)
[debug] (replica): 0: repair_header: op=65 false (checksum clean)
[debug] (replica): 0: repair_header: op=64 false (checksum clean)
[debug] (replica): 0: repair_header: op=63 false (checksum clean)
[debug] (replica): 0: repair_header: op=62 false (checksum clean)
[debug] (replica): 0: repair_header: op=61 false (checksum clean)
[debug] (replica): 0: repair_header: op=60 false (checksum clean)
[debug] (replica): 0: repair_header: op=59 false (checksum clean)
[debug] (replica): 0: repair_header: op=58 false (checksum clean)
[debug] (replica): 0: repair_header: op=57 false (checksum clean)
[debug] (replica): 0: repair_header: op=56 false (checksum clean)
[debug] (replica): 0: repair_header: op=55 false (checksum clean)
[debug] (replica): 0: repair_header: op=54 false (checksum clean)
[debug] (replica): 0: repair_header: op=53 false (checksum clean)
[debug] (replica): 0: repair_header: op=52 false (checksum clean)
[debug] (replica): 0: repair_header: op=51 false (checksum clean)
[debug] (replica): 0: repair_header: op=50 false (checksum clean)
[debug] (replica): 0: repair_header: op=49 false (checksum clean)
[debug] (replica): 0: repair_header: op=48 false (checksum clean)
[debug] (replica): 0: repair_header: op=47 false (checksum clean)
[debug] (replica): 0: repair_header: op=46 false (checksum clean)
[debug] (replica): 0: repair_header: op=45 false (checksum clean)
[debug] (replica): 0: repair_header: op=44 false (checksum clean)
[debug] (replica): 0: repair_header: op=43 false (checksum clean)
[debug] (replica): 0: repair_header: op=42 false (checksum clean)
[debug] (replica): 0: repair_header: op=41 false (checksum clean)
[debug] (replica): 0: repair_header: op=40 false (checksum clean)
[debug] (replica): 0: repair_header: op=39 false (checksum clean)
[debug] (replica): 0: repair_header: op=38 false (checksum clean)
[debug] (replica): 0: repair_header: op=37 false (checksum clean)
[debug] (vsr): 0: repair_timeout started
[debug] (vsr): 0: repair_timeout reset
[debug] (replica): 0: repair_prepare: op=26 checksum=330296907629322072566147740092185258661 (committed, dirty)
[debug] (replica): 0: sending request_prepare to replica 2: Header{ .checksum = 255204168481056663064229495403465633312, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 330296907629322072566147740092185258661, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 26, .commit = 0, .timestamp = 1, .size = 128, .replica = 0, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request_prepare
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=2382 status=Status.view_change Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: 2 message(s)
[debug] (replica): 2: on_start_view_change: ignoring (quorum received already)
[err] (packet_simulator): dropped packet from=2 to=0.
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request_prepare
[debug] (replica): 2: on_message: view=2382 status=Status.view_change Header{ .checksum = 255204168481056663064229495403465633312, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 330296907629322072566147740092185258661, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 26, .commit = 0, .timestamp = 1, .size = 128, .replica = 0, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_request_prepare: op=26 checksum=330296907629322072566147740092185258661 reading
[info] (journal): 2: read_prepare: op=26 checksum=330296907629322072566147740092185258661: faulty
[debug] (replica): 2: on_request_prepare_read: prepare=null
[debug] (packet_simulator): replayed packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 295802827437696205256588745679722478451, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=2)
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 295802827437696205256588745679722478451, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=2)
[debug] (vsr): 0: view_change_message_timeout fired
[debug] (vsr): 0: view_change_message_timeout reset
[debug] (vsr): 1: view_change_message_timeout fired
[debug] (vsr): 1: view_change_message_timeout reset
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (journal): 1: copy_latest_headers_between: op_min=0 op_max=68 dest.len=32 copied=32
[debug] (replica): 1: sending do_view_change to replica 0: Header{ .checksum = 298157826303812085379030874232707016109, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (vsr): 2: view_change_message_timeout fired
[debug] (vsr): 2: view_change_message_timeout reset
[debug] (replica): 2: sending start_view_change to replica 0: Header{ .checksum = 295802827437696205256588745679722478451, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 1: Header{ .checksum = 295802827437696205256588745679722478451, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=68 dest.len=32 copied=32
[debug] (replica): 2: sending do_view_change to replica 0: Header{ .checksum = 209168693276895636277850210138742452951, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.do_view_change
[err] (packet_simulator): dropped packet from=2 to=0.
[debug] (packet_simulator): delivering packet from=2 to=1
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=2382 status=Status.view_change Header{ .checksum = 295802827437696205256588745679722478451, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: 2 message(s)
[debug] (replica): 1: on_start_view_change: ignoring (quorum received already)
[debug] (packet_simulator): replayed packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 298157826303812085379030874232707016109, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: ignoring (duplicate message)
[debug] (packet_simulator): replayed packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 295802827437696205256588745679722478451, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=2)
[debug] (vsr): 0: repair_timeout fired
[debug] (vsr): 0: repair_timeout reset
[debug] (replica): 0: repair_prepare: op=26 checksum=330296907629322072566147740092185258661 (committed, dirty)
[debug] (replica): 0: sending request_prepare to replica 1: Header{ .checksum = 255204168481056663064229495403465633312, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 330296907629322072566147740092185258661, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 26, .commit = 0, .timestamp = 1, .size = 128, .replica = 0, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.request_prepare
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=1)
[err] (packet_simulator): dropped packet from=1 to=0.
[debug] (packet_simulator): replayed packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=2382 status=Status.view_change Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 295802827437696205256588745679722478451, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=2)
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=2382 status=Status.view_change Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[err] (packet_simulator): dropped packet from=0 to=1.
[debug] (vsr): 0: view_change_message_timeout fired
[debug] (vsr): 0: view_change_message_timeout reset
[debug] (vsr): 1: view_change_message_timeout fired
[debug] (vsr): 1: view_change_message_timeout reset
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (journal): 1: copy_latest_headers_between: op_min=0 op_max=68 dest.len=32 copied=32
[debug] (replica): 1: sending do_view_change to replica 0: Header{ .checksum = 298157826303812085379030874232707016109, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (vsr): 2: view_change_message_timeout fired
[debug] (vsr): 2: view_change_message_timeout reset
[debug] (replica): 2: sending start_view_change to replica 0: Header{ .checksum = 295802827437696205256588745679722478451, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 1: Header{ .checksum = 295802827437696205256588745679722478451, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=68 dest.len=32 copied=32
[debug] (replica): 2: sending do_view_change to replica 0: Header{ .checksum = 209168693276895636277850210138742452951, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=1)
[err] (packet_simulator): dropped packet from=2 to=1.
[err] (packet_simulator): dropped packet from=2 to=0.
[debug] (vsr): 0: repair_timeout fired
[debug] (vsr): 0: repair_timeout reset
[debug] (replica): 0: repair_prepare: op=26 checksum=330296907629322072566147740092185258661 (committed, dirty)
[debug] (replica): 0: sending request_prepare to replica 2: Header{ .checksum = 255204168481056663064229495403465633312, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 330296907629322072566147740092185258661, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 26, .commit = 0, .timestamp = 1, .size = 128, .replica = 0, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request_prepare
[err] (packet_simulator): dropped packet from=2 to=0.
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=2382 status=Status.view_change Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 298157826303812085379030874232707016109, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: ignoring (duplicate message)
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request_prepare
[debug] (replica): 2: on_message: view=2382 status=Status.view_change Header{ .checksum = 255204168481056663064229495403465633312, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 330296907629322072566147740092185258661, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 26, .commit = 0, .timestamp = 1, .size = 128, .replica = 0, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_request_prepare: op=26 checksum=330296907629322072566147740092185258661 reading
[info] (journal): 2: read_prepare: op=26 checksum=330296907629322072566147740092185258661: faulty
[debug] (replica): 2: on_request_prepare_read: prepare=null
[debug] (vsr): 0: view_change_message_timeout fired
[debug] (vsr): 0: view_change_message_timeout reset
[debug] (vsr): 1: view_change_message_timeout fired
[debug] (vsr): 1: view_change_message_timeout reset
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (journal): 1: copy_latest_headers_between: op_min=0 op_max=68 dest.len=32 copied=32
[debug] (replica): 1: sending do_view_change to replica 0: Header{ .checksum = 298157826303812085379030874232707016109, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (vsr): 2: view_change_message_timeout fired
[debug] (vsr): 2: view_change_message_timeout reset
[debug] (replica): 2: sending start_view_change to replica 0: Header{ .checksum = 295802827437696205256588745679722478451, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 1: Header{ .checksum = 295802827437696205256588745679722478451, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=68 dest.len=32 copied=32
[debug] (replica): 2: sending do_view_change to replica 0: Header{ .checksum = 209168693276895636277850210138742452951, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.do_view_change
[err] (packet_simulator): dropped packet from=2 to=0.
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 209168693276895636277850210138742452951, .checksum_body = 85493199026511476387022115951477938651, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 68, .commit = 68, .timestamp = 2, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: 3 message(s)
[debug] (replica): 0: on_do_view_change: ignoring (quorum received already)
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2382 status=Status.view_change Header{ .checksum = 177911432912993993546198503214099478402, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (vsr): 0: repair_timeout fired
[debug] (vsr): 0: repair_timeout reset
[debug] (replica): 0: repair_prepare: op=26 checksum=330296907629322072566147740092185258661 (committed, dirty)
[debug] (replica): 0: sending request_prepare to replica 1: Header{ .checksum = 255204168481056663064229495403465633312, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 330296907629322072566147740092185258661, .request = 0, .cluster = 0, .epoch = 0, .view = 2382, .op = 26, .commit = 0, .timestamp = 1, .size = 128, .replica = 0, .command = Command.request_prepare, .o

Time:
2022-07-12 15:35:27.421150085 +0000 UTC

Bug Type:
crash

Seed:
11833592692126088617

Commit Hash:
01494c5

Parameters:
[info] (state_checker):
SEED=11833592692126088617

replicas=4
clients=16
request_probability=41%
idle_on_probability=8%
idle_off_probability=17%
one_way_delay_mean=3 ticks
one_way_delay_min=1 ticks
packet_loss_probability=7%
path_maximum_capacity=20 messages
path_clog_duration_mean=81 ticks
path_clog_probability=0%
packet_replay_probability=38%
partition_mode=PartitionMode.isolate_single
partition_probability=2%
unpartition_probability=2%
partition_stability=195 ticks
unpartition_stability=1 ticks
read_latency_min=2
read_latency_mean=3
write_latency_min=2
write_latency_mean=43
read_fault_probability=4%
write_fault_probability=9%
crash_probability=0.01%
crash_stability=175 ticks
restart_probability=1%
restart_stability=990 ticks

Stack Trace:

thread panic: reached unreachable code
zig/lib/std/debug.zig:225:14: in std.debug.assert (simulator)
if (!ok) unreachable; // assertion failure
^
src/vsr/replica.zig:558:19: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).on_message (simulator)
assert(self.loopback_queue == null);
^
src/test/message_bus.zig:57:27: in test.message_bus.struct:55:35.wrapper (simulator)
on_message(@intToPtr(Context, @ptrToInt(_context)), message);
^
src/test/network.zig:167:41: in test.network.Network.deliver_message (simulator)
target_bus.on_message_callback.?(target_bus.on_message_context, message);
^
src/test/packet_simulator.zig:345:42: in test.packet_simulator.PacketSimulator(test.network.Packet).tick (simulator)
data.callback(data.packet, path);
^
src/simulator.zig:277:46: in main (simulator)
cluster.network.packet_simulator.tick(cluster.health);
^
zig/lib/std/start.zig:561:37: in std.start.callMain (simulator)
const result = root.main() catch |err| {
^
zig/lib/std/start.zig:495:12: in std.start.callMainWithArgs (simulator)
return @call(.{ .modifier = .always_inline }, callMain, .{});
^
zig/lib/std/start.zig:409:17: in std.start.posixCallMainAndExit (simulator)
std.os.exit(@call(.{ .modifier = .always_inline }, callMainWithArgs, .{ argc, argv, envp }));
^
zig/lib/std/start.zig:322:5: in std.start._start (simulator)
@call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{});
^
exit with signal: 6. Indicates a crash bug.


Tail of Debug Logs:
ica = 1 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (replica): 0: on_message: view=4 status=Status.normal Header{ .checksum = 155380764779019561979545705853518099412, .checksum_body = 69263773534116441192706135316961910146, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 0, .commit = 0, .timestamp = 3, .size = 256, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: ignoring (view started)
[debug] (packet_simulator): replayed packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view
[debug] (replica): 1: on_message: view=4 status=Status.view_change Header{ .checksum = 90553412068177260742004761110666390292, .checksum_body = 247498881379572285775948866581259035737, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 0, .size = 384, .replica = 0, .command = Command.start_view, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view: view=4 op=0..1 commit=0..0 checksum=268840143496320758578619296218401986711
[debug] (journal): 1: set_header_as_dirty: op=1 checksum=268840143496320758578619296218401986711
[debug] (journal): 1: remove_entries_from: op_min=2
[debug] (replica): 1: repair_header: op=1 exists, checksum dirty
[debug] (journal): 1: set_header_as_dirty: op=1 checksum=268840143496320758578619296218401986711
[debug] (replica): 1: repair_header: op=0 false (checksum clean)
[debug] (replica): 1: transition_to_normal_from_view_change_status: view=4 follower
[debug] (vsr): 1: ping_timeout started
[debug] (vsr): 1: commit_timeout stopped
[debug] (vsr): 1: normal_status_timeout started
[debug] (vsr): 1: view_change_status_timeout stopped
[debug] (vsr): 1: view_change_message_timeout stopped
[debug] (vsr): 1: repair_timeout started
[debug] (replica): 1: reset 0 do_view_change message(s) from view=null
[debug] (replica): 1: send_prepare_ok: not sending (dirty)
[debug] (vsr): 1: repair_timeout reset
[debug] (replica): 1: repair_prepare: op=1 checksum=268840143496320758578619296218401986711 (uncommitted, dirty)
[debug] (replica): 1: sending request_prepare to replica 2: Header{ .checksum = 78675336814232754273676860177577076796, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 268840143496320758578619296218401986711, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 1, .size = 128, .replica = 1, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.request_prepare
[debug] (packet_simulator): delivering packet from=0 to=3
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.start_view
[debug] (replica): 3: on_message: view=0 status=Status.recovering Header{ .checksum = 90553412068177260742004761110666390292, .checksum_body = 247498881379572285775948866581259035737, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 0, .size = 384, .replica = 0, .command = Command.start_view, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_message: waiting for journal to recover
[debug] (packet_simulator): delivering packet from=0 to=3
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (replica): 3: on_message: view=0 status=Status.recovering Header{ .checksum = 40374604042392984586468012387365318766, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_message: waiting for journal to recover
[debug] (packet_simulator): delivering packet from=1 to=3
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (replica): 3: on_message: view=0 status=Status.recovering Header{ .checksum = 289109741028230525886782900046694583046, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_message: waiting for journal to recover
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (replica): 0: on_message: view=4 status=Status.normal Header{ .checksum = 277742795297551130519057218971887019495, .checksum_body = 69263773534116441192706135316961910146, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 0, .commit = 0, .timestamp = 3, .size = 256, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: ignoring (view started)
[debug] (journal): 3: recover_prepares: recovering slot=88
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=4 status=Status.view_change Header{ .checksum = 40374604042392984586468012387365318766, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=0)
[debug] (packet_simulator): replayed packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view
[debug] (replica): 2: on_message: view=4 status=Status.view_change Header{ .checksum = 90553412068177260742004761110666390292, .checksum_body = 247498881379572285775948866581259035737, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 0, .size = 384, .replica = 0, .command = Command.start_view, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view: view=4 op=0..1 commit=0..0 checksum=268840143496320758578619296218401986711
[debug] (journal): 2: set_header_as_dirty: op=1 checksum=268840143496320758578619296218401986711
[debug] (journal): 2: remove_entries_from: op_min=2
[debug] (replica): 2: repair_header: op=1 exists, checksum dirty
[debug] (journal): 2: set_header_as_dirty: op=1 checksum=268840143496320758578619296218401986711
[debug] (replica): 2: repair_header: op=0 false (checksum clean)
[debug] (replica): 2: transition_to_normal_from_view_change_status: view=4 follower
[debug] (vsr): 2: ping_timeout started
[debug] (vsr): 2: commit_timeout stopped
[debug] (vsr): 2: normal_status_timeout started
[debug] (vsr): 2: view_change_status_timeout stopped
[debug] (vsr): 2: view_change_message_timeout stopped
[debug] (vsr): 2: repair_timeout started
[debug] (replica): 2: reset 0 do_view_change message(s) from view=null
[debug] (replica): 2: send_prepare_ok: not sending (dirty)
[debug] (vsr): 2: repair_timeout reset
[debug] (replica): 2: repair_prepare: op=1 checksum=268840143496320758578619296218401986711 (uncommitted, dirty)
[debug] (replica): 2: sending request_prepare to replica 3: Header{ .checksum = 136833157251937701297687426511046091009, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 268840143496320758578619296218401986711, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 1, .size = 128, .replica = 2, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 3 }: Command.request_prepare
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.request_prepare
[debug] (replica): 2: on_message: view=4 status=Status.normal Header{ .checksum = 78675336814232754273676860177577076796, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 268840143496320758578619296218401986711, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 1, .size = 128, .replica = 1, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): delivering packet from=2 to=3
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 3 }: Command.request_prepare
[debug] (replica): 3: on_message: view=0 status=Status.recovering Header{ .checksum = 136833157251937701297687426511046091009, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 268840143496320758578619296218401986711, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 1, .size = 128, .replica = 2, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_message: waiting for journal to recover
[debug] (journal): 3: recover_prepares: recovering slot=89
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view
[debug] (replica): 1: on_message: view=4 status=Status.normal Header{ .checksum = 90553412068177260742004761110666390292, .checksum_body = 247498881379572285775948866581259035737, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 0, .size = 384, .replica = 0, .command = Command.start_view, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view: ignoring (view started)
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view
[debug] (replica): 2: on_message: view=4 status=Status.normal Header{ .checksum = 90553412068177260742004761110666390292, .checksum_body = 247498881379572285775948866581259035737, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 0, .size = 384, .replica = 0, .command = Command.start_view, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view: ignoring (view started)
[debug] (journal): 3: recover_prepares: recovering slot=90
[debug] (journal): 3: recover_prepares: recovering slot=91
[debug] (journal): 3: recover_prepares: recovering slot=92
[debug] (journal): 3: recover_prepares: recovering slot=93
[debug] (journal): 3: recover_prepares: recovering slot=94
[debug] (journal): 3: recover_prepares: recovering slot=95
[debug] (journal): 3: recover_prepares: recovering slot=96
[debug] (journal): 3: recover_prepares: recovering slot=97
[debug] (journal): 3: recover_prepares: recovering slot=98
[debug] (journal): 3: recover_prepares: recovering slot=99
[debug] (journal): 3: recover_prepares: recovering slot=100
[err] (packet_simulator): partitioned network: partition={ false, false, true, false }
[debug] (journal): 3: recover_prepares: recovering slot=101
[debug] (journal): 3: recover_prepares: recovering slot=102
[debug] (journal): 3: recover_prepares: recovering slot=103
[debug] (journal): 3: recover_prepares: recovering slot=104
[debug] (journal): 3: recover_prepares: recovering slot=105
[debug] (journal): 3: recover_prepares: recovering slot=106
[debug] (vsr): 0: prepare_timeout fired
[debug] (vsr): 0: prepare_timeout backing off
[debug] (vsr): 0: prepare_timeout after=50..76 (rtt=30 min=10 max=1000 attempts=1)
[debug] (replica): 0: on_prepare_timeout: waiting for replica 1
[debug] (replica): 0: on_prepare_timeout: waiting for replica 2
[debug] (replica): 0: on_prepare_timeout: waiting for replica 3
[debug] (replica): 0: on_prepare_timeout: replicating to replica 2
[debug] (replica): 0: sending prepare to replica 2: Header{ .checksum = 268840143496320758578619296218401986711, .checksum_body = 98081761841810822994326224503485043631, .parent = 41826076010321932376190093941186211105, .client = 237793620746173507671246977486275538381, .context = 174952180883201026607060349056975638411, .request = 0, .cluster = 0, .epoch = 0, .view = 3, .op = 1, .commit = 0, .timestamp = 26200000000, .size = 128, .replica = 3, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.prepare
[debug] (vsr): 0: repair_timeout fired
[debug] (vsr): 0: repair_timeout reset
[debug] (journal): 3: recover_prepares: recovering slot=107
[err] (packet_simulator): dropped packet (different partitions): from=0 to=2
[debug] (vsr): 1: repair_timeout fired
[debug] (vsr): 1: repair_timeout reset
[debug] (replica): 1: repair_prepare: op=1 checksum=268840143496320758578619296218401986711 (uncommitted, dirty)
[debug] (replica): 1: sending request_prepare to replica 3: Header{ .checksum = 78675336814232754273676860177577076796, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 268840143496320758578619296218401986711, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 1, .size = 128, .replica = 1, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.request_prepare
[debug] (vsr): 2: repair_timeout fired
[debug] (vsr): 2: repair_timeout reset
[debug] (replica): 2: repair_prepare: op=1 checksum=268840143496320758578619296218401986711 (uncommitted, dirty)
[debug] (replica): 2: sending request_prepare to replica 0: Header{ .checksum = 136833157251937701297687426511046091009, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 268840143496320758578619296218401986711, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 1, .size = 128, .replica = 2, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.request_prepare
[err] (packet_simulator): dropped packet (different partitions): from=2 to=0
[debug] (packet_simulator): delivering packet from=1 to=3
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.request_prepare
[debug] (replica): 3: on_message: view=0 status=Status.recovering Header{ .checksum = 78675336814232754273676860177577076796, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 268840143496320758578619296218401986711, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 1, .size = 128, .replica = 1, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_message: waiting for journal to recover
[debug] (journal): 3: recover_prepares: recovering slot=108
[debug] (journal): 3: recover_prepares: recovering slot=109
[debug] (journal): 3: recover_prepares: recovering slot=110
[debug] (journal): 3: recover_prepares: recovering slot=111
[debug] (journal): 3: recover_prepares: recovering slot=112
[debug] (journal): 3: recover_prepares: recovering slot=113
[debug] (journal): 3: recover_prepares: recovering slot=114
[debug] (journal): 3: recover_prepares: recovering slot=115
[debug] (journal): 3: recover_prepares: recovering slot=116
[debug] (journal): 3: recover_prepares: recovering slot=117
[debug] (journal): 3: recover_prepares: recovering slot=118
[debug] (journal): 3: recover_prepares: recovering slot=119
[debug] (journal): 3: recover_prepares: recovering slot=120
[debug] (journal): 3: recover_prepares: recovering slot=121
[debug] (journal): 3: recover_prepares: recovering slot=122
[debug] (journal): 3: recover_prepares: recovering slot=123
[debug] (journal): 3: recover_prepares: recovering slot=124
[debug] (journal): 3: recover_prepares: recovering slot=125
[debug] (vsr): 0: ping_timeout fired
[debug] (vsr): 0: ping_timeout reset
[debug] (replica): 0: sending ping to replica 1: Header{ .checksum = 258851859345043016821811782666361315191, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32040000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.ping
[debug] (replica): 0: sending ping to replica 2: Header{ .checksum = 258851859345043016821811782666361315191, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32040000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 0: sending ping to replica 3: Header{ .checksum = 258851859345043016821811782666361315191, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32040000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.ping
[debug] (vsr): 0: commit_timeout fired
[debug] (vsr): 0: commit_timeout reset
[debug] (replica): 0: sending commit to replica 1: Header{ .checksum = 55389471341199156389722183142700569230, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 41826076010321932376190093941186211105, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.commit
[debug] (replica): 0: sending commit to replica 2: Header{ .checksum = 55389471341199156389722183142700569230, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 41826076010321932376190093941186211105, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.commit
[debug] (replica): 0: sending commit to replica 3: Header{ .checksum = 55389471341199156389722183142700569230, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 41826076010321932376190093941186211105, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.commit
[debug] (vsr): 0: repair_timeout fired
[debug] (vsr): 0: repair_timeout reset
[err] (packet_simulator): dropped packet (different partitions): from=0 to=2
[err] (packet_simulator): dropped packet (different partitions): from=0 to=2
[debug] (packet_simulator): delivering packet from=0 to=3
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.commit
[debug] (replica): 3: on_message: view=0 status=Status.recovering Header{ .checksum = 55389471341199156389722183142700569230, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 41826076010321932376190093941186211105, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_message: waiting for journal to recover
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.ping
[debug] (replica): 1: on_message: view=4 status=Status.normal Header{ .checksum = 258851859345043016821811782666361315191, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32040000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: sending pong to replica 0: Header{ .checksum = 336135443963642268759092352694838055874, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32040000000, .commit = 0, .timestamp = 32050000000, .size = 128, .replica = 1, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.pong
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.commit
[debug] (replica): 1: on_message: view=4 status=Status.normal Header{ .checksum = 55389471341199156389722183142700569230, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 41826076010321932376190093941186211105, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_commit: checksum verified
[debug] (vsr): 1: normal_status_timeout reset
[debug] (journal): 3: recover_prepares: recovering slot=126
[debug] (vsr): 1: ping_timeout fired
[debug] (vsr): 1: ping_timeout reset
[debug] (replica): 1: sending ping to replica 0: Header{ .checksum = 23498872623160497826195160217715798739, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32060000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 1: sending ping to replica 2: Header{ .checksum = 23498872623160497826195160217715798739, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32060000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 1: sending ping to replica 3: Header{ .checksum = 23498872623160497826195160217715798739, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32060000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.ping
[debug] (vsr): 1: repair_timeout fired
[debug] (vsr): 1: repair_timeout reset
[debug] (replica): 1: repair_prepare: op=1 checksum=268840143496320758578619296218401986711 (uncommitted, dirty)
[debug] (replica): 1: sending request_prepare to replica 0: Header{ .checksum = 78675336814232754273676860177577076796, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 268840143496320758578619296218401986711, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 1, .size = 128, .replica = 1, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.request_prepare
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.pong
[debug] (replica): 0: on_message: view=4 status=Status.normal Header{ .checksum = 336135443963642268759092352694838055874, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32040000000, .commit = 0, .timestamp = 32050000000, .size = 128, .replica = 1, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (clock): 0: learn: replica=1 m0=32040000000 t1=32050000000 m2=32060000000 t2=32060000000 one_way_delay=10000000 asymmetric_delay=0 clock_offset=0
[debug] (vsr): 2: ping_timeout fired
[debug] (vsr): 2: ping_timeout reset
[debug] (replica): 2: sending ping to replica 0: Header{ .checksum = 208186603263737554286491575604436420940, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32070000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 2: sending ping to replica 1: Header{ .checksum = 208186603263737554286491575604436420940, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32070000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.ping
[debug] (replica): 2: sending ping to replica 3: Header{ .checksum = 208186603263737554286491575604436420940, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32070000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 3 }: Command.ping
[debug] (vsr): 2: repair_timeout fired
[debug] (vsr): 2: repair_timeout reset
[debug] (replica): 2: repair_prepare: op=1 checksum=268840143496320758578619296218401986711 (uncommitted, dirty)
[debug] (replica): 2: sending request_prepare to replica 1: Header{ .checksum = 136833157251937701297687426511046091009, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 268840143496320758578619296218401986711, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 1, .size = 128, .replica = 2, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.request_prepare
[debug] (packet_simulator): replayed packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.request_prepare
[debug] (replica): 0: on_message: view=4 status=Status.normal Header{ .checksum = 78675336814232754273676860177577076796, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 268840143496320758578619296218401986711, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 1, .size = 128, .replica = 1, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_request_prepare: op=1 checksum=268840143496320758578619296218401986711 reply from pipeline
[debug] (replica): 0: sending prepare to replica 1: Header{ .checksum = 268840143496320758578619296218401986711, .checksum_body = 98081761841810822994326224503485043631, .parent = 41826076010321932376190093941186211105, .client = 237793620746173507671246977486275538381, .context = 174952180883201026607060349056975638411, .request = 0, .cluster = 0, .epoch = 0, .view = 3, .op = 1, .commit = 0, .timestamp = 26200000000, .size = 128, .replica = 3, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.prepare
[debug] (packet_simulator): delivering packet from=1 to=3
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.ping
[debug] (replica): 3: on_message: view=0 status=Status.recovering Header{ .checksum = 23498872623160497826195160217715798739, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32060000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_message: waiting for journal to recover
[err] (packet_simulator): dropped packet (different partitions): from=2 to=1
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.prepare
[debug] (replica): 1: on_message: view=4 status=Status.normal Header{ .checksum = 268840143496320758578619296218401986711, .checksum_body = 98081761841810822994326224503485043631, .parent = 41826076010321932376190093941186211105, .client = 237793620746173507671246977486275538381, .context = 174952180883201026607060349056975638411, .request = 0, .cluster = 0, .epoch = 0, .view = 3, .op = 1, .commit = 0, .timestamp = 26200000000, .size = 128, .replica = 3, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_prepare: ignoring (repair)
[debug] (replica): 1: repair_header: op=1 exists, checksum dirty
[debug] (journal): 1: set_header_as_dirty: op=1 checksum=268840143496320758578619296218401986711
[debug] (replica): 1: on_repair: repairing journal
[debug] (journal): 1: write: view=3 op=1 len=128: 268840143496320758578619296218401986711 starting
[debug] (journal): 1: write_sectors: offset=1064960 len=4096 locked
[err] (packet_simulator): dropped packet (different partitions): from=2 to=3
[debug] (journal): 3: recover_prepares: recovering slot=127
[debug] (journal): 1: write_sectors: offset=1064960 len=4096 unlocked
[debug] (journal): 1: write_header: op=1 sectors[0..4096]
[debug] (journal): 1: write_sectors: offset=0 len=4096 locked
[debug] (packet_simulator): replayed packet from=0 to=3
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.ping
[debug] (replica): 3: on_message: view=0 status=Status.recovering Header{ .checksum = 258851859345043016821811782666361315191, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32040000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_message: waiting for journal to recover
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.request_prepare
[debug] (replica): 0: on_message: view=4 status=Status.normal Header{ .checksum = 78675336814232754273676860177577076796, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 268840143496320758578619296218401986711, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 1, .size = 128, .replica = 1, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_request_prepare: op=1 checksum=268840143496320758578619296218401986711 reply from pipeline
[debug] (replica): 0: sending prepare to replica 1: Header{ .checksum = 268840143496320758578619296218401986711, .checksum_body = 98081761841810822994326224503485043631, .parent = 41826076010321932376190093941186211105, .client = 237793620746173507671246977486275538381, .context = 174952180883201026607060349056975638411, .request = 0, .cluster = 0, .epoch = 0, .view = 3, .op = 1, .commit = 0, .timestamp = 26200000000, .size = 128, .replica = 3, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.prepare
[debug] (journal): 3: recover_slot: recovered slot=0 label=@n decision=eql
[debug] (journal): 3: recover_slot: recovered slot=1 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=2 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=3 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=4 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=5 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=6 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=7 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=8 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=9 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=10 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=11 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=12 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=13 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=14 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=15 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=16 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=17 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=18 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=19 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=20 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=21 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=22 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=23 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=24 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=25 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=26 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=27 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=28 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=29 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=30 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=31 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=32 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=33 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=34 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=35 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=36 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=37 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=38 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=39 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=40 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=41 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=42 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=43 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=44 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=45 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=46 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=47 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=48 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=49 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=50 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=51 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=52 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=53 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=54 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=55 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=56 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=57 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=58 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=59 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=60 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=61 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=62 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=63 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=64 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=65 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=66 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=67 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=68 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=69 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=70 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=71 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=72 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=73 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=74 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=75 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=76 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=77 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=78 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=79 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=80 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=81 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=82 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=83 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=84 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=85 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=86 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=87 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=88 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=89 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=90 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=91 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=92 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=93 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=94 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=95 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=96 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=97 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=98 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=99 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=100 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=101 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=102 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=103 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=104 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=105 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=106 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=107 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=108 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=109 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=110 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=111 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=112 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=113 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=114 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=115 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=116 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=117 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=118 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=119 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=120 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=121 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=122 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=123 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=124 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=125 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=126 label=@j decision=nil
[debug] (journal): 3: recover_slot: recovered slot=127 label=@j decision=nil
[debug] (journal): 3: recover_slots: dirty=0 faulty=0
[debug] (replica): 3: transition_to_normal_from_recovering_status: view=0 follower
[debug] (vsr): 3: ping_timeout started
[debug] (vsr): 3: normal_status_timeout started
[debug] (vsr): 3: repair_timeout started
[debug] (vsr): 3: recovery_timeout stopped
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 0: on_message: view=4 status=Status.normal Header{ .checksum = 23498872623160497826195160217715798739, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32060000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: sending pong to replica 1: Header{ .checksum = 109233202099033338564783076933237443391, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32060000000, .commit = 0, .timestamp = 32110000000, .size = 128, .replica = 0, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.pong
[err] (packet_simulator): dropped packet (different partitions): from=1 to=2
[err] (packet_simulator): dropped packet (different partitions): from=2 to=1
[debug] (packet_simulator): replayed packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.prepare
[debug] (replica): 1: on_message: view=4 status=Status.normal Header{ .checksum = 268840143496320758578619296218401986711, .checksum_body = 98081761841810822994326224503485043631, .parent = 41826076010321932376190093941186211105, .client = 237793620746173507671246977486275538381, .context = 174952180883201026607060349056975638411, .request = 0, .cluster = 0, .epoch = 0, .view = 3, .op = 1, .commit = 0, .timestamp = 26200000000, .size = 128, .replica = 3, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_prepare: ignoring (repair)
[debug] (replica): 1: repair_header: op=1 exists, checksum dirty
[debug] (journal): 1: set_header_as_dirty: op=1 checksum=268840143496320758578619296218401986711
[debug] (replica): 1: on_repair: repairing journal
[debug] (replica): 1: write_prepare: ignoring op=1 checksum=268840143496320758578619296218401986711 (already writing)
[debug] (packet_simulator): replayed packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.pong
[debug] (replica): 1: on_message: view=4 status=Status.normal Header{ .checksum = 109233202099033338564783076933237443391, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32060000000, .commit = 0, .timestamp = 32110000000, .size = 128, .replica = 0, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (clock): 1: learn: replica=0 m0=32060000000 t1=32110000000 m2=32130000000 t2=32130000000 one_way_delay=35000000 asymmetric_delay=-15000000 clock_offset=0
[debug] (packet_simulator): replayed packet from=0 to=3
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.ping
[debug] (replica): 3: on_message: view=0 status=Status.normal Header{ .checksum = 258851859345043016821811782666361315191, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32040000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: sending pong to replica 0: Header{ .checksum = 276433559088926796997377753035742153328, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 32040000000, .commit = 0, .timestamp = 32130000000, .size = 128, .replica = 3, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 0 }: Command.pong
[err] (packet_simulator): dropped packet (different partitions): from=2 to=0
[debug] (clock): 1: synchronized: truechimers=3/4 clock_offset=0ns..0ns accuracy=0ns
[debug] (packet_simulator): replayed packet from=3 to=0
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 0 }: Command.pong
[debug] (replica): 0: on_message: view=4 status=Status.normal Header{ .checksum = 276433559088926796997377753035742153328, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 32040000000, .commit = 0, .timestamp = 32130000000, .size = 128, .replica = 3, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (clock): 0: learn: replica=3 m0=32040000000 t1=32130000000 m2=32140000000 t2=32140000000 one_way_delay=50000000 asymmetric_delay=-35000000 clock_offset=5000000
[debug] (clock): 0: synchronized: truechimers=3/4 clock_offset=0ns..0ns accuracy=0ns
[debug] (packet_simulator): replayed packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.prepare
[debug] (replica): 1: on_message: view=4 status=Status.normal Header{ .checksum = 268840143496320758578619296218401986711, .checksum_body = 98081761841810822994326224503485043631, .parent = 41826076010321932376190093941186211105, .client = 237793620746173507671246977486275538381, .context = 174952180883201026607060349056975638411, .request = 0, .cluster = 0, .epoch = 0, .view = 3, .op = 1, .commit = 0, .timestamp = 26200000000, .size = 128, .replica = 3, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_prepare: ignoring (repair)
[debug] (replica): 1: repair_header: op=1 exists, checksum dirty
[debug] (journal): 1: set_header_as_dirty: op=1 checksum=268840143496320758578619296218401986711
[debug] (replica): 1: on_repair: repairing journal
[debug] (replica): 1: write_prepare: ignoring op=1 checksum=268840143496320758578619296218401986711 (already writing)
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.pong
[debug] (replica): 1: on_message: view=4 status=Status.normal Header{ .checksum = 109233202099033338564783076933237443391, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 32060000000, .commit = 0, .timestamp = 32110000000, .size = 128, .replica = 0, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[warn] (clock): 1: learn: m0=32060000000 < window.monotonic=32140000000
[err] (packet_simulator): dropped packet from=0 to=3.
[debug] (packet_simulator): delivering packet from=3 to=0
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 0 }: Command.pong
[debug] (replica): 0: on_message: view=4 status=Status.normal Header{ .checksum = 276433559088926796997377753035742153328, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 32040000000, .commit = 0, .timestamp = 32130000000, .size = 128, .replica = 3, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[warn] (clock): 0: learn: m0=32040000000 < window.monotonic=32150000000
[debug] (packet_simulator): replayed packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.prepare
[debug] (replica): 1: on_message: view=4 status=Status.normal Header{ .checksum = 268840143496320758578619296218401986711, .checksum_body = 98081761841810822994326224503485043631, .parent = 41826076010321932376190093941186211105, .client = 237793620746173507671246977486275538381, .context = 174952180883201026607060349056975638411, .request = 0, .cluster = 0, .epoch = 0, .view = 3, .op = 1, .commit = 0, .timestamp = 26200000000, .size = 128, .replica = 3, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_prepare: ignoring (repair)
[debug] (replica): 1: repair_header: op=1 exists, checksum dirty
[debug] (journal): 1: set_header_as_dirty: op=1 checksum=268840143496320758578619296218401986711
[debug] (replica): 1: on_repair: repairing journal
[debug] (replica): 1: write_prepare: ignoring op=1 checksum=268840143496320758578619296218401986711 (already writing)
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.prepare
[debug] (replica): 1: on_message: view=4 status=Status.normal Header{ .checksum = 268840143496320758578619296218401986711, .checksum_body = 98081761841810822994326224503485043631, .parent = 41826076010321932376190093941186211105, .client = 237793620746173507671246977486275538381, .context = 174952180883201026607060349056975638411, .request = 0, .cluster = 0, .epoch = 0, .view = 3, .op = 1, .commit = 0, .timestamp = 26200000000, .size = 128, .replica = 3, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_prepare: ignoring (repair)
[debug] (replica): 1: repair_header: op=1 exists, checksum dirty
[debug] (journal): 1: set_header_as_dirty: op=1 checksum=268840143496320758578619296218401986711
[debug] (replica): 1: on_repair: repairing journal
[debug] (replica): 1: write_prepare: ignoring op=1 checksum=268840143496320758578619296218401986711 (already writing)
[debug] (journal): 1: write_sectors: offset=0 len=4096 unlocked
[debug] (journal): 1: write: view=3 op=1 len=128: 268840143496320758578619296218401986711 complete, marking clean
[debug] (replica): 1: send_prepare_ok: op=1 checksum=268840143496320758578619296218401986711
[debug] (replica): 1: sending prepare_ok to replica 0: Header{ .checksum = 16682743658641968250990409736020086628, .checksum_body = 98081761841810822994326224503485043631, .parent = 41826076010321932376190093941186211105, .client = 237793620746173507671246977486275538381, .context = 268840143496320758578619296218401986711, .request = 0, .cluster = 0, .epoch = 0, .view = 4, .op = 1, .commit = 0, .timestamp = 26200000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.prepare_ok
[debug] (vsr): 1: repair_timeout reset
[debug] (vsr): 0: prepare_timeout fired
[debug] (vsr): 0: prepare_timeout backing off
[debug] (vsr): 0: prepare_timeout after=76..85 (rtt=30 min=10 max=1000 attempts=2)
[debug] (replica): 0: on_prepare_timeout: waiting for replica 1
[debug] (replica): 0: on_prepare_timeout: waiting for replica 2
[debug] (replica): 0: on_prepare_timeout: waiting for replica 3
[debug] (replica): 0: on_prepare_timeout: replicating to replica 3
[debug] (replica): 0: sending prepare to replica 3: Header{ .checksum = 268840143496320758578619296218401986711, .checksum_body = 98081761841810822994326224503485043631, .parent = 41826076010321932376190093941186211105, .client = 237793620746173507671246977486275538381, .context = 174952180883201026607060349056975638411, .request = 0, .cluster = 0, .epoch = 0, .view = 3, .op = 1, .commit = 0, .timestamp = 26200000000, .size = 128, .replica = 3, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.prepare
[debug] (packet_simulator): delivering packet from=0 to=3
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.prepare
[debug] (replica): 3: on_message: view=0 status=Status.normal Header{ .checksum = 268840143496320758578619296218401986711, .checksum_body = 98081761841810822994326224503485043631, .parent = 41826076010321932376190093941186211105, .client = 237793620746173507671246977486275538381, .context = 174952180883201026607060349056975638411, .request = 0, .cluster = 0, .epoch = 0, .view = 3, .op = 1, .commit = 0, .timestamp = 26200000000, .size = 128, .replica = 3, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 3: view_jump: waiting to jump to newer view
[debug] (replica): 3: view_jump: requesting start_view message
[debug] (replica): 3: sending request_start_view to replica 3: Header{ .checksum = 102881407742356223729951634595514505927, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 3, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.request_start_view, .operation = Operation.reserved, .version = 0 }
[info] (replica): 3: send_message_to_replica: TODO request_start_view
[debug] (replica): 3: on_prepare: ignoring (newer view)
[err] (replica): 3: on_message: on_prepare() queued a request_start_view loopback message with no flush

Time:
2022-07-11 05:46:19.942304775 +0000 UTC

If we fix #70 then we need to do the same in Network.deliver_message to match this logic.

But this is a sign that we actually need to refactor MessageBus a little so that it accepts a swappable Network which does the TCP connection stuff that MessageBus.Connection currently does.

The goal of this refactor would be to get rid of test/message_bus.zig completely and make sure that we’re testing the exact same message bus logic.

@ifreund what are your thoughts on this?

Some Tigerbeetle Docker images crash when run:

$ docker run ghcr.io/wilsonianb/tigerbeetle@sha256:e48c8b65227f0edc8828e9d8854c42537d68472338b1276a274fc07dc0bfa3f7
error: no command provided, expected 'start' or 'init'

$ docker run ghcr.io/coilhq/tigerbeetle@sha256:0809faad8ca8089c3602c367fcef967a76458911bef684322737e2f5086a1cb8
$ docker run -it --entrypoint /bin/bash ghcr.io/coilhq/tigerbeetle@sha256:0809faad8ca8089c3602c367fcef967a76458911bef684322737e2f5086a1cb8
root@57e41b6bef46:/opt/beta-beetle# ./tigerbeetle 
Illegal instruction (core dumped)
root@57e41b6bef46:/opt/beta-beetle# apt-get update
root@57e41b6bef46:/opt/beta-beetle# apt-get install gdb
root@57e41b6bef46:/opt/beta-beetle# gdb ./tigerbeetle
(gdb) start
Temporary breakpoint 1 at 0x210022: file /opt/beta-beetle/zig/lib/std/heap/arena_allocator.zig, line 28.
Starting program: /opt/beta-beetle/tigerbeetle 
warning: Error disabling address space randomization: Operation not permitted

Program received signal SIGILL, Illegal instruction.
0x000000000024ecf4 in memset ()
(gdb) where
#0  0x000000000024ecf4 in memset ()
#1  0x000000000020fccf in std.mem.set (dest=..., value=<optimized out>) at /opt/beta-beetle/zig/lib/std/mem.zig:182
#2  std.os.linux.tls.prepareTLS (area=...) at /opt/beta-beetle/zig/lib/std/os/linux/tls.zig:283
#3  std.os.linux.tls.initStaticTLS (phdrs=...) at /opt/beta-beetle/zig/lib/std/os/linux/tls.zig:340
#4  std.start.posixCallMainAndExit () at /opt/beta-beetle/zig/lib/std/start.zig:323
#5  0x000000000020f9ef in std.start._start () at /opt/beta-beetle/zig/lib/std/start.zig:245

Could that fact that this is happening with some images built via the GitHub Action and not others be due inconsistencies across GitHub's servers (similar to tigerbeetle/tigerbeetle-node#13)?

Bug Type:
crash

Seed:
11746549870101989313

Commit Hash:
01494c5

Parameters:
[info] (state_checker):
SEED=11746549870101989313

replicas=5
clients=17
request_probability=2%
idle_on_probability=15%
idle_off_probability=19%
one_way_delay_mean=4 ticks
one_way_delay_min=1 ticks
packet_loss_probability=22%
path_maximum_capacity=19 messages
path_clog_duration_mean=291 ticks
path_clog_probability=1%
packet_replay_probability=13%
partition_mode=PartitionMode.uniform_partition
partition_probability=1%
unpartition_probability=8%
partition_stability=190 ticks
unpartition_stability=6 ticks
read_latency_min=0
read_latency_mean=3
write_latency_min=2
write_latency_mean=10
read_fault_probability=3%
write_fault_probability=8%
crash_probability=0.01%
crash_stability=777 ticks
restart_probability=1%
restart_stability=423 ticks

Stack Trace:

thread panic: reached unreachable code
zig/lib/std/debug.zig:225:14: in std.debug.assert (simulator)
if (!ok) unreachable; // assertion failure
^
src/vsr/journal.zig:623:35: in vsr.journal.Journal(vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time),test.storage.Storage).find_latest_headers_break_between (simulator)
assert(a.op > op_min);
^
src/vsr/replica.zig:3422:72: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).repair (simulator)
var broken = self.journal.find_latest_headers_break_between(self.commit_min, self.op);
^
src/vsr/replica.zig:1099:24: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).on_do_view_change (simulator)
self.repair();
^
src/vsr/replica.zig:529:58: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).on_message (simulator)
.do_view_change => self.on_do_view_change(message),
^
src/test/message_bus.zig:57:27: in test.message_bus.struct:55:35.wrapper (simulator)
on_message(@intToPtr(Context, @ptrToInt(_context)), message);
^
src/test/network.zig:167:41: in test.network.Network.deliver_message (simulator)
target_bus.on_message_callback.?(target_bus.on_message_context, message);
^
src/test/packet_simulator.zig:345:42: in test.packet_simulator.PacketSimulator(test.network.Packet).tick (simulator)
data.callback(data.packet, path);
^
src/simulator.zig:277:46: in main (simulator)
cluster.network.packet_simulator.tick(cluster.health);
^
zig/lib/std/start.zig:561:37: in std.start.callMain (simulator)
const result = root.main() catch |err| {
^
zig/lib/std/start.zig:495:12: in std.start.callMainWithArgs (simulator)
return @call(.{ .modifier = .always_inline }, callMain, .{});
^
zig/lib/std/start.zig:409:17: in std.start.posixCallMainAndExit (simulator)
std.os.exit(@call(.{ .modifier = .always_inline }, callMainWithArgs, .{ argc, argv, envp }));
^
zig/lib/std/start.zig:322:5: in std.start._start (simulator)
@call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{});
^
exit with signal: 6. Indicates a crash bug.


Tail of Debug Logs:
, .epoch = 0, .view = 1, .op = 14960000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 1: sending ping to replica 2: Header{ .checksum = 184245398039798532329853387547165508405, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 14960000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 1: sending ping to replica 3: Header{ .checksum = 184245398039798532329853387547165508405, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 14960000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.ping
[debug] (replica): 1: sending ping to replica 4: Header{ .checksum = 184245398039798532329853387547165508405, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 14960000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 4 }: Command.ping
[debug] (vsr): 1: commit_timeout fired
[debug] (vsr): 1: commit_timeout reset
[debug] (replica): 1: sending commit to replica 0: Header{ .checksum = 86790830683783432182162740515292319334, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 272547437104785252098024083079431926410, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 0, .commit = 1, .timestamp = 0, .size = 128, .replica = 1, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.commit
[debug] (replica): 1: sending commit to replica 2: Header{ .checksum = 86790830683783432182162740515292319334, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 272547437104785252098024083079431926410, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 0, .commit = 1, .timestamp = 0, .size = 128, .replica = 1, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.commit
[debug] (replica): 1: sending commit to replica 3: Header{ .checksum = 86790830683783432182162740515292319334, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 272547437104785252098024083079431926410, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 0, .commit = 1, .timestamp = 0, .size = 128, .replica = 1, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.commit
[debug] (replica): 1: sending commit to replica 4: Header{ .checksum = 86790830683783432182162740515292319334, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 272547437104785252098024083079431926410, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 0, .commit = 1, .timestamp = 0, .size = 128, .replica = 1, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 4 }: Command.commit
[debug] (vsr): 1: repair_timeout fired
[debug] (vsr): 1: repair_timeout reset
[err] (packet_simulator): dropped packet from=1 to=2.
[err] (packet_simulator): dropped packet (different partitions): from=1 to=4
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=3 clogged for ticks=348
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=13 clogged for ticks=70
[err] (packet_simulator): dropped packet (different partitions): from=0 to=1
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.commit
[debug] (replica): 2: on_message: view=1 status=Status.normal Header{ .checksum = 86790830683783432182162740515292319334, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 272547437104785252098024083079431926410, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 0, .commit = 1, .timestamp = 0, .size = 128, .replica = 1, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_commit: checksum verified
[debug] (vsr): 2: normal_status_timeout reset
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=6 clogged for ticks=208
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=6 clogged for ticks=117
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=10 clogged for ticks=64
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=14 clogged for ticks=75
[err] (packet_simulator): dropped packet (different partitions): from=0 to=2
[err] (packet_simulator): dropped packet (different partitions): from=0 to=2
[err] (packet_simulator): dropped packet (different partitions): from=0 to=2
[err] (packet_simulator): dropped packet (different partitions): from=1 to=4
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=16 clogged for ticks=1
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=17 clogged for ticks=191
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=5 clogged for ticks=896
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=16 clogged for ticks=11
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=17 clogged for ticks=110
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=14 clogged for ticks=371
[err] (packet_simulator): dropped packet from=16 to=1.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=18 clogged for ticks=228
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=4 clogged for ticks=23
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=20 clogged for ticks=135
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=0 clogged for ticks=543
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=8 clogged for ticks=384
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=9 clogged for ticks=23
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=18 clogged for ticks=166
[debug] (vsr): 2: ping_timeout fired
[debug] (vsr): 2: ping_timeout reset
[debug] (replica): 2: sending ping to replica 0: Header{ .checksum = 125589496854383582928743613402714603829, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 15040000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 2: sending ping to replica 1: Header{ .checksum = 125589496854383582928743613402714603829, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 15040000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.ping
[debug] (replica): 2: sending ping to replica 3: Header{ .checksum = 125589496854383582928743613402714603829, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 15040000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 3 }: Command.ping
[debug] (replica): 2: sending ping to replica 4: Header{ .checksum = 125589496854383582928743613402714603829, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 15040000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 4 }: Command.ping
[debug] (vsr): 2: repair_timeout fired
[debug] (vsr): 2: repair_timeout reset
[debug] (replica): 2: repair: break: view=1 op_min=2 op_max=3 (commit=1..1 op=4)
[debug] (replica): 2: sending request_headers to replica 0: Header{ .checksum = 302393410004030818820448862473753780522, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 3, .commit = 2, .timestamp = 0, .size = 128, .replica = 2, .command = Command.request_headers, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.request_headers
[err] (packet_simulator): dropped packet (different partitions): from=2 to=0
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=2 clogged for ticks=665
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=2 clogged for ticks=260
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=6 clogged for ticks=96
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=8 clogged for ticks=175
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=9 clogged for ticks=87
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=11 clogged for ticks=533
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=12 clogged for ticks=615
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=6 clogged for ticks=645
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=10 clogged for ticks=543
[err] (packet_simulator): dropped packet (different partitions): from=2 to=0
[err] (packet_simulator): dropped packet from=3 to=0.
[debug] (packet_simulator): delivering packet from=3 to=0
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2 status=Status.view_change Header{ .checksum = 62844024048599795057020649544476481084, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: 1 message(s)
[debug] (replica): 0: on_start_view_change: waiting for quorum
[err] (packet_simulator): dropped packet from=3 to=0.
[debug] (packet_simulator): delivering packet from=3 to=0
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2 status=Status.view_change Header{ .checksum = 62844024048599795057020649544476481084, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=3)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=9 clogged for ticks=84
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=20 clogged for ticks=360
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=5 clogged for ticks=97
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=8 clogged for ticks=126
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=19 clogged for ticks=149
[err] (packet_simulator): dropped packet (different partitions): from=2 to=3
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=2 clogged for ticks=49
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=19 clogged for ticks=196
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=3 clogged for ticks=151
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=8 clogged for ticks=446
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=14 clogged for ticks=125
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=3 clogged for ticks=313
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=21 clogged for ticks=137
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=6 clogged for ticks=338
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=8 clogged for ticks=195
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=20 clogged for ticks=322
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=15 clogged for ticks=244
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=0 clogged for ticks=51
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=2 clogged for ticks=36
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=8 clogged for ticks=342
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=11 clogged for ticks=124
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=12 clogged for ticks=39
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=14 clogged for ticks=226
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=0 clogged for ticks=81
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=8 clogged for ticks=364
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=8 clogged for ticks=127
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=10 clogged for ticks=639
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=19 clogged for ticks=589
[err] (packet_simulator): unpartitioned network: partition={ false, false, false, false, false }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=4 clogged for ticks=149
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=5 clogged for ticks=533
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=10 clogged for ticks=235
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=18 clogged for ticks=251
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=0 clogged for ticks=358
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=1 clogged for ticks=326
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=9 clogged for ticks=224
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=1 clogged for ticks=281
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=6 clogged for ticks=615
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=13 clogged for ticks=398
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=16 clogged for ticks=47
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=17 clogged for ticks=775
[err] (packet_simulator): dropped packet (destination is crashed): from=16 to=4
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=2 clogged for ticks=52
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=8 clogged for ticks=208
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=10 clogged for ticks=915
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=21 clogged for ticks=547
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=0 clogged for ticks=46
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=1 clogged for ticks=25
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=1 clogged for ticks=26
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=17 clogged for ticks=157
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=1 clogged for ticks=51
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=1 clogged for ticks=53
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=16 clogged for ticks=49
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=18 clogged for ticks=607
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=18 clogged for ticks=482
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=14 clogged for ticks=93
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=18 clogged for ticks=119
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=2 clogged for ticks=317
[debug] (vsr): 137104105323283174215073365599202958438: request_timeout fired
[debug] (vsr): 137104105323283174215073365599202958438: request_timeout backing off
[debug] (vsr): 137104105323283174215073365599202958438: request_timeout after=156..381 (rtt=30 min=10 max=1000 attempts=5)
[debug] (client): 137104105323283174215073365599202958438: on_request_timeout: resending request=0 checksum=8786034670376991329784881759978823021
[debug] (client): 137104105323283174215073365599202958438: sending request to replica 0: Header{ .checksum = 8786034670376991329784881759978823021, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 137104105323283174215073365599202958438, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .client = 137104105323283174215073365599202958438 } > Process{ .replica = 0 }: Command.request
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=4 clogged for ticks=109
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=5 clogged for ticks=381
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=13 clogged for ticks=247
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=20 clogged for ticks=32
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=2 clogged for ticks=1161
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=18 clogged for ticks=42
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=1 clogged for ticks=31
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=5 clogged for ticks=386
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=11 clogged for ticks=108
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=12 clogged for ticks=196
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=19 clogged for ticks=73
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=5 clogged for ticks=458
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=12 clogged for ticks=330
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=15 clogged for ticks=826
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=8 clogged for ticks=227
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=2 clogged for ticks=166
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=15 clogged for ticks=73
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=15 clogged for ticks=285
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=18 clogged for ticks=229
[debug] (vsr): 0: view_change_message_timeout fired
[debug] (vsr): 0: view_change_message_timeout reset
[debug] (replica): 0: sending start_view_change to replica 1: Header{ .checksum = 59831424487230592914298009848171186311, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 2: Header{ .checksum = 59831424487230592914298009848171186311, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 3: Header{ .checksum = 59831424487230592914298009848171186311, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 4: Header{ .checksum = 59831424487230592914298009848171186311, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 4 }: Command.start_view_change
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=1 status=Status.normal Header{ .checksum = 59831424487230592914298009848171186311, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: view_jump: jumping to view change
[debug] (replica): 2: transition_to_view_change_status: view=1..2
[debug] (vsr): 2: ping_timeout stopped
[debug] (vsr): 2: commit_timeout stopped
[debug] (vsr): 2: normal_status_timeout stopped
[debug] (vsr): 2: view_change_status_timeout started
[debug] (vsr): 2: view_change_message_timeout started
[debug] (vsr): 2: repair_timeout stopped
[debug] (vsr): 2: prepare_timeout stopped
[debug] (replica): 2: reset 0 do_view_change message(s) from view=null
[debug] (replica): 2: sending start_view_change to replica 0: Header{ .checksum = 136425021088429627563473849390192892040, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 1: Header{ .checksum = 136425021088429627563473849390192892040, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 3: Header{ .checksum = 136425021088429627563473849390192892040, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 4: Header{ .checksum = 136425021088429627563473849390192892040, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 4 }: Command.start_view_change
[debug] (replica): 2: on_start_view_change: 1 message(s)
[debug] (replica): 2: on_start_view_change: waiting for quorum
[debug] (packet_simulator): delivering packet from=0 to=3
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (replica): 3: on_message: view=2 status=Status.view_change Header{ .checksum = 59831424487230592914298009848171186311, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_start_view_change: ignoring (duplicate message replica=0)
[err] (packet_simulator): dropped packet (destination is crashed): from=0 to=4
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=8 clogged for ticks=376
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=17 clogged for ticks=50
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=18 clogged for ticks=452
[debug] (vsr): 3: view_change_message_timeout fired
[debug] (vsr): 3: view_change_message_timeout reset
[debug] (replica): 3: sending start_view_change to replica 0: Header{ .checksum = 62844024048599795057020649544476481084, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 3: sending start_view_change to replica 1: Header{ .checksum = 62844024048599795057020649544476481084, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 3: sending start_view_change to replica 2: Header{ .checksum = 62844024048599795057020649544476481084, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 3: sending start_view_change to replica 4: Header{ .checksum = 62844024048599795057020649544476481084, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 4 }: Command.start_view_change
[err] (packet_simulator): dropped packet from=3 to=1.
[debug] (packet_simulator): delivering packet from=3 to=2
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=2 status=Status.view_change Header{ .checksum = 62844024048599795057020649544476481084, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: 2 message(s)
[debug] (replica): 2: on_start_view_change: view=2 quorum received
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=4 dest.len=32 copied=3
[debug] (replica): 2: sending do_view_change to replica 2: Header{ .checksum = 203803697529050773946246957892979984432, .checksum_body = 244996537321732496618077015798867701943, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 4, .commit = 1, .timestamp = 1, .size = 512, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_message: view=2 status=Status.view_change Header{ .checksum = 203803697529050773946246957892979984432, .checksum_body = 244996537321732496618077015798867701943, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 4, .commit = 1, .timestamp = 1, .size = 512, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_do_view_change: 1 message(s)
[debug] (replica): 2: on_do_view_change: waiting for quorum
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=9 clogged for ticks=74
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=18 clogged for ticks=28
[debug] (packet_simulator): replayed packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=1 status=Status.normal Header{ .checksum = 59831424487230592914298009848171186311, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: view_jump: jumping to view change
[debug] (replica): 1: transition_to_view_change_status: view=1..2
[debug] (vsr): 1: ping_timeout stopped
[debug] (vsr): 1: commit_timeout stopped
[debug] (vsr): 1: normal_status_timeout stopped
[debug] (vsr): 1: view_change_status_timeout started
[debug] (vsr): 1: view_change_message_timeout started
[debug] (vsr): 1: repair_timeout stopped
[debug] (vsr): 1: prepare_timeout stopped
[debug] (replica): 1: reset 0 do_view_change message(s) from view=null
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 264862478639762090336795378579111683270, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 264862478639762090336795378579111683270, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 3: Header{ .checksum = 264862478639762090336795378579111683270, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 4: Header{ .checksum = 264862478639762090336795378579111683270, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 4 }: Command.start_view_change
[debug] (replica): 1: on_start_view_change: 1 message(s)
[debug] (replica): 1: on_start_view_change: waiting for quorum
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=2 clogged for ticks=158
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=10 clogged for ticks=174
[debug] (vsr): 257464513956530928660994217023415938498: request_timeout fired
[debug] (vsr): 257464513956530928660994217023415938498: request_timeout backing off
[debug] (vsr): 257464513956530928660994217023415938498: request_timeout after=225..371 (rtt=30 min=10 max=1000 attempts=8)
[debug] (client): 257464513956530928660994217023415938498: on_request_timeout: resending request=0 checksum=278009621999753767736185294821202942642
[debug] (client): 257464513956530928660994217023415938498: sending request to replica 3: Header{ .checksum = 278009621999753767736185294821202942642, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 257464513956530928660994217023415938498, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .client = 257464513956530928660994217023415938498 } > Process{ .replica = 3 }: Command.request
[debug] (test_client): client 5 sending input=b2222bc634f8e25e687e773918244a4b
[debug] (client): 195590855347696333573199661806245137839: register: registering a session with the cluster
[debug] (client): 195590855347696333573199661806245137839: send_request_for_the_first_time: request=0 checksum=149606785874840276036033255480676906146
[debug] (vsr): 195590855347696333573199661806245137839: request_timeout started
[debug] (client): 195590855347696333573199661806245137839: sending request to replica 0: Header{ .checksum = 149606785874840276036033255480676906146, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 195590855347696333573199661806245137839, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .client = 195590855347696333573199661806245137839 } > Process{ .replica = 0 }: Command.request
[debug] (client): 195590855347696333573199661806245137839: request: user_data=0 request=1 size=900948 hash
[err] (packet_simulator): dropped packet from=0 to=1.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=0 clogged for ticks=7
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=1 clogged for ticks=39
[debug] (packet_simulator): delivering packet from=2 to=3
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (replica): 3: on_message: view=2 status=Status.view_change Header{ .checksum = 136425021088429627563473849390192892040, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_start_view_change: 2 message(s)
[debug] (replica): 3: on_start_view_change: view=2 quorum received
[debug] (journal): 3: copy_latest_headers_between: op_min=0 op_max=1 dest.len=32 copied=2
[debug] (replica): 3: sending do_view_change to replica 2: Header{ .checksum = 33045645776996403287555362055745826116, .checksum_body = 324194444706493311253014218772077319190, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 1, .commit = 1, .timestamp = 1, .size = 384, .replica = 3, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=2 clogged for ticks=2
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=2 clogged for ticks=192
[debug] (packet_simulator): delivering packet from=3 to=0
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2 status=Status.view_change Header{ .checksum = 62844024048599795057020649544476481084, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=3)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=18 clogged for ticks=188
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=0 clogged for ticks=121
[debug] (packet_simulator): delivering packet from=3 to=2
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (replica): 2: on_message: view=2 status=Status.view_change Header{ .checksum = 33045645776996403287555362055745826116, .checksum_body = 324194444706493311253014218772077319190, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 1, .commit = 1, .timestamp = 1, .size = 384, .replica = 3, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_do_view_change: 2 message(s)
[debug] (replica): 2: on_do_view_change: waiting for quorum
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=14 clogged for ticks=100
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=14 clogged for ticks=126
[debug] (packet_simulator): delivering packet from=18 to=3
[debug] (network): deliver_message: Process{ .client = 257464513956530928660994217023415938498 } > Process{ .replica = 3 }: Command.request
[debug] (replica): 3: on_message: view=2 status=Status.view_change Header{ .checksum = 278009621999753767736185294821202942642, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 257464513956530928660994217023415938498, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (replica): 3: on_request: ignoring (Status.view_change)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=7 clogged for ticks=97
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=8 clogged for ticks=199
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=17 clogged for ticks=350
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=19 clogged for ticks=156
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=2 status=Status.view_change Header{ .checksum = 264862478639762090336795378579111683270, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: 3 message(s)
[debug] (replica): 2: on_start_view_change: ignoring (quorum received already)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=16 clogged for ticks=466
[debug] (vsr): 325466199050141668327283753387245365558: request_timeout fired
[debug] (vsr): 325466199050141668327283753387245365558: request_timeout backing off
[debug] (vsr): 325466199050141668327283753387245365558: request_timeout after=86..123 (rtt=30 min=10 max=1000 attempts=3)
[debug] (client): 325466199050141668327283753387245365558: on_request_timeout: resending request=0 checksum=210765682250165618195543584595303768595
[debug] (client): 325466199050141668327283753387245365558: sending request to replica 3: Header{ .checksum = 210765682250165618195543584595303768595, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 325466199050141668327283753387245365558, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .client = 325466199050141668327283753387245365558 } > Process{ .replica = 3 }: Command.request
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=11 clogged for ticks=189
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=2 status=Status.view_change Header{ .checksum = 136425021088429627563473849390192892040, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: 2 message(s)
[debug] (replica): 0: on_start_view_change: view=2 quorum received
[debug] (journal): 0: copy_latest_headers_between: op_min=0 op_max=2 dest.len=32 copied=3
[debug] (replica): 0: sending do_view_change to replica 2: Header{ .checksum = 77722613206554353978369960632179373793, .checksum_body = 272492274498488483664453974108305932924, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 2, .commit = 0, .timestamp = 0, .size = 512, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=12 clogged for ticks=19
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=14 clogged for ticks=95
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=16 clogged for ticks=14
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=19 clogged for ticks=24
[debug] (packet_simulator): replayed packet from=13 to=0
[debug] (network): deliver_message: Process{ .client = 130181617704579180304885987215078420842 } > Process{ .replica = 0 }: Command.request
[debug] (replica): 0: on_message: view=2 status=Status.view_change Header{ .checksum = 320271865386523006375040117394838967225, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 130181617704579180304885987215078420842, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (replica): 0: on_request: ignoring (Status.view_change)
[debug] (packet_simulator): delivering packet from=14 to=3
[debug] (network): deliver_message: Process{ .client = 325466199050141668327283753387245365558 } > Process{ .replica = 3 }: Command.request
[debug] (replica): 3: on_message: view=2 status=Status.view_change Header{ .checksum = 210765682250165618195543584595303768595, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 325466199050141668327283753387245365558, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (replica): 3: on_request: ignoring (Status.view_change)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=14 clogged for ticks=149
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=18 clogged for ticks=445
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=4 clogged for ticks=1274
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=6 clogged for ticks=598
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=8 clogged for ticks=180
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (replica): 2: on_message: view=2 status=Status.view_change Header{ .checksum = 77722613206554353978369960632179373793, .checksum_body = 272492274498488483664453974108305932924, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 2, .commit = 0, .timestamp = 0, .size = 512, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_do_view_change: 3 message(s)
[debug] (replica): 2: on_do_view_change: view=2 quorum received
[debug] (replica): 2: on_do_view_change: replica=0 v'=0 op=2 commit=0 latest=Header{ .checksum = 3336240371590269624346461537204620709, .checksum_body = 98081761841810822994326224503485043631, .parent = 294262304064010895990498172186876555485, .client = 297217825344381698614179426234564775451, .context = 14843489691424821822323893732174248428, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 2, .commit = 0, .timestamp = 8290000000, .size = 128, .replica = 0, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 2: on_do_view_change: replica=2 v'=1 op=4 commit=1 latest=Header{ .checksum = 319341630697624241070987526820358795764, .checksum_body = 98081761841810822994326224503485043631, .parent = 329722307872151252293894011094716301819, .client = 137104105323283174215073365599202958438, .context = 8786034670376991329784881759978823021, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 4, .commit = 1, .timestamp = 13780000000, .size = 128, .replica = 1, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 2: on_do_view_change: replica=3 v'=1 op=1 commit=1 latest=Header{ .checksum = 272547437104785252098024083079431926410, .checksum_body = 98081761841810822994326224503485043631, .parent = 41826076010321932376190093941186211105, .client = 297217825344381698614179426234564775451, .context = 14843489691424821822323893732174248428, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 1, .commit = 0, .timestamp = 10380000000, .size = 128, .replica = 1, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 2: on_do_view_change: view=2 op=4..4 commit=1..1 checksum=319341630697624241070987526820358795764
[debug] (replica): 2: on_do_view_change: latest op exists exactly
[debug] (journal): 2: remove_entries_from: op_min=5
[debug] (replica): 2: repair_header: op=2 gap
[debug] (journal): 2: set_header_as_dirty: op=2 checksum=3336240371590269624346461537204620709
[debug] (replica): 2: repair_header: op=1 false (exists)
[debug] (replica): 2: repair_header: op=0 false (checksum clean)
[debug] (replica): 2: repair_header: op=4 false (checksum clean)
[debug] (replica): 2: repair_header: op=1 false (checksum clean)
[debug] (replica): 2: repair_header: op=0 false (checksum clean)
[debug] (replica): 2: repair_header: op=1 false (checksum clean)
[debug] (replica): 2: repair_header: op=0 false (checksum clean)
[debug] (replica): 2: discard_uncommitted_headers: op=3 gap
[debug] (replica): 2: discard_uncommitted_headers: replica=0 op=3 nack=true
[debug] (replica): 2: discard_uncommitted_headers: replica=3 op=3 nack=true
[debug] (replica): 2: discard_uncommitted_headers: op=3 nacks=2 threshold=2
[debug] (journal): 2: remove_entries_from: op_min=3
[debug] (vsr): 2: repair_timeout started
[debug] (vsr): 2: repair_timeout reset

Time:
2022-07-12 00:22:00.285364914 +0000 UTC

The code in src/test/storage.zig is still hardcoded to support up to 5 replicas

We do have a separate pool containing many more message_bus_headers_max messages in addition to the full-sized messages in message_bus_messages_max but MessageBus is using up a whole full-sized message for the incoming prepare_ok messages.

We should optimize this in MessageBus, so that if the header.size is exactly @sizeOf(Header) then we copy the header into a header-only message, and unref the full message.

This would make it less likely that the MessageBus would run out of full-sized messages. It also means we can reduce our memory overhead by reducing the duration for which we hold onto full messages that we recv into at the TCP layer. i.e. Our pool becomes more just-in-time, for less message "inventory holding costs".

At present, our config only allows each replica to have 152 full-sized messages allocated but the pipeline alone (not counting any outgoing send queues even) actually requires 192 messages, so we can deadlock and not have enough messages to make progress if the network reorders all available messages before the first message in the pipeline.

This is because we have an off-by-one when we calculate config.message_bus_messages_max since we’re only counting config.replicas_max * prepare_ok messages in the pipeline, but not also the corresponding single prepare message per pipeline slot.

We should also add a comptime assertion to cross-check all these things, for example that message_bus_messages_max is sufficient to cover the pipeline plus send queues. Let's add these at the end of config.zig also so that any file using the config knows that it's already been checked.

Bug Type:
correctness

Seed:
13067532511945936011

Commit Hash:
01494c5

Parameters:
[info] (state_checker):
SEED=13067532511945936011

replicas=3
clients=20
request_probability=38%
idle_on_probability=14%
idle_off_probability=11%
one_way_delay_mean=8 ticks
one_way_delay_min=1 ticks
packet_loss_probability=9%
path_maximum_capacity=15 messages
path_clog_duration_mean=399 ticks
path_clog_probability=1%
packet_replay_probability=43%
partition_mode=PartitionMode.isolate_single
partition_probability=1%
unpartition_probability=10%
partition_stability=188 ticks
unpartition_stability=1 ticks
read_latency_min=1
read_latency_mean=5
write_latency_min=1
write_latency_mean=39
read_fault_probability=7%
write_fault_probability=0%
crash_probability=0.01%
crash_stability=67 ticks
restart_probability=1%
restart_stability=540 ticks

Stack Trace:

exit with code: 129


Tail of Debug Logs:
ica = 2 } > Process{ .replica = 1 }: Command.pong
[debug] (replica): 1: on_message: view=20 status=Status.view_change Header{ .checksum = 107244410931821212313958410271505547117, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 111480000000, .commit = 0, .timestamp = 111480000000, .size = 128, .replica = 2, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[warn] (clock): 1: learn: m0=111480000000 < window.monotonic=113470000000
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=13 clogged for ticks=41
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=1 clogged for ticks=2255
[debug] (packet_simulator): delivering packet from=8 to=2
[debug] (network): deliver_message: Process{ .client = 337882773686550778471552993841944164769 } > Process{ .replica = 2 }: Command.request
[debug] (replica): 2: on_message: view=20 status=Status.view_change Header{ .checksum = 283204971513986773042534633090453936650, .checksum_body = 141623023583469275815627528134669028928, .parent = 10308280239650074120905105847187127255, .client = 337882773686550778471552993841944164769, .context = 2, .request = 2, .cluster = 0, .epoch = 0, .view = 17, .op = 0, .commit = 0, .timestamp = 0, .size = 902066, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (replica): 2: on_request: ignoring (Status.view_change)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=10 clogged for ticks=134
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=15 clogged for ticks=10
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=21 clogged for ticks=1160
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=12 clogged for ticks=654
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=22 clogged for ticks=215
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=4 clogged for ticks=700
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=12 clogged for ticks=286
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=17 clogged for ticks=231
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=18 clogged for ticks=405
[debug] (vsr): 209283735857251286905679783441773726948: request_timeout fired
[debug] (vsr): 209283735857251286905679783441773726948: request_timeout backing off
[debug] (vsr): 209283735857251286905679783441773726948: request_timeout after=734..90 (rtt=30 min=10 max=1000 attempts=1)
[debug] (client): 209283735857251286905679783441773726948: on_request_timeout: resending request=1 checksum=312243986642379266535882236367791395305
[debug] (client): 209283735857251286905679783441773726948: sending request to replica 0: Header{ .checksum = 312243986642379266535882236367791395305, .checksum_body = 305969766633115278791438799026354498963, .parent = 308410828616968087229748016578281429217, .client = 209283735857251286905679783441773726948, .context = 10, .request = 1, .cluster = 0, .epoch = 0, .view = 17, .op = 0, .commit = 0, .timestamp = 0, .size = 1048576, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (network): send_message: Process{ .client = 209283735857251286905679783441773726948 } > Process{ .replica = 0 }: Command.request
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=2 clogged for ticks=314
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=3 clogged for ticks=11
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=18 clogged for ticks=528
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=11 clogged for ticks=692
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=15 clogged for ticks=23
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=18 clogged for ticks=391
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=20 clogged for ticks=414
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=20 clogged for ticks=11
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=22 clogged for ticks=52
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=1 clogged for ticks=133
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=4 clogged for ticks=577
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=9 clogged for ticks=226
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=10 clogged for ticks=464
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=21 clogged for ticks=1927
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=13 clogged for ticks=127
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=18 clogged for ticks=256
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=12 clogged for ticks=1139
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=14 clogged for ticks=103
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (replica): 2: on_message: view=20 status=Status.view_change Header{ .checksum = 202284101321035611215284895469681652497, .checksum_body = 299103359377350969821927441215918488563, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 20, .op = 25, .commit = 14, .timestamp = 18, .size = 2176, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_do_view_change: waiting for start_view_change quorum
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=11 clogged for ticks=40
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=14 clogged for ticks=14
[debug] (packet_simulator): replayed packet from=17 to=0
[debug] (network): deliver_message: Process{ .client = 209283735857251286905679783441773726948 } > Process{ .replica = 0 }: Command.request
[debug] (replica): 0: on_message: view=18 status=Status.normal Header{ .checksum = 312243986642379266535882236367791395305, .checksum_body = 305969766633115278791438799026354498963, .parent = 308410828616968087229748016578281429217, .client = 209283735857251286905679783441773726948, .context = 10, .request = 1, .cluster = 0, .epoch = 0, .view = 17, .op = 0, .commit = 0, .timestamp = 0, .size = 1048576, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_request: new request
[debug] (replica): 0: on_request: request 312243986642379266535882236367791395305
[debug] (replica): 0: on_request: prepare 167597866757206314479722347334910509730
[debug] (replica): 0: on_prepare: advancing: op=25..26 checksum=234541018469718942296355808415637235823..167597866757206314479722347334910509730
[debug] (journal): 0: set_header_as_dirty: op=26 checksum=167597866757206314479722347334910509730
[debug] (replica): 0: replicate: replicating to replica 1
[debug] (replica): 0: sending prepare to replica 1: Header{ .checksum = 167597866757206314479722347334910509730, .checksum_body = 305969766633115278791438799026354498963, .parent = 234541018469718942296355808415637235823, .client = 209283735857251286905679783441773726948, .context = 312243986642379266535882236367791395305, .request = 1, .cluster = 0, .epoch = 0, .view = 18, .op = 26, .commit = 14, .timestamp = 113610000000, .size = 1048576, .replica = 0, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.prepare
[debug] (replica): 0: append: appending to journal
[debug] (journal): 0: write: view=18 op=26 len=1048576: 167597866757206314479722347334910509730 starting
[debug] (journal): 0: write_sectors: offset=27279360 len=1048576 locked
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=19 clogged for ticks=373
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=21 clogged for ticks=772
[debug] (vsr): 2: view_change_message_timeout fired
[debug] (vsr): 2: view_change_message_timeout reset
[debug] (replica): 2: sending start_view_change to replica 0: Header{ .checksum = 58533117269483449274851672850594667363, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 20, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 1: Header{ .checksum = 58533117269483449274851672850594667363, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 20, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (packet_simulator): replayed packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=20 status=Status.view_change Header{ .checksum = 291127752152920970030249963442998711118, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 20, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: 1 message(s)
[debug] (replica): 2: on_start_view_change: view=20 quorum received
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=17 dest.len=32 copied=18
[debug] (replica): 2: sending do_view_change to replica 2: Header{ .checksum = 298691366167055538548430212034771731219, .checksum_body = 141552154120120400152394237438003193339, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 20, .op = 17, .commit = 9, .timestamp = 17, .size = 2432, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_message: view=20 status=Status.view_change Header{ .checksum = 298691366167055538548430212034771731219, .checksum_body = 141552154120120400152394237438003193339, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 20, .op = 17, .commit = 9, .timestamp = 17, .size = 2432, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_do_view_change: 1 message(s)
[debug] (replica): 2: on_do_view_change: waiting for quorum
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=3 clogged for ticks=539
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=13 clogged for ticks=1384
[debug] (packet_simulator): replayed packet from=17 to=0
[debug] (network): deliver_message: Process{ .client = 209283735857251286905679783441773726948 } > Process{ .replica = 0 }: Command.request
[debug] (replica): 0: on_message: view=18 status=Status.normal Header{ .checksum = 312243986642379266535882236367791395305, .checksum_body = 305969766633115278791438799026354498963, .parent = 308410828616968087229748016578281429217, .client = 209283735857251286905679783441773726948, .context = 10, .request = 1, .cluster = 0, .epoch = 0, .view = 17, .op = 0, .commit = 0, .timestamp = 0, .size = 1048576, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_request: new request
[debug] (replica): 0: on_request: ignoring (already preparing)
[debug] (vsr): 0: ping_timeout fired
[debug] (vsr): 0: ping_timeout reset
[debug] (replica): 0: sending ping to replica 1: Header{ .checksum = 155719238220543796211495761449761301443, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 113630000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.ping
[debug] (replica): 0: sending ping to replica 2: Header{ .checksum = 155719238220543796211495761449761301443, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 113630000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.ping
[debug] (vsr): 0: commit_timeout fired
[debug] (vsr): 0: commit_timeout reset
[debug] (replica): 0: sending commit to replica 1: Header{ .checksum = 169850792923418422215244263764010841136, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 38948890896869931947192825972651555993, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 14, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.commit
[debug] (replica): 0: sending commit to replica 2: Header{ .checksum = 169850792923418422215244263764010841136, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 38948890896869931947192825972651555993, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 14, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.commit
[debug] (vsr): 0: repair_timeout fired
[debug] (vsr): 0: repair_timeout reset
[debug] (replica): 0: repair_prepare: op=26 checksum=167597866757206314479722347334910509730 (already writing)
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=20 status=Status.view_change Header{ .checksum = 155719238220543796211495761449761301443, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 113630000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: sending pong to replica 0: Header{ .checksum = 151196784454632972269891926465425243761, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 20, .op = 113630000000, .commit = 0, .timestamp = 113630000000, .size = 128, .replica = 2, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.pong
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=3 clogged for ticks=21
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=12 clogged for ticks=1075
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=14 clogged for ticks=40
[debug] (packet_simulator): delivering packet from=17 to=0
[debug] (network): deliver_message: Process{ .client = 209283735857251286905679783441773726948 } > Process{ .replica = 0 }: Command.request
[debug] (replica): 0: on_message: view=18 status=Status.normal Header{ .checksum = 312243986642379266535882236367791395305, .checksum_body = 305969766633115278791438799026354498963, .parent = 308410828616968087229748016578281429217, .client = 209283735857251286905679783441773726948, .context = 10, .request = 1, .cluster = 0, .epoch = 0, .view = 17, .op = 0, .commit = 0, .timestamp = 0, .size = 1048576, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_request: new request
[debug] (replica): 0: on_request: ignoring (already preparing)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=20 clogged for ticks=45
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.ping
[debug] (replica): 1: on_message: view=20 status=Status.view_change Header{ .checksum = 155719238220543796211495761449761301443, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 113630000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: sending pong to replica 0: Header{ .checksum = 58908959453307971727823687339353469896, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 20, .op = 113630000000, .commit = 0, .timestamp = 113640000000, .size = 128, .replica = 1, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.pong
[debug] (packet_simulator): replayed packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.commit
[debug] (replica): 2: on_message: view=20 status=Status.view_change Header{ .checksum = 169850792923418422215244263764010841136, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 38948890896869931947192825972651555993, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 14, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_commit: ignoring (Status.view_change)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=2 clogged for ticks=87
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=5 clogged for ticks=714
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=16 clogged for ticks=238
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=16 clogged for ticks=283
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=18 clogged for ticks=282
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=22 clogged for ticks=110
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=20 status=Status.view_change Header{ .checksum = 210585977827242776312673320260768662487, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (older view)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=6 clogged for ticks=58
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=16 clogged for ticks=264
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=3 clogged for ticks=155
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=4 clogged for ticks=304
[debug] (packet_simulator): replayed packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.commit
[debug] (replica): 2: on_message: view=20 status=Status.view_change Header{ .checksum = 169850792923418422215244263764010841136, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 38948890896869931947192825972651555993, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 14, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_commit: ignoring (Status.view_change)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=15 clogged for ticks=714
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=16 clogged for ticks=723
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=8 clogged for ticks=79
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=14 clogged for ticks=7
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=3 clogged for ticks=282
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=8 clogged for ticks=1058
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=9 clogged for ticks=373
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=11 clogged for ticks=73
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=2 clogged for ticks=333
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.commit
[debug] (replica): 1: on_message: view=20 status=Status.view_change Header{ .checksum = 169850792923418422215244263764010841136, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 38948890896869931947192825972651555993, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 14, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_commit: ignoring (Status.view_change)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=0 clogged for ticks=1201
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=1 clogged for ticks=74
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=2 clogged for ticks=111
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=16 clogged for ticks=359
[debug] (packet_simulator): replayed packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.commit
[debug] (replica): 2: on_message: view=20 status=Status.view_change Header{ .checksum = 169850792923418422215244263764010841136, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 38948890896869931947192825972651555993, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 14, .timestamp = 0, .size = 128, .replica = 0, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_commit: ignoring (Status.view_change)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=3 clogged for ticks=246
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=5 clogged for ticks=364
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=11 clogged for ticks=1022
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=1 clogged for ticks=1014
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=13 clogged for ticks=86
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=15 clogged for ticks=228
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=19 clogged for ticks=384
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=0 clogged for ticks=3
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=3 clogged for ticks=66
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=11 clogged for ticks=142
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=0 clogged for ticks=75
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=8 clogged for ticks=703
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=10 clogged for ticks=172
[debug] (packet_simulator): replayed packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.prepare
[debug] (replica): 1: on_message: view=20 status=Status.view_change Header{ .checksum = 167597866757206314479722347334910509730, .checksum_body = 305969766633115278791438799026354498963, .parent = 234541018469718942296355808415637235823, .client = 209283735857251286905679783441773726948, .context = 312243986642379266535882236367791395305, .request = 1, .cluster = 0, .epoch = 0, .view = 18, .op = 26, .commit = 14, .timestamp = 113610000000, .size = 1048576, .replica = 0, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_prepare: ignoring (repair)
[debug] (replica): 1: on_repair: ignoring (view change, follower)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=7 clogged for ticks=289
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=8 clogged for ticks=155
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=12 clogged for ticks=122
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=19 clogged for ticks=557
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=9 clogged for ticks=269
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=16 clogged for ticks=238
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=19 clogged for ticks=42
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=21 clogged for ticks=1382
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=8 clogged for ticks=223
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=10 clogged for ticks=59
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=16 clogged for ticks=147
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=18 clogged for ticks=684
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=7 clogged for ticks=200
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=16 clogged for ticks=71
[debug] (journal): 0: write_sectors: offset=27279360 len=1048576 unlocked
[debug] (journal): 0: write_header: op=26 sectors[0..4096]
[debug] (journal): 0: write_sectors: offset=0 len=4096 locked
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=10 clogged for ticks=128
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=14 clogged for ticks=1227
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=17 clogged for ticks=1564
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=2 clogged for ticks=265
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=11 clogged for ticks=822
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=12 clogged for ticks=452
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=2 clogged for ticks=34
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=15 clogged for ticks=208
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=17 clogged for ticks=230
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=17 clogged for ticks=97
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=14 clogged for ticks=399
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=16 clogged for ticks=371
[err] (packet_simulator): dropped packet from=0 to=2.
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=20 status=Status.view_change Header{ .checksum = 291127752152920970030249963442998711118, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 20, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=4 clogged for ticks=156
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=15 clogged for ticks=374
[debug] (vsr): 212401276541284697895766386339951576846: request_timeout fired
[debug] (vsr): 212401276541284697895766386339951576846: request_timeout backing off
[debug] (vsr): 212401276541284697895766386339951576846: request_timeout after=110..517 (rtt=30 min=10 max=1000 attempts=6)
[debug] (client): 212401276541284697895766386339951576846: on_request_timeout: resending request=2 checksum=335975804618576310236530128956878056971
[debug] (client): 212401276541284697895766386339951576846: sending request to replica 0: Header{ .checksum = 335975804618576310236530128956878056971, .checksum_body = 158347831312111685315074398771423407686, .parent = 138279496085124878379187927512268428742, .client = 212401276541284697895766386339951576846, .context = 8, .request = 2, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 1048576, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (network): send_message: Process{ .client = 212401276541284697895766386339951576846 } > Process{ .replica = 0 }: Command.request
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=6 clogged for ticks=75
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=7 clogged for ticks=539
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=8 clogged for ticks=178
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=22 clogged for ticks=6
[err] (packet_simulator): dropped packet from=0 to=1.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=4 clogged for ticks=122
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=7 clogged for ticks=1928
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=8 clogged for ticks=581
[debug] (packet_simulator): replayed packet from=19 to=0
[debug] (network): deliver_message: Process{ .client = 212401276541284697895766386339951576846 } > Process{ .replica = 0 }: Command.request
[debug] (replica): 0: on_message: view=18 status=Status.normal Header{ .checksum = 335975804618576310236530128956878056971, .checksum_body = 158347831312111685315074398771423407686, .parent = 138279496085124878379187927512268428742, .client = 212401276541284697895766386339951576846, .context = 8, .request = 2, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 1048576, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_request: new request
[debug] (replica): 0: on_request: ignoring (already preparing)
[debug] (journal): 0: write_sectors: offset=0 len=4096 unlocked
[debug] (journal): 0: write: view=18 op=26 len=1048576: 167597866757206314479722347334910509730 complete, marking clean
[debug] (replica): 0: send_prepare_ok: op=26 checksum=167597866757206314479722347334910509730
[debug] (replica): 0: sending prepare_ok to replica 0: Header{ .checksum = 128711660974010125394480439472375274449, .checksum_body = 98081761841810822994326224503485043631, .parent = 234541018469718942296355808415637235823, .client = 209283735857251286905679783441773726948, .context = 167597866757206314479722347334910509730, .request = 1, .cluster = 0, .epoch = 0, .view = 18, .op = 26, .commit = 14, .timestamp = 113610000000, .size = 128, .replica = 0, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_message: view=18 status=Status.normal Header{ .checksum = 128711660974010125394480439472375274449, .checksum_body = 98081761841810822994326224503485043631, .parent = 234541018469718942296355808415637235823, .client = 209283735857251286905679783441773726948, .context = 167597866757206314479722347334910509730, .request = 1, .cluster = 0, .epoch = 0, .view = 18, .op = 26, .commit = 14, .timestamp = 113610000000, .size = 128, .replica = 0, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_prepare_ok: 1 message(s)
[debug] (replica): 0: on_prepare_ok: waiting for quorum
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=4 clogged for ticks=478
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=13 clogged for ticks=1196
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=7 clogged for ticks=37
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=7 clogged for ticks=89
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=9 clogged for ticks=7
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=1 clogged for ticks=220
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=13 clogged for ticks=391
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=19 clogged for ticks=354
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=7 clogged for ticks=84
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=16 clogged for ticks=1606
[debug] (packet_simulator): replayed packet from=19 to=0
[debug] (network): deliver_message: Process{ .client = 212401276541284697895766386339951576846 } > Process{ .replica = 0 }: Command.request
[debug] (replica): 0: on_message: view=18 status=Status.normal Header{ .checksum = 335975804618576310236530128956878056971, .checksum_body = 158347831312111685315074398771423407686, .parent = 138279496085124878379187927512268428742, .client = 212401276541284697895766386339951576846, .context = 8, .request = 2, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 1048576, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_request: new request
[debug] (replica): 0: on_request: ignoring (already preparing)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=21 clogged for ticks=534
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=9 clogged for ticks=1260
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=12 clogged for ticks=471
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=5 clogged for ticks=38
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=11 clogged for ticks=372
[err] (packet_simulator): dropped packet from=19 to=0.
[debug] (vsr): 1: view_change_message_timeout fired
[debug] (vsr): 1: view_change_message_timeout reset
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 291127752152920970030249963442998711118, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 20, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 291127752152920970030249963442998711118, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 20, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (journal): 1: copy_latest_headers_between: op_min=0 op_max=25 dest.len=32 copied=16
[debug] (replica): 1: sending do_view_change to replica 2: Header{ .checksum = 202284101321035611215284895469681652497, .checksum_body = 299103359377350969821927441215918488563, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 20, .op = 25, .commit = 14, .timestamp = 18, .size = 2176, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=0 clogged for ticks=454
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (replica): 2: on_message: view=20 status=Status.view_change Header{ .checksum = 202284101321035611215284895469681652497, .checksum_body = 299103359377350969821927441215918488563, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 20, .op = 25, .commit = 14, .timestamp = 18, .size = 2176, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_do_view_change: 2 message(s)
[debug] (replica): 2: on_do_view_change: view=20 quorum received
[debug] (replica): 2: on_do_view_change: replica=1 v'=18 op=25 commit=14 latest=Header{ .checksum = 234541018469718942296355808415637235823, .checksum_body = 98081761841810822994326224503485043631, .parent = 103606552601480774144810063704895237530, .client = 252234934906304373536025557650101738668, .context = 214627569931152731055840611069873400814, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 25, .commit = 14, .timestamp = 112490000000, .size = 128, .replica = 0, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 2: on_do_view_change: replica=2 v'=17 op=17 commit=9 latest=Header{ .checksum = 289087230569821905875060524640216893302, .checksum_body = 98081761841810822994326224503485043631, .parent = 84622029394225746913070492487395759659, .client = 272951884569803585199917041759433741751, .context = 212417210501670122555292913301569348336, .request = 0, .cluster = 0, .epoch = 0, .view = 17, .op = 17, .commit = 9, .timestamp = 100570000000, .size = 128, .replica = 2, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 2: on_do_view_change: view=20 op=17..25 commit=9..14 checksum=234541018469718942296355808415637235823
[debug] (journal): 2: set_header_as_dirty: op=25 checksum=234541018469718942296355808415637235823
[debug] (journal): 2: remove_entries_from: op_min=26
[debug] (replica): 2: repair_header: op=25 exists, checksum dirty
[debug] (journal): 2: set_header_as_dirty: op=25 checksum=234541018469718942296355808415637235823
[debug] (replica): 2: repair_header: op=14 false (exists)
[debug] (replica): 2: repair_header: op=13 false (exists)
[debug] (replica): 2: repair_header: op=12 false (exists)
[debug] (replica): 2: repair_header: op=11 false (exists)
[debug] (replica): 2: repair_header: op=10 false (checksum clean)
[debug] (replica): 2: repair_header: op=9 false (checksum clean)
[debug] (replica): 2: repair_header: op=8 false (checksum clean)
[debug] (replica): 2: repair_header: op=7 false (checksum clean)
[debug] (replica): 2: repair_header: op=6 false (checksum clean)
[debug] (replica): 2: repair_header: op=5 false (checksum clean)
[debug] (replica): 2: repair_header: op=4 false (checksum clean)
[debug] (replica): 2: repair_header: op=3 false (checksum clean)
[debug] (replica): 2: repair_header: op=2 false (checksum clean)
[debug] (replica): 2: repair_header: op=1 false (checksum clean)
[debug] (replica): 2: repair_header: op=0 false (checksum clean)
[debug] (replica): 2: repair_header: op=17 false (checksum clean)
[debug] (replica): 2: repair_header: op=16 false (checksum clean)
[debug] (replica): 2: repair_header: op=15 false (checksum clean)
[debug] (replica): 2: repair_header: op=14 false (checksum clean)
[debug] (replica): 2: repair_header: op=13 false (checksum clean)
[debug] (replica): 2: repair_header: op=12 false (checksum clean)
[debug] (replica): 2: repair_header: op=11 false (checksum clean)
[debug] (replica): 2: repair_header: op=10 false (checksum clean)
[debug] (replica): 2: repair_header: op=9 false (checksum clean)
[debug] (replica): 2: repair_header: op=8 false (checksum clean)
[debug] (replica): 2: repair_header: op=7 false (checksum clean)
[debug] (replica): 2: repair_header: op=6 false (checksum clean)
[debug] (replica): 2: repair_header: op=5 false (checksum clean)
[debug] (replica): 2: repair_header: op=4 false (checksum clean)
[debug] (replica): 2: repair_header: op=3 false (checksum clean)
[debug] (replica): 2: repair_header: op=2 false (checksum clean)
[debug] (replica): 2: repair_header: op=1 false (checksum clean)
[debug] (replica): 2: repair_header: op=0 false (checksum clean)
[debug] (replica): 2: discard_uncommitted_headers: op=24 gap
[debug] (replica): 2: discard_uncommitted_headers: replica=1 op=24 nack=true
[debug] (replica): 2: discard_uncommitted_headers: op=24 nacks=1 threshold=1
[debug] (journal): 2: remove_entries_from: op_min=24
[debug] (replica): 2: discard_uncommitted_headers: op=23 gap
[debug] (replica): 2: discard_uncommitted_headers: replica=1 op=23 nack=true
[debug] (replica): 2: discard_uncommitted_headers: op=23 nacks=1 threshold=1
[debug] (journal): 2: remove_entries_from: op_min=23
[debug] (replica): 2: discard_uncommitted_headers: op=22 gap
[debug] (replica): 2: discard_uncommitted_headers: replica=1 op=22 nack=true
[debug] (replica): 2: discard_uncommitted_headers: op=22 nacks=1 threshold=1
[debug] (journal): 2: remove_entries_from: op_min=22
[debug] (replica): 2: discard_uncommitted_headers: op=21 gap
[debug] (replica): 2: discard_uncommitted_headers: replica=1 op=21 nack=true
[debug] (replica): 2: discard_uncommitted_headers: op=21 nacks=1 threshold=1
[debug] (journal): 2: remove_entries_from: op_min=21
[debug] (replica): 2: discard_uncommitted_headers: op=20 gap
[debug] (replica): 2: discard_uncommitted_headers: replica=1 op=20 nack=true
[debug] (replica): 2: discard_uncommitted_headers: op=20 nacks=1 threshold=1
[debug] (journal): 2: remove_entries_from: op_min=20
[debug] (replica): 2: discard_uncommitted_headers: op=19 gap
[debug] (replica): 2: discard_uncommitted_headers: replica=1 op=19 nack=true
[debug] (replica): 2: discard_uncommitted_headers: op=19 nacks=1 threshold=1
[debug] (journal): 2: remove_entries_from: op_min=19
[debug] (replica): 2: discard_uncommitted_headers: op=18 gap
[debug] (replica): 2: discard_uncommitted_headers: replica=1 op=18 nack=true
[debug] (replica): 2: discard_uncommitted_headers: op=18 nacks=1 threshold=1
[debug] (journal): 2: remove_entries_from: op_min=18
[debug] (vsr): 2: repair_timeout started
[debug] (vsr): 2: repair_timeout reset
[debug] (replica): 2: commit_op: executing view=20 true op=10 checksum=256904362636808906929971033742983250143 (register)
[debug] (replica): 2: create_client_table_entry: client=209283735857251286905679783441773726948 session=10 request=0
[debug] (replica): 2: commit_op: replying to client: Header{ .checksum = 308410828616968087229748016578281429217, .checksum_body = 98081761841810822994326224503485043631, .parent = 286891332077001832404198855330460153528, .client = 209283735857251286905679783441773726948, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 17, .op = 10, .commit = 10, .timestamp = 0, .size = 128, .replica = 2, .command = Command.reply, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .client = 209283735857251286905679783441773726948 }: Command.reply
[debug] (journal): 2: read_sectors: offset=11550720 len=1048576
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=4 clogged for ticks=1331
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=11 clogged for ticks=151
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=18 clogged for ticks=1
[debug] (replica): 2: commit_op: executing view=20 true op=11 checksum=88072021177562956271002900914963400066 (hash)
[debug] (state_machine): state=b2b805e00d487ca52d9c55b60236d67f input=2e0e7a8fdbbd0adfd8069a99fe773028 input.len=976178 new state=1cea4b8d22dac28d3dd14918c0b21f97
[info] (state_checker): 0004/0005 b2b805e00d487ca52d9c55b60236d67f > 1cea4b8d22dac28d3dd14918c0b21f97 2
[debug] (replica): 2: update_client_table_entry: client=212401276541284697895766386339951576846 session=8 request=1
[debug] (replica): 2: commit_op: replying to client: Header{ .checksum = 36952118961666063813739311034045959413, .checksum_body = 319409141306211583479868627761916011735, .parent = 67993457298148682704411641454420814897, .client = 212401276541284697895766386339951576846, .context = 0, .request = 1, .cluster = 0, .epoch = 0, .view = 17, .op = 11, .commit = 11, .timestamp = 0, .size = 144, .replica = 2, .command = Command.reply, .operation = Operation(3), .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .client = 212401276541284697895766386339951576846 }: Command.reply
[debug] (journal): 2: read_sectors: offset=12599296 len=1048576
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=8 clogged for ticks=338
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=20 clogged for ticks=41
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=0 clogged for ticks=639
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=11 clogged for ticks=1889
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=12 clogged for ticks=1021
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=18 clogged for ticks=1068
[debug] (vsr): 44291002323262337191216532845732199627: request_timeout fired
[debug] (vsr): 44291002323262337191216532845732199627: request_timeout backing off
[debug] (vsr): 44291002323262337191216532845732199627: request_timeout after=674..1042 (rtt=30 min=10 max=1000 attempts=27)
[debug] (client): 44291002323262337191216532845732199627: on_request_timeout: resending request=0 checksum=217892753380012814771842539267169007078
[debug] (client): 44291002323262337191216532845732199627: sending request to replica 1: Header{ .checksum = 217892753380012814771842539267169007078, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 44291002323262337191216532845732199627, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .client = 44291002323262337191216532845732199627 } > Process{ .replica = 1 }: Command.request
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=0 clogged for ticks=88
[err] (packet_simulator): dropped packet from=2 to=19.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=8 clogged for ticks=39
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=15 clogged for ticks=2
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=21 clogged for ticks=20
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=12 clogged for ticks=375
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=15 clogged for ticks=212
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=2 clogged for ticks=123
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=14 clogged for ticks=280
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=18 clogged for ticks=261
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=0 clogged for ticks=0
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=0 clogged for ticks=679
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=0 clogged for ticks=88
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=0 clogged for ticks=65
[debug] (packet_simulator): replayed packet from=10 to=0
[debug] (network): deliver_message: Process{ .client = 302169491783070118976203969933931846101 } > Process{ .replica = 0 }: Command.request
[debug] (replica): 0: on_message: view=18 status=Status.normal Header{ .checksum = 87958317699727610431879967363289346322, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 302169491783070118976203969933931846101, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (replica): 0: on_request: new session
[debug] (replica): 0: on_request: request 87958317699727610431879967363289346322
[debug] (replica): 0: on_request: prepare 48097682185907856889461627111733601320
[debug] (replica): 0: on_prepare: advancing: op=26..27 checksum=167597866757206314479722347334910509730..48097682185907856889461627111733601320
[debug] (journal): 0: set_header_as_dirty: op=27 checksum=48097682185907856889461627111733601320
[debug] (replica): 0: replicate: replicating to replica 1
[debug] (replica): 0: sending prepare to replica 1: Header{ .checksum = 48097682185907856889461627111733601320, .checksum_body = 98081761841810822994326224503485043631, .parent = 167597866757206314479722347334910509730, .client = 302169491783070118976203969933931846101, .context = 87958317699727610431879967363289346322, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 27, .commit = 14, .timestamp = 114020000000, .size = 128, .replica = 0, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.prepare
[debug] (replica): 0: append: appending to journal
[debug] (journal): 0: write: view=18 op=27 len=128: 48097682185907856889461627111733601320 starting
[debug] (journal): 0: write_sectors: offset=28327936 len=4096 locked
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=20 clogged for ticks=50
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=15 clogged for ticks=64
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=16 clogged for ticks=953
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=18 clogged for ticks=368
[debug] (journal): 0: write_sectors: offset=28327936 len=4096 unlocked
[debug] (journal): 0: write_header: op=27 sectors[0..4096]
[debug] (journal): 0: write_sectors: offset=0 len=4096 locked
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=3 clogged for ticks=41
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=6 clogged for ticks=589
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=6 clogged for ticks=1
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=11 clogged for ticks=122
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=15 clogged for ticks=204
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=7 clogged for ticks=105
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=8 clogged for ticks=611
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=11 clogged for ticks=191
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=12 clogged for ticks=549
[debug] (replica): 2: commit_op: executing view=20 true op=12 checksum=47803963679225224580599779628170066054 (hash)
[debug] (state_machine): state=1cea4b8d22dac28d3dd14918c0b21f97 input=52331fd869949e6ce0a6b5f20047c11 input.len=339249 new state=1624b7301223ef08019b4b2b489442de
[err] (state_checker): state checker error: error.ReplicaTransitionedToInvalidState

Time:
2022-07-14 03:06:56.697969482 +0000 UTC

When running tigerbeetle within a Docker container on mac I get the following error

info(storage): opening "cluster_0000000001_replica_000.tigerbeetle"...
error: SystemOutdated
/opt/beta-beetle/zig/lib/std/os/linux/io_uring.zig:0:0: 0x223110 in start (tigerbeetle)
/opt/beta-beetle/zig/lib/std/os/linux/io_uring.zig:36:16: 0x223118 in start (tigerbeetle)
/opt/beta-beetle/src/main.zig:0:0: 0x223120 in start (tigerbeetle)
/opt/beta-beetle/src/main.zig:0:0: 0x223128 in start (tigerbeetle)
/opt/beta-beetle/src/main.zig:28:26: 0x20fa1f in std.start.posixCallMainAndExit (tigerbeetle)

If I run a uname -a within a container I get the following

Linux ca2c201c1d3a 5.10.47-linuxkit #1 SMP Sat Jul 3 21:51:47 UTC 2021 x86_64 GNU/Linux

I suspect it may be there is some features in this kernel that TB requires from io_uring thats not supported. Or this actual kernel version has io_uring stripped out. I'm not 100% sure.

Bug Type:
crash

Seed:
18100658869621668726

Commit Hash:
01494c5

Parameters:
[info] (state_checker):
SEED=18100658869621668726

replicas=3
clients=29
request_probability=38%
idle_on_probability=17%
idle_off_probability=15%
one_way_delay_mean=8 ticks
one_way_delay_min=1 ticks
packet_loss_probability=16%
path_maximum_capacity=9 messages
path_clog_duration_mean=89 ticks
path_clog_probability=1%
packet_replay_probability=2%
partition_mode=PartitionMode.isolate_single
partition_probability=2%
unpartition_probability=2%
partition_stability=153 ticks
unpartition_stability=2 ticks
read_latency_min=1
read_latency_mean=6
write_latency_min=0
write_latency_mean=39
read_fault_probability=7%
write_fault_probability=4%
crash_probability=0.01%
crash_stability=423 ticks
restart_probability=1%
restart_stability=887 ticks

Stack Trace:

thread panic: reached unreachable code
zig/lib/std/debug.zig:225:14: in std.debug.assert (simulator)
if (!ok) unreachable; // assertion failure
^
src/vsr/replica.zig:4542:19: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).set_latest_op_and_k (simulator)
assert(latest.op >= self.commit_max);
^
src/vsr/replica.zig:1126:37: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).on_start_view (simulator)
self.set_latest_op_and_k(&latest, message.header.commit, "on_start_view");
^
src/vsr/replica.zig:530:50: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).on_message (simulator)
.start_view => self.on_start_view(message),
^
src/test/message_bus.zig:57:27: in test.message_bus.struct:55:35.wrapper (simulator)
on_message(@intToPtr(Context, @ptrToInt(_context)), message);
^
src/test/network.zig:167:41: in test.network.Network.deliver_message (simulator)
target_bus.on_message_callback.?(target_bus.on_message_context, message);
^
src/test/packet_simulator.zig:345:42: in test.packet_simulator.PacketSimulator(test.network.Packet).tick (simulator)
data.callback(data.packet, path);
^
src/simulator.zig:277:46: in main (simulator)
cluster.network.packet_simulator.tick(cluster.health);
^
zig/lib/std/start.zig:561:37: in std.start.callMain (simulator)
const result = root.main() catch |err| {
^
zig/lib/std/start.zig:495:12: in std.start.callMainWithArgs (simulator)
return @call(.{ .modifier = .always_inline }, callMain, .{});
^
zig/lib/std/start.zig:409:17: in std.start.posixCallMainAndExit (simulator)
std.os.exit(@call(.{ .modifier = .always_inline }, callMainWithArgs, .{ argc, argv, envp }));
^
zig/lib/std/start.zig:322:5: in std.start._start (simulator)
@call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{});
^
exit with signal: 6. Indicates a crash bug.


Tail of Debug Logs:
1841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=52 dest.len=32 copied=32
[debug] (replica): 2: sending do_view_change to replica 0: Header{ .checksum = 330939700465125267732301467498605751421, .checksum_body = 132738057279334392004174554584220144824, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 52, .commit = 33, .timestamp = 5, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (packet_simulator): delivering packet from=2 to=1
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=6 status=Status.view_change Header{ .checksum = 212879274351980240570388629026058950507, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: ignoring (duplicate message replica=2)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=13 clogged for ticks=81
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=22 clogged for ticks=33
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=26 path.target=0 clogged for ticks=33
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=26 path.target=3 clogged for ticks=0
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=10 clogged for ticks=230
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=10 clogged for ticks=24
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=13 clogged for ticks=174
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=14 clogged for ticks=110
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=19 clogged for ticks=335
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=31 path.target=19 clogged for ticks=68
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=24 clogged for ticks=50
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=28 clogged for ticks=20
[debug] (clock): 2: synchronized: truechimers=2/3 clock_offset=0ns..0ns accuracy=0ns
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=11 clogged for ticks=9
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=15 clogged for ticks=104
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=24 clogged for ticks=15
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=27 path.target=24 clogged for ticks=32
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=26 clogged for ticks=147
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=29 clogged for ticks=121
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=31 clogged for ticks=59
[debug] (vsr): 1: view_change_message_timeout fired
[debug] (vsr): 1: view_change_message_timeout reset
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 186948264389726097974816603876012990665, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 186948264389726097974816603876012990665, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (journal): 1: copy_latest_headers_between: op_min=0 op_max=46 dest.len=32 copied=32
[debug] (replica): 1: sending do_view_change to replica 0: Header{ .checksum = 53269459277375954586188978538843589174, .checksum_body = 188118643336781645133023700446750716187, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 46, .commit = 31, .timestamp = 5, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.do_view_change
[err] (packet_simulator): dropped packet from=2 to=0.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=5 clogged for ticks=84
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=27 path.target=5 clogged for ticks=14
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=19 clogged for ticks=1
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=23 clogged for ticks=56
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=31 path.target=0 clogged for ticks=55
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=5 clogged for ticks=11
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=10 clogged for ticks=24
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=10 clogged for ticks=9
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=3 clogged for ticks=175
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=5 clogged for ticks=1
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=5 clogged for ticks=196
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=11 clogged for ticks=83
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=14 clogged for ticks=23
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=26 path.target=27 clogged for ticks=29
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=30 path.target=29 clogged for ticks=54
[debug] (vsr): 0: view_change_message_timeout fired
[debug] (vsr): 0: view_change_message_timeout reset
[debug] (replica): 0: sending start_view_change to replica 1: Header{ .checksum = 337445460462094094034531792561799726439, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 2: Header{ .checksum = 337445460462094094034531792561799726439, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=6 status=Status.view_change Header{ .checksum = 337445460462094094034531792561799726439, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: 2 message(s)
[debug] (replica): 1: on_start_view_change: ignoring (quorum received already)
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=6 status=Status.view_change Header{ .checksum = 337445460462094094034531792561799726439, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: 2 message(s)
[debug] (replica): 2: on_start_view_change: ignoring (quorum received already)
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (replica): 0: on_message: view=6 status=Status.view_change Header{ .checksum = 53269459277375954586188978538843589174, .checksum_body = 188118643336781645133023700446750716187, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 46, .commit = 31, .timestamp = 5, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: 2 message(s)
[debug] (replica): 0: on_do_view_change: view=6 quorum received
[debug] (replica): 0: on_do_view_change: replica=0 v'=5 op=49 commit=31 latest=Header{ .checksum = 128334290962585362590947241944926282653, .checksum_body = 95698056645061224751788033946603871486, .parent = 71380198035544811229094820255703666353, .client = 1009796729557555445663605208948009159, .context = 264055746472038930822726837248326181567, .request = 1, .cluster = 0, .epoch = 0, .view = 5, .op = 49, .commit = 31, .timestamp = 66420000000, .size = 972280, .replica = 2, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_do_view_change: replica=1 v'=5 op=46 commit=31 latest=Header{ .checksum = 220259789038112435354735367446412842420, .checksum_body = 64923765208179887814319030326234393531, .parent = 301688015253494243092599734550039755922, .client = 7204602051265642679317511360251059264, .context = 62296186407368046334905458534271828556, .request = 1, .cluster = 0, .epoch = 0, .view = 5, .op = 46, .commit = 31, .timestamp = 63430000000, .size = 209143, .replica = 2, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_do_view_change: view=6 op=49..49 commit=31..31 checksum=128334290962585362590947241944926282653
[debug] (replica): 0: on_do_view_change: latest op exists exactly
[debug] (journal): 0: remove_entries_from: op_min=50
[debug] (replica): 0: repair_header: op=49 false (checksum clean)
[debug] (replica): 0: repair_header: op=48 false (checksum clean)
[debug] (replica): 0: repair_header: op=47 false (checksum clean)
[debug] (replica): 0: repair_header: op=46 false (checksum clean)
[debug] (replica): 0: repair_header: op=45 false (checksum clean)
[debug] (replica): 0: repair_header: op=44 false (checksum clean)
[debug] (replica): 0: repair_header: op=43 false (checksum clean)
[debug] (replica): 0: repair_header: op=42 false (checksum clean)
[debug] (replica): 0: repair_header: op=41 false (checksum clean)
[debug] (replica): 0: repair_header: op=40 false (checksum clean)
[debug] (replica): 0: repair_header: op=39 false (checksum clean)
[debug] (replica): 0: repair_header: op=38 false (checksum clean)
[debug] (replica): 0: repair_header: op=37 false (checksum clean)
[debug] (replica): 0: repair_header: op=36 false (checksum clean)
[debug] (replica): 0: repair_header: op=35 false (checksum clean)
[debug] (replica): 0: repair_header: op=34 false (checksum clean)
[debug] (replica): 0: repair_header: op=33 exists, checksum dirty
[debug] (journal): 0: set_header_as_dirty: op=33 checksum=122007769816831763096686706175976176185
[debug] (replica): 0: repair_header: op=32 false (checksum clean)
[debug] (replica): 0: repair_header: op=31 false (checksum clean)
[debug] (replica): 0: repair_header: op=30 false (checksum clean)
[debug] (replica): 0: repair_header: op=29 false (checksum clean)
[debug] (replica): 0: repair_header: op=28 false (checksum clean)
[debug] (replica): 0: repair_header: op=27 false (checksum clean)
[debug] (replica): 0: repair_header: op=26 false (checksum clean)
[debug] (replica): 0: repair_header: op=25 false (checksum clean)
[debug] (replica): 0: repair_header: op=24 false (checksum clean)
[debug] (replica): 0: repair_header: op=23 false (checksum clean)
[debug] (replica): 0: repair_header: op=22 false (checksum clean)
[debug] (replica): 0: repair_header: op=21 false (checksum clean)
[debug] (replica): 0: repair_header: op=20 false (checksum clean)
[debug] (replica): 0: repair_header: op=19 false (checksum clean)
[debug] (replica): 0: repair_header: op=18 false (checksum clean)
[debug] (replica): 0: repair_header: op=46 false (checksum clean)
[debug] (replica): 0: repair_header: op=45 false (checksum clean)
[debug] (replica): 0: repair_header: op=44 false (checksum clean)
[debug] (replica): 0: repair_header: op=42 false (checksum clean)
[debug] (replica): 0: repair_header: op=41 false (checksum clean)
[debug] (replica): 0: repair_header: op=40 false (checksum clean)
[debug] (replica): 0: repair_header: op=39 false (checksum clean)
[debug] (replica): 0: repair_header: op=38 false (checksum clean)
[debug] (replica): 0: repair_header: op=37 false (checksum clean)
[debug] (replica): 0: repair_header: op=36 false (checksum clean)
[debug] (replica): 0: repair_header: op=35 false (checksum clean)
[debug] (replica): 0: repair_header: op=34 false (checksum clean)
[debug] (replica): 0: repair_header: op=33 exists, checksum dirty
[debug] (journal): 0: set_header_as_dirty: op=33 checksum=122007769816831763096686706175976176185
[debug] (replica): 0: repair_header: op=32 false (checksum clean)
[debug] (replica): 0: repair_header: op=31 false (checksum clean)
[debug] (replica): 0: repair_header: op=30 false (checksum clean)
[debug] (replica): 0: repair_header: op=29 false (checksum clean)
[debug] (replica): 0: repair_header: op=28 false (checksum clean)
[debug] (replica): 0: repair_header: op=27 false (checksum clean)
[debug] (replica): 0: repair_header: op=26 false (checksum clean)
[debug] (replica): 0: repair_header: op=25 false (checksum clean)
[debug] (replica): 0: repair_header: op=24 false (checksum clean)
[debug] (replica): 0: repair_header: op=23 false (checksum clean)
[debug] (replica): 0: repair_header: op=22 false (checksum clean)
[debug] (replica): 0: repair_header: op=21 false (checksum clean)
[debug] (replica): 0: repair_header: op=20 false (checksum clean)
[debug] (replica): 0: repair_header: op=19 false (checksum clean)
[debug] (replica): 0: repair_header: op=18 false (checksum clean)
[debug] (replica): 0: repair_header: op=17 false (checksum clean)
[debug] (replica): 0: repair_header: op=16 false (checksum clean)
[debug] (replica): 0: repair_header: op=15 false (checksum clean)
[debug] (replica): 0: repair_header: op=14 false (checksum clean)
[debug] (vsr): 0: repair_timeout started
[debug] (vsr): 0: repair_timeout reset
[debug] (replica): 0: repair_prepare: op=33 checksum=122007769816831763096686706175976176185 (uncommitted, dirty, view_change)
[debug] (replica): 0: sending request_prepare to replica 1: Header{ .checksum = 6946806429024352976332939649012911095, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 122007769816831763096686706175976176185, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 33, .commit = 0, .timestamp = 1, .size = 128, .replica = 0, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.request_prepare
[debug] (replica): 0: sending request_prepare to replica 2: Header{ .checksum = 6946806429024352976332939649012911095, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 122007769816831763096686706175976176185, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 33, .commit = 0, .timestamp = 1, .size = 128, .replica = 0, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.request_prepare
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=5 clogged for ticks=10
[debug] (packet_simulator): delivering packet from=6 to=2
[debug] (network): deliver_message: Process{ .client = 281387399549688022644293776299680125657 } > Process{ .replica = 2 }: Command.request
[debug] (replica): 2: on_message: view=6 status=Status.view_change Header{ .checksum = 12575597129520764939155373008799140151, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 281387399549688022644293776299680125657, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 1, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (replica): 2: on_request: ignoring (Status.view_change)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=7 clogged for ticks=12
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=7 clogged for ticks=38
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=10 clogged for ticks=23
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=11 clogged for ticks=38
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=13 clogged for ticks=10
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=31 path.target=18 clogged for ticks=209
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=20 clogged for ticks=31
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=24 clogged for ticks=96
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=28 clogged for ticks=43
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=0 clogged for ticks=201
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=3 clogged for ticks=70
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=10 clogged for ticks=50
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=27 clogged for ticks=140
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=30 clogged for ticks=109
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=3 clogged for ticks=8
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=5 clogged for ticks=34
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=7 clogged for ticks=161
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=17 clogged for ticks=8
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=22 clogged for ticks=8
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=24 clogged for ticks=9
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=24 clogged for ticks=222
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=25 clogged for ticks=51
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=27 clogged for ticks=105
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=29 clogged for ticks=150
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=1 clogged for ticks=113
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=2 clogged for ticks=9
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=13 clogged for ticks=83
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=20 clogged for ticks=645
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=22 clogged for ticks=25
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=26 clogged for ticks=8
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=26 clogged for ticks=43
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=26 clogged for ticks=53
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=26 path.target=27 clogged for ticks=60
[debug] (vsr): 266296002880452076906940415151592338417: request_timeout fired
[debug] (vsr): 266296002880452076906940415151592338417: request_timeout backing off
[debug] (vsr): 266296002880452076906940415151592338417: request_timeout after=716..800 (rtt=30 min=10 max=1000 attempts=20)
[debug] (client): 266296002880452076906940415151592338417: on_request_timeout: resending request=0 checksum=118667114096750284439290496229801692319
[debug] (client): 266296002880452076906940415151592338417: sending request to replica 1: Header{ .checksum = 118667114096750284439290496229801692319, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 266296002880452076906940415151592338417, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .client = 266296002880452076906940415151592338417 } > Process{ .replica = 1 }: Command.request
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=2 clogged for ticks=38
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=4 clogged for ticks=10
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=25 clogged for ticks=1
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=29 clogged for ticks=155
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=3 clogged for ticks=23
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=12 clogged for ticks=46
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=12 clogged for ticks=109
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=21 clogged for ticks=390
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=30 path.target=27 clogged for ticks=84
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=1 clogged for ticks=54
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (replica): 0: on_message: view=6 status=Status.view_change Header{ .checksum = 330939700465125267732301467498605751421, .checksum_body = 132738057279334392004174554584220144824, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 52, .commit = 33, .timestamp = 5, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: 3 message(s)
[debug] (replica): 0: on_do_view_change: ignoring (quorum received already)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=3 clogged for ticks=108
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=27 path.target=10 clogged for ticks=98
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=13 clogged for ticks=25
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=26 clogged for ticks=26
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=4 clogged for ticks=50
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=9 clogged for ticks=51
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=16 clogged for ticks=112
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=18 clogged for ticks=17
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=19 clogged for ticks=65
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=26 clogged for ticks=42
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=26 clogged for ticks=207
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=29 clogged for ticks=220
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=29 clogged for ticks=23
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=26 path.target=29 clogged for ticks=19
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=30 clogged for ticks=15
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=4 clogged for ticks=217
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=5 clogged for ticks=41
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=8 clogged for ticks=38
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=19 clogged for ticks=81
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=20 clogged for ticks=3
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=26 clogged for ticks=43
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=30 clogged for ticks=26
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=30 clogged for ticks=140
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=4 clogged for ticks=87
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=8 clogged for ticks=47
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=11 clogged for ticks=58
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=31 path.target=0 clogged for ticks=4
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=2 clogged for ticks=185
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=2 clogged for ticks=61
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=11 clogged for ticks=83
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=14 clogged for ticks=199
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=20 clogged for ticks=52
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=30 clogged for ticks=52
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.request_prepare
[debug] (replica): 1: on_message: view=6 status=Status.view_change Header{ .checksum = 6946806429024352976332939649012911095, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 122007769816831763096686706175976176185, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 33, .commit = 0, .timestamp = 1, .size = 128, .replica = 0, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_request_prepare: op=33 checksum=122007769816831763096686706175976176185 nacking
[debug] (replica): 1: sending nack_prepare to replica 0: Header{ .checksum = 269106774709670219009616421549381944244, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 122007769816831763096686706175976176185, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 33, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.nack_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.nack_prepare
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=30 path.target=10 clogged for ticks=68
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=12 clogged for ticks=136
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=18 clogged for ticks=7
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=18 clogged for ticks=65
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=19 clogged for ticks=71
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=21 clogged for ticks=42
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=24 clogged for ticks=31
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=24 clogged for ticks=13
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=31 path.target=31 clogged for ticks=210
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.nack_prepare
[debug] (replica): 0: on_message: view=6 status=Status.view_change Header{ .checksum = 269106774709670219009616421549381944244, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 122007769816831763096686706175976176185, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 33, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.nack_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_nack_prepare: quorum_replication=2 threshold=1 op=33
[debug] (replica): 0: on_nack_prepare: 1 message(s)
[debug] (replica): 0: on_nack_prepare: quorum received op=33
[debug] (replica): 0: discard_uncommitted_ops_from: ops=33..49 view=6
[debug] (journal): 0: remove_entries_from: op_min=33
[debug] (vsr): 0: repair_timeout reset
[debug] (replica): 0: repair_pipeline_diff: 0 prepare(s)
[debug] (replica): 0: repair_pipeline: repairing
[debug] (replica): 0: repair_pipeline_diff: 0 prepare(s)
[debug] (replica): 0: repair_pipeline_read: op=32 checksum=336351996484442950290764607947182983196
[debug] (journal): 0: read_sectors: offset=33570816 len=1048576
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=26 path.target=1 clogged for ticks=115
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=3 clogged for ticks=40
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=7 clogged for ticks=39
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=20 clogged for ticks=34
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=26 clogged for ticks=22
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=28 clogged for ticks=20
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=30 path.target=10 clogged for ticks=8
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=20 clogged for ticks=53
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=21 clogged for ticks=16
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=22 clogged for ticks=94
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=23 clogged for ticks=41
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=30 clogged for ticks=35
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=0 clogged for ticks=38
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=2 clogged for ticks=156
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=5 clogged for ticks=3
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=5 clogged for ticks=122
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=7 clogged for ticks=24
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=8 clogged for ticks=130
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=9 clogged for ticks=123
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=14 clogged for ticks=146
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=20 clogged for ticks=84
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=20 clogged for ticks=3
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=25 clogged for ticks=117
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=29 clogged for ticks=55
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=0 clogged for ticks=108
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=6 clogged for ticks=87
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=18 clogged for ticks=7
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=20 clogged for ticks=44
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=21 clogged for ticks=73
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=27 path.target=25 clogged for ticks=55
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=3 clogged for ticks=111
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=5 clogged for ticks=123
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=13 clogged for ticks=50
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=14 clogged for ticks=45
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=15 clogged for ticks=32
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=18 clogged for ticks=0
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=22 clogged for ticks=37
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=23 clogged for ticks=13
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=29 clogged for ticks=82
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=7 clogged for ticks=120
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=9 clogged for ticks=14
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=20 clogged for ticks=186
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=20 clogged for ticks=39
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=21 clogged for ticks=71
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=25 clogged for ticks=202
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=31 clogged for ticks=43
[debug] (replica): 0: repair_pipeline_diff: 0 prepare(s)
[debug] (replica): 0: repair_pipeline_push: op=32 checksum=336351996484442950290764607947182983196
[debug] (replica): 0: repair_pipeline_diff: 1 prepare(s)
[debug] (replica): 0: verify_pipeline: op=32 checksum=fd0b09c6b6bb54a5ca20c331db50481c parent=3f4f6fe889a5faa960f508d3492209a0
[debug] (replica): 0: repair_pipeline_read: repaired
[debug] (vsr): 0: repair_timeout reset
[debug] (replica): 0: repair_pipeline_diff: 1 prepare(s)
[debug] (replica): 0: verify_pipeline: op=32 checksum=fd0b09c6b6bb54a5ca20c331db50481c parent=3f4f6fe889a5faa960f508d3492209a0
[debug] (replica): 0: repair_pipeline_diff: 1 prepare(s)
[debug] (replica): 0: verify_pipeline: op=32 checksum=fd0b09c6b6bb54a5ca20c331db50481c parent=3f4f6fe889a5faa960f508d3492209a0
[debug] (replica): 0: verify_pipeline: op=32 checksum=fd0b09c6b6bb54a5ca20c331db50481c parent=3f4f6fe889a5faa960f508d3492209a0
[debug] (journal): 0: copy_latest_headers_between: op_min=0 op_max=32 dest.len=32 copied=32
[debug] (replica): 0: transition_to_normal_from_view_change_status: view=6 leader
[debug] (vsr): 0: ping_timeout started
[debug] (vsr): 0: commit_timeout started
[debug] (vsr): 0: normal_status_timeout stopped
[debug] (vsr): 0: view_change_status_timeout stopped
[debug] (vsr): 0: view_change_message_timeout stopped
[debug] (vsr): 0: repair_timeout started
[debug] (vsr): 0: prepare_timeout started
[debug] (replica): 0: reset 3 do_view_change message(s) from view=6
[debug] (replica): 0: send_prepare_ok: op=32 checksum=336351996484442950290764607947182983196
[debug] (replica): 0: sending prepare_ok to replica 0: Header{ .checksum = 98140776031211791920067026070252167432, .checksum_body = 98081761841810822994326224503485043631, .parent = 84153824957273762842204788052591774112, .client = 174512839020594822866659353657821073327, .context = 336351996484442950290764607947182983196, .request = 1, .cluster = 0, .epoch = 0, .view = 6, .op = 32, .commit = 21, .timestamp = 45760000000, .size = 128, .replica = 0, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_message: view=6 status=Status.normal Header{ .checksum = 98140776031211791920067026070252167432, .checksum_body = 98081761841810822994326224503485043631, .parent = 84153824957273762842204788052591774112, .client = 174512839020594822866659353657821073327, .context = 336351996484442950290764607947182983196, .request = 1, .cluster = 0, .epoch = 0, .view = 6, .op = 32, .commit = 21, .timestamp = 45760000000, .size = 128, .replica = 0, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_prepare_ok: 1 message(s)
[debug] (replica): 0: on_prepare_ok: waiting for quorum
[debug] (replica): 0: sending start_view to replica 1: Header{ .checksum = 265301836423630104305909902214800848631, .checksum_body = 87381500561674972023499709457017590277, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 32, .commit = 31, .timestamp = 0, .size = 4224, .replica = 0, .command = Command.start_view, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view
[debug] (replica): 0: sending start_view to replica 2: Header{ .checksum = 265301836423630104305909902214800848631, .checksum_body = 87381500561674972023499709457017590277, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 32, .commit = 31, .timestamp = 0, .size = 4224, .replica = 0, .command = Command.start_view, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view
[debug] (replica): 1: on_message: view=6 status=Status.view_change Header{ .checksum = 265301836423630104305909902214800848631, .checksum_body = 87381500561674972023499709457017590277, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 32, .commit = 31, .timestamp = 0, .size = 4224, .replica = 0, .command = Command.start_view, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view: view=6 op=46..32 commit=31..31 checksum=336351996484442950290764607947182983196
[debug] (replica): 1: on_start_view: latest op exists exactly
[debug] (journal): 1: remove_entries_from: op_min=33
[debug] (replica): 1: repair_header: op=32 false (checksum clean)
[debug] (replica): 1: repair_header: op=31 false (checksum clean)
[debug] (replica): 1: repair_header: op=30 false (checksum clean)
[debug] (replica): 1: repair_header: op=29 false (checksum clean)
[debug] (replica): 1: repair_header: op=28 false (checksum clean)
[debug] (replica): 1: repair_header: op=27 exists, checksum dirty
[debug] (journal): 1: set_header_as_dirty: op=27 checksum=48497988437301501205911521605416395675
[debug] (replica): 1: repair_header: op=26 exists, checksum dirty
[debug] (journal): 1: set_header_as_dirty: op=26 checksum=20184652399617286728776349237048428486
[debug] (replica): 1: repair_header: op=25 exists, checksum dirty
[debug] (journal): 1: set_header_as_dirty: op=25 checksum=45324309832085034192143812951103768691
[debug] (replica): 1: repair_header: op=24 false (checksum clean)
[debug] (replica): 1: repair_header: op=23 false (checksum clean)
[debug] (replica): 1: repair_header: op=22 false (checksum clean)
[debug] (replica): 1: repair_header: op=21 false (checksum clean)
[debug] (replica): 1: repair_header: op=20 false (checksum clean)
[debug] (replica): 1: repair_header: op=19 false (checksum clean)
[debug] (replica): 1: repair_header: op=18 false (checksum clean)
[debug] (replica): 1: repair_header: op=17 false (checksum clean)
[debug] (replica): 1: repair_header: op=16 false (checksum clean)
[debug] (replica): 1: repair_header: op=15 false (checksum clean)
[debug] (replica): 1: repair_header: op=14 false (checksum clean)
[debug] (replica): 1: repair_header: op=13 false (checksum clean)
[debug] (replica): 1: repair_header: op=12 false (checksum clean)
[debug] (replica): 1: repair_header: op=11 false (checksum clean)
[debug] (replica): 1: repair_header: op=10 false (checksum clean)
[debug] (replica): 1: repair_header: op=9 false (checksum clean)
[debug] (replica): 1: repair_header: op=8 false (checksum clean)
[debug] (replica): 1: repair_header: op=7 false (checksum clean)
[debug] (replica): 1: repair_header: op=6 false (checksum clean)
[debug] (replica): 1: repair_header: op=5 false (checksum clean)
[debug] (replica): 1: repair_header: op=4 false (checksum clean)
[debug] (replica): 1: repair_header: op=3 false (checksum clean)
[debug] (replica): 1: repair_header: op=2 false (checksum clean)
[debug] (replica): 1: repair_header: op=1 false (checksum clean)
[debug] (replica): 1: transition_to_normal_from_view_change_status: view=6 follower
[debug] (vsr): 1: ping_timeout started
[debug] (vsr): 1: commit_timeout stopped
[debug] (vsr): 1: normal_status_timeout started
[debug] (vsr): 1: view_change_status_timeout stopped
[debug] (vsr): 1: view_change_message_timeout stopped
[debug] (vsr): 1: repair_timeout started
[debug] (replica): 1: reset 0 do_view_change message(s) from view=null
[debug] (replica): 1: send_prepare_ok: op=32 checksum=336351996484442950290764607947182983196
[debug] (replica): 1: sending prepare_ok to replica 0: Header{ .checksum = 14084619405369517758516251252088857830, .checksum_body = 98081761841810822994326224503485043631, .parent = 84153824957273762842204788052591774112, .client = 174512839020594822866659353657821073327, .context = 336351996484442950290764607947182983196, .request = 1, .cluster = 0, .epoch = 0, .view = 6, .op = 32, .commit = 21, .timestamp = 45760000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.prepare_ok
[info] (journal): 1: read_prepare: op=25 checksum=45324309832085034192143812951103768691: dirty
[debug] (replica): 1: commit_ops_commit: prepare == null
[debug] (vsr): 1: repair_timeout reset
[debug] (replica): 1: repair_prepare: op=27 checksum=48497988437301501205911521605416395675 (committed, dirty)
[debug] (replica): 1: sending request_prepare to replica 2: Header{ .checksum = 129547921826550588732936701385919202153, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 48497988437301501205911521605416395675, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 27, .commit = 0, .timestamp = 1, .size = 128, .replica = 1, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.request_prepare
[debug] (replica): 1: repair_prepare: op=26 checksum=20184652399617286728776349237048428486 (committed, dirty)
[debug] (replica): 1: sending request_prepare to replica 0: Header{ .checksum = 148658696523747883378271744712187722821, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 20184652399617286728776349237048428486, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 26, .commit = 0, .timestamp = 1, .size = 128, .replica = 1, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.request_prepare
[debug] (replica): 1: repair_prepare: op=25 checksum=45324309832085034192143812951103768691 (committed, dirty)
[debug] (replica): 1: sending request_prepare to replica 2: Header{ .checksum = 144103446305359984950086427989674622229, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 45324309832085034192143812951103768691, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 25, .commit = 0, .timestamp = 1, .size = 128, .replica = 1, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.request_prepare
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=6 clogged for ticks=49
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=21 clogged for ticks=68
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=31 clogged for ticks=208
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.request_prepare
[debug] (replica): 0: on_message: view=6 status=Status.normal Header{ .checksum = 148658696523747883378271744712187722821, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 20184652399617286728776349237048428486, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 26, .commit = 0, .timestamp = 1, .size = 128, .replica = 1, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_request_prepare: op=26 checksum=20184652399617286728776349237048428486 reading
[debug] (replica): 0: on_request_prepare_read: op=26 checksum=20184652399617286728776349237048428486 sending to replica=1
[debug] (replica): 0: sending prepare to replica 1: Header{ .checksum = 20184652399617286728776349237048428486, .checksum_body = 98081761841810822994326224503485043631, .parent = 45324309832085034192143812951103768691, .client = 7204602051265642679317511360251059264, .context = 38309757949973966338534495299819067648, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 26, .commit = 16, .timestamp = 43540000000, .size = 128, .replica = 2, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.prepare
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=2 clogged for ticks=60
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=27 path.target=4 clogged for ticks=32
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=13 clogged for ticks=58
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=14 clogged for ticks=33
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=17 clogged for ticks=35
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=18 clogged for ticks=75
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=22 clogged for ticks=280
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=25 clogged for ticks=54
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=28 clogged for ticks=12
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=27 path.target=28 clogged for ticks=192
[debug] (packet_simulator): delivering packet from=0 to=1
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.prepare
[debug] (replica): 1: on_message: view=6 status=Status.normal Header{ .checksum = 20184652399617286728776349237048428486, .checksum_body = 98081761841810822994326224503485043631, .parent = 45324309832085034192143812951103768691, .client = 7204602051265642679317511360251059264, .context = 38309757949973966338534495299819067648, .request = 0, .cluster = 0, .epoch = 0, .view = 2, .op = 26, .commit = 16, .timestamp = 43540000000, .size = 128, .replica = 2, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_prepare: ignoring (repair)
[debug] (replica): 1: repair_header: op=26 exists, checksum dirty
[debug] (journal): 1: set_header_as_dirty: op=26 checksum=20184652399617286728776349237048428486
[debug] (replica): 1: on_repair: repairing journal
[debug] (journal): 1: write: view=2 op=26 len=128: 20184652399617286728776349237048428486 starting
[debug] (journal): 1: write_sectors: offset=27279360 len=4096 locked
[debug] (packet_simulator): delivering packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view
[debug] (replica): 2: on_message: view=6 status=Status.view_change Header{ .checksum = 265301836423630104305909902214800848631, .checksum_body = 87381500561674972023499709457017590277, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 6, .op = 32, .commit = 31, .timestamp = 0, .size = 4224, .replica = 0, .command = Command.start_view, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view: view=6 op=52..32 commit=33..31 checksum=336351996484442950290764607947182983196

Time:
2022-07-13 11:48:20.969312034 +0000 UTC

Commit: 53656dcb540bb3801132668ce6d43e3dc0669d9a

Branches: sj-zig-vopr
Parameters:

SEED=9857522920421219267

replicas=4
clients=22
request_probability=31%
idle_on_probability=6%
idle_off_probability=17%
one_way_delay_mean=7ticks
one_way_delay_min=2ticks
packet_loss_probability=18%
path_maximum_capacity=19messages
path_clog_duration_mean=46ticks
path_clog_probability=1%
packet_replay_probability=41%
partition_mode=PartitionMode.uniform_partition
partition_probability=1%
unpartition_probability=1%
partition_stability=155ticks
unpartition_stability=19ticks
read_latency_min=1
read_latency_mean=5
write_latency_min=0
write_latency_mean=58
read_fault_probability=8%
write_fault_probability=5%
crash_probability=0.01%
crash_stability=929ticks
restart_probability=1%
restart_stability=912ticks

Stack Trace:

thread panic: reached unreachable code
zig/lib/std/debug.zig:225:14: in std.debug.assert (simulator)
if (!ok) unreachable; // assertion failure

src/vsr/journal.zig:623:35: in vsr.journal.Journal(vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time),test.storage.Storage).find_latest_headers_break_between (simulator)
assert(a.op > op_min);

src/vsr/replica.zig:3422:72: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).repair (simulator)
var broken = self.journal.find_latest_headers_break_between(self.commit_min, self.op);

src/vsr/replica.zig:1099:24: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).on_do_view_change (simulator)
self.repair();

src/vsr/replica.zig:529:58: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).on_message (simulator)
.do_view_change => self.on_do_view_change(message),

src/test/message_bus.zig:57:27: in test.message_bus.struct:55:35.wrapper (simulator)
on_message(@intToPtr(Context, @ptrToInt(_context)), message);

src/test/network.zig:167:41: in test.network.Network.deliver_message (simulator)
target_bus.on_message_callback.?(target_bus.on_message_context, message);

src/test/packet_simulator.zig:345:42: in test.packet_simulator.PacketSimulator(test.network.Packet).tick (simulator)
data.callback(data.packet, path);

src/simulator.zig:277:46: in main (simulator)
cluster.network.packet_simulator.tick(cluster.health);

zig/lib/std/start.zig:561:37: in std.start.callMain (simulator)
const result = root.main() catch |err| {

zig/lib/std/start.zig:495:12: in std.start.callMainWithArgs (simulator)
return @call(.{ .modifier = .always_inline }, callMain, .{});

zig/lib/std/start.zig:409:17: in std.start.posixCallMainAndExit (simulator)
std.os.exit(@call(.{ .modifier = .always_inline }, callMainWithArgs, .{ argc, argv, envp }));

zig/lib/std/start.zig:322:5: in std.start._start (simulator)
@call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{});

debug: exit with signal: 6. Indicates a crash bug.

Debug Logs:

[debug] (replica): 0: on_request: ignoring (Status.view_change)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=0 clogged for ticks=54
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=0 clogged for ticks=56
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=1 clogged for ticks=63
[debug] (packet_simulator): delivering packet from=3 to=2
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 2 }: Command.pong
[debug] (replica): 2: on_message: view=18 status=Status.view_change Header{ .checksum = 74882088287114611382205139093675478874, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 17, .op = 248960000000, .commit = 0, .timestamp = 249150000000, .size = 128, .replica = 3, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[warn] (clock): 2: learn: m0=248960000000 < window.monotonic=249190000000
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=6 clogged for ticks=9
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=8 clogged for ticks=30
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=11 clogged for ticks=98
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=13 clogged for ticks=0
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=14 clogged for ticks=51
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=15 clogged for ticks=60
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=1 clogged for ticks=36
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=1 clogged for ticks=71
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=10 clogged for ticks=107
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=17 clogged for ticks=24
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=22 clogged for ticks=45
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=2 clogged for ticks=20
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=3 clogged for ticks=25
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=8 clogged for ticks=103
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=11 clogged for ticks=85
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=15 clogged for ticks=58
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=11 clogged for ticks=27
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=6 clogged for ticks=20
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=7 clogged for ticks=18
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=8 clogged for ticks=3
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=13 clogged for ticks=51
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=20 clogged for ticks=2
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=25 clogged for ticks=2
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=7 clogged for ticks=123
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=8 clogged for ticks=49
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=11 clogged for ticks=11
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=11 clogged for ticks=22
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=14 clogged for ticks=92
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=23 clogged for ticks=0
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=23 clogged for ticks=207
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=6 clogged for ticks=101
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=16 clogged for ticks=2
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=20 clogged for ticks=68
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=22 clogged for ticks=123
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=1 clogged for ticks=2
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=2 clogged for ticks=44
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=4 clogged for ticks=11
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=12 clogged for ticks=22
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=14 clogged for ticks=18
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=6 clogged for ticks=2
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=7 clogged for ticks=46
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=4 clogged for ticks=17
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=6 clogged for ticks=65
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=8 clogged for ticks=66
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=19 clogged for ticks=31
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=20 clogged for ticks=106
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=21 clogged for ticks=60
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=23 clogged for ticks=2
[debug] (journal): 1: write_sectors: offset=85999616 len=4096 unlocked
[debug] (journal): 1: write_header: op=82 sectors[8192..12288]
[debug] (journal): 1: write_sectors: offset=8192 len=4096 locked
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=12 clogged for ticks=31
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=13 clogged for ticks=29
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=14 clogged for ticks=83
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=15 clogged for ticks=5
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=21 clogged for ticks=103
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=23 clogged for ticks=85
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=7 clogged for ticks=48
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=12 clogged for ticks=63
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=13 clogged for ticks=22
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=14 clogged for ticks=8
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=20 clogged for ticks=38
[debug] (vsr): 0: view_change_message_timeout fired
[debug] (vsr): 0: view_change_message_timeout reset
[debug] (replica): 0: sending start_view_change to replica 1: Header{ .checksum = 79691489871353510324282307009295525427, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 2: Header{ .checksum = 79691489871353510324282307009295525427, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 3: Header{ .checksum = 79691489871353510324282307009295525427, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.start_view_change
[err] (packet_simulator): dropped packet from=1 to=2.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=10 clogged for ticks=179
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=18 clogged for ticks=93
[err] (packet_simulator): dropped packet from=0 to=2.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=1 clogged for ticks=44
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=2 clogged for ticks=24
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=5 clogged for ticks=3
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=6 clogged for ticks=25
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=6 clogged for ticks=136
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=7 clogged for ticks=6
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=11 clogged for ticks=50
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=13 clogged for ticks=45
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=20 clogged for ticks=37
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=21 clogged for ticks=18
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=0 clogged for ticks=44
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=18 status=Status.view_change Header{ .checksum = 335947023014213264043141552889620115420, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: 2 message(s)
[debug] (replica): 0: on_start_view_change: view=18 quorum received
[debug] (journal): 0: copy_latest_headers_between: op_min=0 op_max=81 dest.len=32 copied=32
[debug] (replica): 0: sending do_view_change to replica 2: Header{ .checksum = 80535606993499355962816532108311077137, .checksum_body = 171743324600902983844775337055006240609, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 81, .commit = 78, .timestamp = 16, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=14 clogged for ticks=15
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=15 clogged for ticks=47
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=15 clogged for ticks=46
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=21 clogged for ticks=44
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=20 clogged for ticks=56
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=9 clogged for ticks=116
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=17 clogged for ticks=50
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=19 clogged for ticks=68
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=22 clogged for ticks=0
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=24 clogged for ticks=27
[debug] (vsr): 1: view_change_message_timeout fired
[debug] (vsr): 1: view_change_message_timeout reset
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 110198014001222544951758937179618828926, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 110198014001222544951758937179618828926, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 3: Header{ .checksum = 110198014001222544951758937179618828926, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (journal): 1: copy_latest_headers_between: op_min=0 op_max=82 dest.len=32 copied=32
[debug] (replica): 1: sending do_view_change to replica 2: Header{ .checksum = 327598597969139091364681125839608223024, .checksum_body = 143880784004785652646303095528758566152, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 82, .commit = 79, .timestamp = 17, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=7 clogged for ticks=29
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=11 clogged for ticks=5
[debug] (packet_simulator): replayed packet from=0 to=3
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (replica): 3: on_message: view=18 status=Status.view_change Header{ .checksum = 79691489871353510324282307009295525427, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_start_view_change: ignoring (duplicate message replica=0)
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (replica): 2: on_message: view=18 status=Status.view_change Header{ .checksum = 327598597969139091364681125839608223024, .checksum_body = 143880784004785652646303095528758566152, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 82, .commit = 79, .timestamp = 17, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_do_view_change: waiting for start_view_change quorum
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=3 clogged for ticks=117
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=6 clogged for ticks=49
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=7 clogged for ticks=88
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=8 clogged for ticks=93
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=12 clogged for ticks=12
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=8 clogged for ticks=90
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=13 clogged for ticks=5
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=14 clogged for ticks=108
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=17 clogged for ticks=13
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=18 status=Status.view_change Header{ .checksum = 110198014001222544951758937179618828926, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=7 clogged for ticks=56
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=7 clogged for ticks=125
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=9 clogged for ticks=6
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=12 clogged for ticks=62
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=12 clogged for ticks=0
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=12 clogged for ticks=42
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=14 clogged for ticks=18
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=17 clogged for ticks=4
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=22 clogged for ticks=64
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=0 clogged for ticks=8
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=0 clogged for ticks=72
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=11 clogged for ticks=50
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=12 clogged for ticks=67
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=16 clogged for ticks=48
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=18 status=Status.view_change Header{ .checksum = 110198014001222544951758937179618828926, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: 2 message(s)
[debug] (replica): 2: on_start_view_change: view=18 quorum received
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=82 dest.len=32 copied=32
[debug] (replica): 2: sending do_view_change to replica 2: Header{ .checksum = 311854842446521445345324040442712148557, .checksum_body = 194330912195972406861627521223716130876, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 82, .commit = 79, .timestamp = 17, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_message: view=18 status=Status.view_change Header{ .checksum = 311854842446521445345324040442712148557, .checksum_body = 194330912195972406861627521223716130876, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 82, .commit = 79, .timestamp = 17, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_do_view_change: 1 message(s)
[debug] (replica): 2: on_do_view_change: waiting for quorum
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=2 clogged for ticks=168
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=3 clogged for ticks=75
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=19 clogged for ticks=5
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=4 clogged for ticks=32
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=6 clogged for ticks=3
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=8 clogged for ticks=70
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=10 clogged for ticks=23
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=11 clogged for ticks=49
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=20 clogged for ticks=13
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=20 clogged for ticks=9
[debug] (vsr): 3: view_change_message_timeout fired
[debug] (vsr): 3: view_change_message_timeout reset
[debug] (replica): 3: sending start_view_change to replica 0: Header{ .checksum = 275263320366800907402193586049391506400, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 3: sending start_view_change to replica 1: Header{ .checksum = 275263320366800907402193586049391506400, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 3: sending start_view_change to replica 2: Header{ .checksum = 275263320366800907402193586049391506400, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=14 clogged for ticks=29
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=20 clogged for ticks=11
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=21 clogged for ticks=16
[debug] (packet_simulator): delivering packet from=3 to=2
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=18 status=Status.view_change Header{ .checksum = 275263320366800907402193586049391506400, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=3)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=6 clogged for ticks=44
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=6 clogged for ticks=23
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=8 clogged for ticks=28
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=19 clogged for ticks=31
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=23 clogged for ticks=208
[debug] (packet_simulator): replayed packet from=0 to=2
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (replica): 2: on_message: view=18 status=Status.view_change Header{ .checksum = 80535606993499355962816532108311077137, .checksum_body = 171743324600902983844775337055006240609, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 81, .commit = 78, .timestamp = 16, .size = 4224, .replica = 0, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_do_view_change: 2 message(s)
[debug] (replica): 2: on_do_view_change: waiting for quorum
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=0 clogged for ticks=86
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=7 clogged for ticks=46
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=7 clogged for ticks=4
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=9 clogged for ticks=58
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=21 clogged for ticks=13
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=23 clogged for ticks=77
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=3 clogged for ticks=33
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=9 clogged for ticks=175
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=13 clogged for ticks=7
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=13 clogged for ticks=56
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=21 clogged for ticks=8
[debug] (packet_simulator): delivering packet from=0 to=3
[debug] (network): deliver_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (replica): 3: on_message: view=18 status=Status.view_change Header{ .checksum = 79691489871353510324282307009295525427, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_start_view_change: ignoring (duplicate message replica=0)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=8 clogged for ticks=73
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=14 clogged for ticks=66
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=16 clogged for ticks=113
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=19 clogged for ticks=14
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=23 clogged for ticks=36
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=2 clogged for ticks=65
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=2 clogged for ticks=52
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=5 clogged for ticks=30
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=11 clogged for ticks=15
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=15 clogged for ticks=17
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=16 clogged for ticks=18
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=20 clogged for ticks=0
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=4 clogged for ticks=48
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=14 clogged for ticks=14
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=23 clogged for ticks=8
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=24 clogged for ticks=39
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=1 clogged for ticks=42
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=2 clogged for ticks=3
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=4 clogged for ticks=5
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=7 clogged for ticks=4
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=12 clogged for ticks=147
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=17 clogged for ticks=16
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=22 clogged for ticks=9
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=8 clogged for ticks=12
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=12 clogged for ticks=1
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=13 clogged for ticks=32
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=21 clogged for ticks=7
[debug] (packet_simulator): replayed packet from=3 to=1
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=18 status=Status.view_change Header{ .checksum = 275263320366800907402193586049391506400, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: ignoring (duplicate message replica=3)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=16 clogged for ticks=31
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=16 clogged for ticks=3
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=19 clogged for ticks=73
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=20 clogged for ticks=22
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=0 clogged for ticks=68
[debug] (packet_simulator): delivering packet from=2 to=1
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.ping
[debug] (replica): 1: on_message: view=18 status=Status.view_change Header{ .checksum = 225584149302827108228199497684661130365, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 17, .op = 248960000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: sending pong to replica 2: Header{ .checksum = 267954186818048801351871709247660689982, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 248960000000, .commit = 0, .timestamp = 249790000000, .size = 128, .replica = 1, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.pong
[err] (packet_simulator): dropped packet from=2 to=1.
[err] (packet_simulator): dropped packet from=2 to=1.
[debug] (packet_simulator): delivering packet from=2 to=1
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=18 status=Status.view_change Header{ .checksum = 335947023014213264043141552889620115420, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: 3 message(s)
[debug] (replica): 1: on_start_view_change: ignoring (quorum received already)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=3 clogged for ticks=3
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=16 clogged for ticks=19
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=18 clogged for ticks=90
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=19 clogged for ticks=137
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=21 clogged for ticks=23
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=1 clogged for ticks=47
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=2 clogged for ticks=49
[debug] (journal): 2: write_sectors: offset=8192 len=4096 unlocked
[debug] (journal): 2: write: view=17 op=82 len=128: 131723891637292393727078547828733382749 complete, marking clean
[debug] (replica): 2: send_prepare_ok: not sending (Status.view_change)
[debug] (packet_simulator): replayed packet from=1 to=3
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.ping
[debug] (replica): 3: on_message: view=18 status=Status.view_change Header{ .checksum = 41384815787764220396561783741447586338, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 17, .op = 248690000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: sending pong to replica 1: Header{ .checksum = 88068512882694308638439375413111890977, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 248690000000, .commit = 0, .timestamp = 249810000000, .size = 128, .replica = 3, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 1 }: Command.pong
[debug] (packet_simulator): delivering packet from=1 to=3
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.commit
[debug] (replica): 3: on_message: view=18 status=Status.view_change Header{ .checksum = 271788685972016012844003541461364744868, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 27846298036423993207042633844736691276, .request = 0, .cluster = 0, .epoch = 0, .view = 17, .op = 0, .commit = 79, .timestamp = 0, .size = 128, .replica = 1, .command = Command.commit, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_commit: ignoring (Status.view_change)
[debug] (packet_simulator): delivering packet from=1 to=3
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.pong
[debug] (replica): 3: on_message: view=18 status=Status.view_change Header{ .checksum = 140843782617361196247420929915742528370, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 17, .op = 248740000000, .commit = 0, .timestamp = 248750000000, .size = 128, .replica = 1, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (clock): 3: learn: replica=1 m0=248740000000 t1=248750000000 m2=249810000000 t2=249810000000 one_way_delay=535000000 asymmetric_delay=415000000 clock_offset=-110000000
[debug] (packet_simulator): replayed packet from=1 to=3
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (replica): 3: on_message: view=18 status=Status.view_change Header{ .checksum = 110198014001222544951758937179618828926, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_start_view_change: 2 message(s)
[debug] (replica): 3: on_start_view_change: view=18 quorum received
[debug] (journal): 3: copy_latest_headers_between: op_min=0 op_max=82 dest.len=32 copied=32
[debug] (replica): 3: sending do_view_change to replica 2: Header{ .checksum = 207739568171757055822698691061797650894, .checksum_body = 194330912195972406861627521223716130876, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 82, .commit = 79, .timestamp = 17, .size = 4224, .replica = 3, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 2 }: Command.do_view_change
[err] (packet_simulator): dropped packet from=1 to=3.
[debug] (packet_simulator): replayed packet from=3 to=1
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=18 status=Status.view_change Header{ .checksum = 275263320366800907402193586049391506400, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: ignoring (duplicate message replica=3)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=5 clogged for ticks=115
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=10 clogged for ticks=62
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=10 clogged for ticks=1
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=10 clogged for ticks=29
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=17 clogged for ticks=1
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=0 clogged for ticks=38
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=17 clogged for ticks=1
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=25 clogged for ticks=63
[debug] (packet_simulator): delivering packet from=1 to=3
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.ping
[debug] (replica): 3: on_message: view=18 status=Status.view_change Header{ .checksum = 41384815787764220396561783741447586338, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 17, .op = 248690000000, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: sending pong to replica 1: Header{ .checksum = 24952560506853691640710098323459023621, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 248690000000, .commit = 0, .timestamp = 249830000000, .size = 128, .replica = 3, .command = Command.pong, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 1 }: Command.pong
[debug] (packet_simulator): delivering packet from=3 to=2
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 2 }: Command.do_view_change
[debug] (replica): 2: on_message: view=18 status=Status.view_change Header{ .checksum = 207739568171757055822698691061797650894, .checksum_body = 194330912195972406861627521223716130876, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 82, .commit = 79, .timestamp = 17, .size = 4224, .replica = 3, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_do_view_change: 3 message(s)
[debug] (replica): 2: on_do_view_change: view=18 quorum received
[debug] (replica): 2: on_do_view_change: replica=0 v'=16 op=81 commit=78 latest=Header{ .checksum = 204380699904797585429928384535287883919, .checksum_body = 174020142112824342182674129793523357988, .parent = 246403822821410763831653339676394452837, .client = 96067873624099894531741700168861565194, .context = 260626934297476295265020909459009929074, .request = 3, .cluster = 0, .epoch = 0, .view = 16, .op = 81, .commit = 77, .timestamp = 237570000000, .size = 370047, .replica = 0, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 2: on_do_view_change: replica=2 v'=17 op=82 commit=79 latest=Header{ .checksum = 131723891637292393727078547828733382749, .checksum_body = 98081761841810822994326224503485043631, .parent = 119566089017366075523588104946039919477, .client = 100200780260967696291019654539489078092, .context = 156612506004995278786003986632564828519, .request = 6, .cluster = 0, .epoch = 0, .view = 17, .op = 82, .commit = 79, .timestamp = 248820000000, .size = 128, .replica = 1, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 2: on_do_view_change: replica=3 v'=17 op=82 commit=79 latest=Header{ .checksum = 131723891637292393727078547828733382749, .checksum_body = 98081761841810822994326224503485043631, .parent = 119566089017366075523588104946039919477, .client = 100200780260967696291019654539489078092, .context = 156612506004995278786003986632564828519, .request = 6, .cluster = 0, .epoch = 0, .view = 17, .op = 82, .commit = 79, .timestamp = 248820000000, .size = 128, .replica = 1, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 2: on_do_view_change: view=18 op=82..82 commit=79..79 checksum=131723891637292393727078547828733382749
[debug] (replica): 2: on_do_view_change: latest op exists exactly
[debug] (journal): 2: remove_entries_from: op_min=83
[debug] (replica): 2: repair_header: op=81 gap
[debug] (replica): 2: repair_header: op=81 false (breaks hash chain)
[debug] (replica): 2: repair_header: op=80 gap
[debug] (journal): 2: set_header_as_dirty: op=80 checksum=246403822821410763831653339676394452837
[debug] (replica): 2: repair_header: op=79 false (exists)
[debug] (replica): 2: repair_header: op=78 false (checksum clean)
[debug] (replica): 2: repair_header: op=77 false (checksum clean)
[debug] (replica): 2: repair_header: op=76 false (checksum clean)
[debug] (replica): 2: repair_header: op=75 false (checksum clean)
[debug] (replica): 2: repair_header: op=74 false (checksum clean)
[debug] (replica): 2: repair_header: op=73 false (checksum clean)
[debug] (replica): 2: repair_header: op=72 false (checksum clean)
[debug] (replica): 2: repair_header: op=71 false (checksum clean)
[debug] (replica): 2: repair_header: op=70 false (checksum clean)
[debug] (replica): 2: repair_header: op=69 false (checksum clean)
[debug] (replica): 2: repair_header: op=68 false (checksum clean)
[debug] (replica): 2: repair_header: op=67 false (checksum clean)
[debug] (replica): 2: repair_header: op=66 false (checksum clean)
[debug] (replica): 2: repair_header: op=65 false (checksum clean)
[debug] (replica): 2: repair_header: op=64 false (checksum clean)
[debug] (replica): 2: repair_header: op=63 false (checksum clean)
[debug] (replica): 2: repair_header: op=62 false (checksum clean)
[debug] (replica): 2: repair_header: op=61 false (checksum clean)
[debug] (replica): 2: repair_header: op=60 false (checksum clean)
[debug] (replica): 2: repair_header: op=59 false (checksum clean)
[debug] (replica): 2: repair_header: op=58 false (checksum clean)
[debug] (replica): 2: repair_header: op=57 false (checksum clean)
[debug] (replica): 2: repair_header: op=56 false (checksum clean)
[debug] (replica): 2: repair_header: op=55 false (checksum clean)
[debug] (replica): 2: repair_header: op=54 false (checksum clean)
[debug] (replica): 2: repair_header: op=53 false (checksum clean)
[debug] (replica): 2: repair_header: op=52 false (checksum clean)
[debug] (replica): 2: repair_header: op=51 false (checksum clean)
[debug] (replica): 2: repair_header: op=50 false (checksum clean)
[debug] (replica): 2: repair_header: op=82 false (checksum clean)
[debug] (replica): 2: repair_header: op=79 false (checksum clean)
[debug] (replica): 2: repair_header: op=78 false (checksum clean)
[debug] (replica): 2: repair_header: op=77 false (checksum clean)
[debug] (replica): 2: repair_header: op=76 false (checksum clean)
[debug] (replica): 2: repair_header: op=75 false (checksum clean)
[debug] (replica): 2: repair_header: op=74 false (checksum clean)
[debug] (replica): 2: repair_header: op=73 false (checksum clean)
[debug] (replica): 2: repair_header: op=72 false (checksum clean)
[debug] (replica): 2: repair_header: op=71 false (checksum clean)
[debug] (replica): 2: repair_header: op=70 false (checksum clean)
[debug] (replica): 2: repair_header: op=69 false (checksum clean)
[debug] (replica): 2: repair_header: op=68 false (checksum clean)
[debug] (replica): 2: repair_header: op=67 false (checksum clean)
[debug] (replica): 2: repair_header: op=66 false (checksum clean)
[debug] (replica): 2: repair_header: op=65 false (checksum clean)
[debug] (replica): 2: repair_header: op=64 false (checksum clean)
[debug] (replica): 2: repair_header: op=63 false (checksum clean)
[debug] (replica): 2: repair_header: op=62 false (checksum clean)
[debug] (replica): 2: repair_header: op=61 false (checksum clean)
[debug] (replica): 2: repair_header: op=60 false (checksum clean)
[debug] (replica): 2: repair_header: op=59 false (checksum clean)
[debug] (replica): 2: repair_header: op=58 false (checksum clean)
[debug] (replica): 2: repair_header: op=57 false (checksum clean)
[debug] (replica): 2: repair_header: op=56 false (checksum clean)
[debug] (replica): 2: repair_header: op=55 false (checksum clean)
[debug] (replica): 2: repair_header: op=54 false (checksum clean)
[debug] (replica): 2: repair_header: op=53 false (checksum clean)
[debug] (replica): 2: repair_header: op=52 false (checksum clean)
[debug] (replica): 2: repair_header: op=51 false (checksum clean)
[debug] (replica): 2: repair_header: op=50 false (checksum clean)
[debug] (replica): 2: repair_header: op=49 false (checksum clean)
[debug] (replica): 2: repair_header: op=82 false (checksum clean)
[debug] (replica): 2: repair_header: op=79 false (checksum clean)
[debug] (replica): 2: repair_header: op=78 false (checksum clean)
[debug] (replica): 2: repair_header: op=77 false (checksum clean)
[debug] (replica): 2: repair_header: op=76 false (checksum clean)
[debug] (replica): 2: repair_header: op=75 false (checksum clean)
[debug] (replica): 2: repair_header: op=74 false (checksum clean)
[debug] (replica): 2: repair_header: op=73 false (checksum clean)
[debug] (replica): 2: repair_header: op=72 false (checksum clean)
[debug] (replica): 2: repair_header: op=71 false (checksum clean)
[debug] (replica): 2: repair_header: op=70 false (checksum clean)
[debug] (replica): 2: repair_header: op=69 false (checksum clean)
[debug] (replica): 2: repair_header: op=68 false (checksum clean)
[debug] (replica): 2: repair_header: op=67 false (checksum clean)
[debug] (replica): 2: repair_header: op=66 false (checksum clean)
[debug] (replica): 2: repair_header: op=65 false (checksum clean)
[debug] (replica): 2: repair_header: op=64 false (checksum clean)
[debug] (replica): 2: repair_header: op=63 false (checksum clean)
[debug] (replica): 2: repair_header: op=62 false (checksum clean)
[debug] (replica): 2: repair_header: op=61 false (checksum clean)
[debug] (replica): 2: repair_header: op=60 false (checksum clean)
[debug] (replica): 2: repair_header: op=59 false (checksum clean)
[debug] (replica): 2: repair_header: op=58 false (checksum clean)
[debug] (replica): 2: repair_header: op=57 false (checksum clean)
[debug] (replica): 2: repair_header: op=56 false (checksum clean)
[debug] (replica): 2: repair_header: op=55 false (checksum clean)
[debug] (replica): 2: repair_header: op=54 false (checksum clean)
[debug] (replica): 2: repair_header: op=53 false (checksum clean)
[debug] (replica): 2: repair_header: op=52 false (checksum clean)
[debug] (replica): 2: repair_header: op=51 false (checksum clean)
[debug] (replica): 2: repair_header: op=50 false (checksum clean)
[debug] (replica): 2: repair_header: op=49 false (checksum clean)
[debug] (replica): 2: discard_uncommitted_headers: op=81 gap
[debug] (replica): 2: discard_uncommitted_headers: replica=0 op=81 nack=false
[debug] (replica): 2: discard_uncommitted_headers: replica=3 op=81 nack=true
[debug] (replica): 2: discard_uncommitted_headers: op=81 nacks=1 threshold=1
[debug] (journal): 2: remove_entries_from: op_min=81
[debug] (vsr): 2: repair_timeout started
[debug] (vsr): 2: repair_timeout reset

Before dc45c90 the benchmark completes successfully. Afterwards it gets stuck at https://github.com/tigerbeetledb/tigerbeetle/blob/156b04750626d5fd11f017b5bfb9c2858c983926/src/vsr/replica.zig#L5466 because storage.tick() is now a noop.

It would be great to be able to fetch a list of transactions from TigerBeetle.
Possible filter criteria:

• By debit account
• By credit account
• By amount range
• By user data (e.g. find all transfers with the same external transaction reference)

Prob need to limit result count and always have a timestamp range.

Hi TB Team,

So we are running into an issue where the TB client is crashing due to an assert error. @cairin picked this up in our local env

tigerbeetle- 0  url: 10.244.0.54:8080
2022/04/11 11:39:59 grpc server: 0.0.0.0:8080
info(message_bus): connected to replica 0
thread 10 panic: reached unreachable code
/home/don/github/coilhq/tigerbeetle/zig/lib/std/debug.zig:226:14: 0x7f8fc1cbf378 in std.debug.assert (client_c)
/home/don/github/fynbos/fynbos/go/tigerbeetle_go/internal/tigerbeetle/src/vsr/client.zig:484:19: 0x7f8fc1cce7d5 in tigerbeetle.src.vsr.client.Client(tigerbeetle.src.state_machine.StateMachine,tigerbeetle.src.message_bus.MessageBusImpl(tigerbeetle.src.message_bus.ProcessType.client)).send_request_for_the_first_time (client_c)
/home/don/github/fynbos/fynbos/go/tigerbeetle_go/internal/tigerbeetle/src/vsr/client.zig:205:64: 0x7f8fc1cc2f38 in tigerbeetle.src.vsr.client.Client(tigerbeetle.src.state_machine.StateMachine,tigerbeetle.src.message_bus.MessageBusImpl(tigerbeetle.src.message_bus.ProcessType.client)).request (client_c)
/home/don/github/fynbos/fynbos/go/tigerbeetle_go/internal/client_c/client_c.zig:189:19: 0x7f8fc1cc2b2f in request (client_c)

But not 100% what is going on sure as I don't know the client code well enough but the failure is happening here https://github.com/coilhq/tigerbeetle/blob/bd9f03a7f9bf31c62a20c8864a5cd05a775256ba/src/vsr/client.zig#L484

Note i've liked a previous commit as our build is from that commit.

This seems to occur when doing quite a few requests quickly.

At 2a134ec.

debug(lsm_forest_fuzz): Running fuzz_ops[14384] == FuzzOp{ .get_account = 6 }
thread 23939 panic: attempt to use null value
/home/jamie/tigerbeetle/src/lsm/forest_fuzz.zig:231:56: 0x27b1bc in Environment.run (lsm_forest_fuzz)
                            std.mem.asBytes(lsm_account.?),
                                                       ^
/home/jamie/tigerbeetle/src/lsm/forest_fuzz.zig:264:24: 0x2721b9 in fuzz (lsm_forest_fuzz)
    try Environment.run(&storage, fuzz_ops);
                       ^
/home/jamie/tigerbeetle/src/lsm/forest_fuzz.zig:414:13: 0x269c26 in main (lsm_forest_fuzz)
    try fuzz(fuzz_ops);
            ^
/nix/store/g94blv8pk5z0h4maivk3ymcrafrcbwl8-zig-0.9.1/lib/zig/std/start.zig:561:37: 0x262d0a in std.start.callMain (lsm_forest_fuzz)
            const result = root.main() catch |err| {
                                    ^
/nix/store/g94blv8pk5z0h4maivk3ymcrafrcbwl8-zig-0.9.1/lib/zig/std/start.zig:495:12: 0x2438fe in std.start.callMainWithArgs (lsm_forest_fuzz)
    return @call(.{ .modifier = .always_inline }, callMain, .{});
           ^
/nix/store/g94blv8pk5z0h4maivk3ymcrafrcbwl8-zig-0.9.1/lib/zig/std/start.zig:409:17: 0x242996 in std.start.posixCallMainAndExit (lsm_forest_fuzz)
    std.os.exit(@call(.{ .modifier = .always_inline }, callMainWithArgs, .{ argc, argv, envp }));
                ^
/nix/store/g94blv8pk5z0h4maivk3ymcrafrcbwl8-zig-0.9.1/lib/zig/std/start.zig:322:5: 0x2427a2 in std.start._start (lsm_forest_fuzz)
    @call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{});
    ^
The following command terminated unexpectedly:
cd /home/jamie/tigerbeetle && /home/jamie/tigerbeetle/zig-cache/o/7c816c2d3f9fb064b650a8d398257bf3/lsm_forest_fuzz --seed 4075310295854600248
error: the following build command failed with exit code 1:
/home/jamie/tigerbeetle/zig-cache/o/04e062ed50064d3f166819b2149e5107/build /nix/store/g94blv8pk5z0h4maivk3ymcrafrcbwl8-zig-0.9.1/bin/zig /home/jamie/tigerbeetle /home/jamie/tigerbeetle/zig-cache /home/jamie/.cache/zig lsm_forest_fuzz -- --seed 4075310295854600248

Bug Type:
correctness

Seed:
10143829985302419566

Commit Hash:
0ac50f1

Parameters:
[info] (state_checker):
SEED=10143829985302419566

replicas=5
clients=27
request_probability=62%
idle_on_probability=3%
idle_off_probability=18%
one_way_delay_mean=5 ticks
one_way_delay_min=0 ticks
packet_loss_probability=19%
path_maximum_capacity=19 messages
path_clog_duration_mean=375 ticks
path_clog_probability=1%
packet_replay_probability=0%
partition_mode=PartitionMode.uniform_size
partition_probability=2%
unpartition_probability=8%
partition_stability=172 ticks
unpartition_stability=1 ticks
read_latency_min=2
read_latency_mean=7
write_latency_min=2
write_latency_mean=58
read_fault_probability=9%
write_fault_probability=7%
crash_probability=0.01%
crash_stability=485 ticks
restart_probability=1%
restart_stability=699 ticks

Stack Trace:

exit with code: 129


Tail of Debug Logs:
tatus.view_change Header{ .checksum = 231230108626158836529961381852070378075, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 28032485944705957567111332146818750262, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=14 clogged for ticks=274
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=28 clogged for ticks=1355
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=0 clogged for ticks=1230
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=8 clogged for ticks=806
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=17 clogged for ticks=122
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=18 clogged for ticks=907
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=19 clogged for ticks=1219
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=24 clogged for ticks=68
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=24 clogged for ticks=930
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=25 clogged for ticks=847
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=26 path.target=1 clogged for ticks=1741
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=9 clogged for ticks=106
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=13 clogged for ticks=843
[debug] (packet_simulator): delivering packet from=17 to=3
[debug] (network): deliver_message: Process{ .client = 184008646063086638491395644830649974545 } > Process{ .replica = 3 }: Command.ping
[debug] (replica): 3: on_message: view=90 status=Status.view_change Header{ .checksum = 179121144773030476787369175448513698010, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 184008646063086638491395644830649974545, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=18 clogged for ticks=203
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=21 clogged for ticks=256
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=30 path.target=21 clogged for ticks=913
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=25 clogged for ticks=232
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=26 clogged for ticks=143
[debug] (packet_simulator): delivering packet from=27 to=0
[debug] (network): deliver_message: Process{ .client = 90352418367419507113731605561025685627 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 0: on_message: view=90 status=Status.view_change Header{ .checksum = 157434344877472208367948527857171590047, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 90352418367419507113731605561025685627, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=30 path.target=1 clogged for ticks=709
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=18 clogged for ticks=14
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=18 clogged for ticks=59
[debug] (packet_simulator): delivering packet from=22 to=2
[debug] (network): deliver_message: Process{ .client = 191041920094574337586492960510291567805 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=90 status=Status.view_change Header{ .checksum = 183403731933349383192024031436717333432, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 191041920094574337586492960510291567805, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=30 clogged for ticks=136
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=30 clogged for ticks=468
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=30 path.target=13 clogged for ticks=81
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=14 clogged for ticks=392
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=30 clogged for ticks=1027
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=15 clogged for ticks=229
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=16 clogged for ticks=234
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=16 clogged for ticks=413
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=18 clogged for ticks=723
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=21 clogged for ticks=14
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=27 path.target=26 clogged for ticks=235
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=31 clogged for ticks=710
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=10 clogged for ticks=100
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=12 clogged for ticks=484
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=15 clogged for ticks=614
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=22 clogged for ticks=164
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=23 clogged for ticks=1069
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=30 path.target=27 clogged for ticks=89
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=2 clogged for ticks=204
[debug] (packet_simulator): delivering packet from=11 to=2
[debug] (network): deliver_message: Process{ .client = 220758419762140614207641009363307911556 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=90 status=Status.view_change Header{ .checksum = 261871141884449204832211854122268829440, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 220758419762140614207641009363307911556, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=13 clogged for ticks=254
[debug] (packet_simulator): delivering packet from=28 to=3
[debug] (network): deliver_message: Process{ .client = 278708132862542311991413042886314212872 } > Process{ .replica = 3 }: Command.ping
[debug] (replica): 3: on_message: view=90 status=Status.view_change Header{ .checksum = 281455617262523999562264990261689219958, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 278708132862542311991413042886314212872, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=30 clogged for ticks=26
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=0 clogged for ticks=84
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=1 clogged for ticks=240
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=11 clogged for ticks=78
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=13 clogged for ticks=11
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=31 path.target=15 clogged for ticks=136
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=25 clogged for ticks=121
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=26 clogged for ticks=2495
[err] (packet_simulator): dropped packet (different partitions): from=4 to=0
[err] (packet_simulator): dropped packet (different partitions): from=4 to=0
[err] (packet_simulator): dropped packet (different partitions): from=4 to=0
[err] (packet_simulator): dropped packet (different partitions): from=4 to=0
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=11 clogged for ticks=177
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=31 path.target=13 clogged for ticks=326
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=29 clogged for ticks=140
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=30 clogged for ticks=33
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=18 clogged for ticks=749
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=19 clogged for ticks=322
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=30 path.target=23 clogged for ticks=269
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=25 clogged for ticks=1265
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=25 clogged for ticks=547
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=2 clogged for ticks=189
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=31 path.target=11 clogged for ticks=605
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=24 clogged for ticks=46
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=0 clogged for ticks=181
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=1 clogged for ticks=120
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=9 clogged for ticks=194
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=11 clogged for ticks=355
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=16 clogged for ticks=333
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=22 clogged for ticks=94
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=28 clogged for ticks=507
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=0 clogged for ticks=1542
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=5 clogged for ticks=272
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=7 clogged for ticks=845
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=18 clogged for ticks=159
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=22 clogged for ticks=131
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=27 clogged for ticks=373
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=28 clogged for ticks=567
[err] (packet_simulator): unpartitioned network: partition={ false, false, false, false, false }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=3 clogged for ticks=514
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=7 clogged for ticks=105
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=7 clogged for ticks=230
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=12 clogged for ticks=12
[debug] (packet_simulator): delivering packet from=13 to=3
[debug] (network): deliver_message: Process{ .client = 80880043955383959581035600005050307052 } > Process{ .replica = 3 }: Command.ping
[debug] (replica): 3: on_message: view=90 status=Status.view_change Header{ .checksum = 166107947338593237608619041744620257229, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 80880043955383959581035600005050307052, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=15 clogged for ticks=248
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=28 clogged for ticks=135
[err] (packet_simulator): dropped packet from=29 to=3.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=4 clogged for ticks=168
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=5 clogged for ticks=792
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=5 clogged for ticks=52
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=6 clogged for ticks=61
[debug] (packet_simulator): delivering packet from=8 to=3
[debug] (network): deliver_message: Process{ .client = 179151936192877953111857557920278575556 } > Process{ .replica = 3 }: Command.ping
[debug] (replica): 3: on_message: view=90 status=Status.view_change Header{ .checksum = 49679614180271159278167426559457045973, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 179151936192877953111857557920278575556, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=12 clogged for ticks=36
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=27 path.target=12 clogged for ticks=299
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=15 clogged for ticks=448
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=19 clogged for ticks=96
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=19 clogged for ticks=653
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=2 clogged for ticks=7
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=7 clogged for ticks=326
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=13 clogged for ticks=80
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=18 clogged for ticks=10
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=18 clogged for ticks=489
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=26 clogged for ticks=643
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=29 clogged for ticks=50
[debug] (vsr): 102496780475644985993625935169868884822: request_timeout fired
[debug] (vsr): 102496780475644985993625935169868884822: request_timeout backing off
[debug] (vsr): 102496780475644985993625935169868884822: request_timeout after=463..837 (rtt=30 min=10 max=1000 attempts=94)
[debug] (client): 102496780475644985993625935169868884822: on_request_timeout: resending request=0 checksum=195716653458963300392147633791983930069
[debug] (client): 102496780475644985993625935169868884822: sending request to replica 0: Header{ .checksum = 195716653458963300392147633791983930069, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 102496780475644985993625935169868884822, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .client = 102496780475644985993625935169868884822 } > Process{ .replica = 0 }: Command.request
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=11 clogged for ticks=143
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=17 clogged for ticks=346
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=18 clogged for ticks=102
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=18 clogged for ticks=554
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=22 clogged for ticks=134
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=23 clogged for ticks=194
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=25 clogged for ticks=729
[debug] (packet_simulator): delivering packet from=26 to=0
[debug] (network): deliver_message: Process{ .client = 102496780475644985993625935169868884822 } > Process{ .replica = 0 }: Command.request
[debug] (replica): 0: on_message: view=90 status=Status.view_change Header{ .checksum = 195716653458963300392147633791983930069, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 102496780475644985993625935169868884822, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (replica): 0: on_request: ignoring (Status.view_change)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=26 clogged for ticks=899
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=28 clogged for ticks=802
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=31 clogged for ticks=688
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=1 clogged for ticks=152
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=16 clogged for ticks=185
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=24 clogged for ticks=49
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=28 clogged for ticks=280
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=30 clogged for ticks=181
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=11 clogged for ticks=93
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=31 path.target=13 clogged for ticks=192
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=21 clogged for ticks=164
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=23 clogged for ticks=266
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=24 clogged for ticks=6
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=0 clogged for ticks=64
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=4 clogged for ticks=533
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=12 clogged for ticks=1806
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=14 clogged for ticks=442
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=16 clogged for ticks=319
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=17 clogged for ticks=1760
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=31 clogged for ticks=353
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=2 clogged for ticks=199
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=15 clogged for ticks=261
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=27 clogged for ticks=279
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=30 path.target=27 clogged for ticks=18
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=30 clogged for ticks=49
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=0 clogged for ticks=624
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=3 clogged for ticks=172
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=6 clogged for ticks=116
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=24 clogged for ticks=554
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=27 clogged for ticks=1273
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=31 path.target=27 clogged for ticks=503
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=28 clogged for ticks=290
[err] (packet_simulator): dropped packet from=4 to=2.
[debug] (packet_simulator): delivering packet from=4 to=2
[debug] (network): deliver_message: Process{ .replica = 4 } > Process{ .replica = 2 }: Command.request_prepare
[debug] (replica): 2: on_message: view=90 status=Status.view_change Header{ .checksum = 308169154596410463874944580057323037550, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 177528978530192716694214910988329953203, .request = 0, .cluster = 0, .epoch = 0, .view = 89, .op = 38, .commit = 0, .timestamp = 1, .size = 128, .replica = 4, .command = Command.request_prepare, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_request_prepare: ignoring (older view)
[debug] (packet_simulator): delivering packet from=4 to=2
[debug] (network): deliver_message: Process{ .replica = 4 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=90 status=Status.view_change Header{ .checksum = 319769534077860800428939586429613724616, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 4, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: 2 message(s)
[debug] (replica): 2: on_start_view_change: view=90 quorum received
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=35 dest.len=32 copied=32
[debug] (replica): 2: sending do_view_change to replica 0: Header{ .checksum = 45196314851654425440015472743882102501, .checksum_body = 152253185890823699600189553543394328753, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 35, .commit = 35, .timestamp = 82, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (packet_simulator): delivering packet from=4 to=2
[debug] (network): deliver_message: Process{ .replica = 4 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=90 status=Status.view_change Header{ .checksum = 319769534077860800428939586429613724616, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 4, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=4)
[debug] (packet_simulator): delivering packet from=4 to=2
[debug] (network): deliver_message: Process{ .replica = 4 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=90 status=Status.view_change Header{ .checksum = 319769534077860800428939586429613724616, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 4, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=4)
[debug] (packet_simulator): delivering packet from=4 to=2
[debug] (network): deliver_message: Process{ .replica = 4 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=90 status=Status.view_change Header{ .checksum = 319769534077860800428939586429613724616, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 4, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=4)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=27 path.target=11 clogged for ticks=524
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=13 clogged for ticks=7
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=14 clogged for ticks=808
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=1 clogged for ticks=198
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=2 clogged for ticks=84
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=3 clogged for ticks=669
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=14 clogged for ticks=201
[debug] (packet_simulator): delivering packet from=28 to=0
[debug] (network): deliver_message: Process{ .client = 278708132862542311991413042886314212872 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 0: on_message: view=90 status=Status.view_change Header{ .checksum = 281455617262523999562264990261689219958, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 278708132862542311991413042886314212872, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=8 clogged for ticks=521
[debug] (packet_simulator): delivering packet from=15 to=2
[debug] (network): deliver_message: Process{ .client = 190632285135834425178144386915196864306 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=90 status=Status.view_change Header{ .checksum = 265009982025563618494008476493792273633, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 190632285135834425178144386915196864306, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=24 clogged for ticks=352
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=26 clogged for ticks=233
[debug] (packet_simulator): delivering packet from=2 to=0
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (replica): 0: on_message: view=90 status=Status.view_change Header{ .checksum = 45196314851654425440015472743882102501, .checksum_body = 152253185890823699600189553543394328753, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 35, .commit = 35, .timestamp = 82, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: 3 message(s)
[debug] (replica): 0: on_do_view_change: view=90 quorum received
[debug] (replica): 0: on_do_view_change: replica=0 v'=85 op=38 commit=34 latest=Header{ .checksum = 177528978530192716694214910988329953203, .checksum_body = 98081761841810822994326224503485043631, .parent = 244673275921834611234593479550153127027, .client = 102496780475644985993625935169868884822, .context = 195716653458963300392147633791983930069, .request = 0, .cluster = 0, .epoch = 0, .view = 85, .op = 38, .commit = 34, .timestamp = 457400000000, .size = 128, .replica = 0, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 0: on_do_view_change: replica=1 v'=82 op=38 commit=34 latest=Header{ .checksum = 177528978530192716694214910988329953203, .checksum_body = 98081761841810822994326224503485043631, .parent = 244673275921834611234593479550153127027, .client = 102496780475644985993625935169868884822, .context = 195716653458963300392147633791983930069, .request = 0, .cluster = 0, .epoch = 0, .view = 85, .op = 38, .commit = 34, .timestamp = 457400000000, .size = 128, .replica = 0, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 0: on_do_view_change: replica=2 v'=82 op=35 commit=35 latest=Header{ .checksum = 58910217213832069309723205167041263321, .checksum_body = 186267705366786651301956611012641839025, .parent = 95628262848101973267514574589428634313, .client = 143629594721711401520561648670630190607, .context = 163876801737218779152715503400216392368, .request = 1, .cluster = 0, .epoch = 0, .view = 81, .op = 35, .commit = 34, .timestamp = 437240000000, .size = 592428, .replica = 1, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_do_view_change: view=90 op=38..38 commit=34..35 checksum=177528978530192716694214910988329953203
[debug] (replica): 0: on_do_view_change: latest op exists exactly
[debug] (journal): 0: remove_entries_from: op_min=39
[debug] (replica): 0: repair_header: op=38 false (checksum clean)
[debug] (replica): 0: repair_header: op=37 false (checksum clean)
[debug] (replica): 0: repair_header: op=36 false (checksum clean)
[debug] (replica): 0: repair_header: op=35 false (checksum clean)
[debug] (replica): 0: repair_header: op=34 false (checksum clean)
[debug] (replica): 0: repair_header: op=33 false (checksum clean)
[debug] (replica): 0: repair_header: op=32 false (checksum clean)
[debug] (replica): 0: repair_header: op=31 false (checksum clean)
[debug] (replica): 0: repair_header: op=30 false (checksum clean)
[debug] (replica): 0: repair_header: op=29 false (checksum clean)
[debug] (replica): 0: repair_header: op=28 false (checksum clean)
[debug] (replica): 0: repair_header: op=27 false (checksum clean)
[debug] (replica): 0: repair_header: op=26 false (checksum clean)
[debug] (replica): 0: repair_header: op=25 false (checksum clean)
[debug] (replica): 0: repair_header: op=24 false (checksum clean)
[debug] (replica): 0: repair_header: op=23 false (checksum clean)
[debug] (replica): 0: repair_header: op=22 false (checksum clean)
[debug] (replica): 0: repair_header: op=21 false (checksum clean)
[debug] (replica): 0: repair_header: op=20 false (checksum clean)
[debug] (replica): 0: repair_header: op=19 false (checksum clean)
[debug] (replica): 0: repair_header: op=18 false (checksum clean)
[debug] (replica): 0: repair_header: op=17 false (checksum clean)
[debug] (replica): 0: repair_header: op=16 false (checksum clean)
[debug] (replica): 0: repair_header: op=15 false (checksum clean)
[debug] (replica): 0: repair_header: op=14 false (checksum clean)
[debug] (replica): 0: repair_header: op=13 false (checksum clean)
[debug] (replica): 0: repair_header: op=12 false (checksum clean)
[debug] (replica): 0: repair_header: op=11 false (checksum clean)
[debug] (replica): 0: repair_header: op=10 false (checksum clean)
[debug] (replica): 0: repair_header: op=9 false (checksum clean)
[debug] (replica): 0: repair_header: op=8 false (checksum clean)
[debug] (replica): 0: repair_header: op=7 false (checksum clean)
[debug] (replica): 0: repair_header: op=38 false (checksum clean)
[debug] (replica): 0: repair_header: op=37 false (checksum clean)
[debug] (replica): 0: repair_header: op=36 false (checksum clean)
[debug] (replica): 0: repair_header: op=35 false (checksum clean)
[debug] (replica): 0: repair_header: op=34 false (checksum clean)
[debug] (replica): 0: repair_header: op=33 false (checksum clean)
[debug] (replica): 0: repair_header: op=32 false (checksum clean)
[debug] (replica): 0: repair_header: op=31 false (checksum clean)
[debug] (replica): 0: repair_header: op=30 false (checksum clean)
[debug] (replica): 0: repair_header: op=29 false (checksum clean)
[debug] (replica): 0: repair_header: op=28 false (checksum clean)
[debug] (replica): 0: repair_header: op=27 false (checksum clean)
[debug] (replica): 0: repair_header: op=26 false (checksum clean)
[debug] (replica): 0: repair_header: op=25 false (checksum clean)
[debug] (replica): 0: repair_header: op=24 false (checksum clean)
[debug] (replica): 0: repair_header: op=23 false (checksum clean)
[debug] (replica): 0: repair_header: op=22 false (checksum clean)
[debug] (replica): 0: repair_header: op=21 false (checksum clean)
[debug] (replica): 0: repair_header: op=20 false (checksum clean)
[debug] (replica): 0: repair_header: op=19 false (checksum clean)
[debug] (replica): 0: repair_header: op=18 false (checksum clean)
[debug] (replica): 0: repair_header: op=17 false (checksum clean)
[debug] (replica): 0: repair_header: op=16 false (checksum clean)
[debug] (replica): 0: repair_header: op=15 false (checksum clean)
[debug] (replica): 0: repair_header: op=14 false (checksum clean)
[debug] (replica): 0: repair_header: op=13 false (checksum clean)
[debug] (replica): 0: repair_header: op=12 false (checksum clean)
[debug] (replica): 0: repair_header: op=11 false (checksum clean)
[debug] (replica): 0: repair_header: op=10 false (checksum clean)
[debug] (replica): 0: repair_header: op=9 false (checksum clean)
[debug] (replica): 0: repair_header: op=8 false (checksum clean)
[debug] (replica): 0: repair_header: op=7 false (checksum clean)
[debug] (replica): 0: repair_header: op=35 false (checksum clean)
[debug] (replica): 0: repair_header: op=34 false (checksum clean)
[debug] (replica): 0: repair_header: op=33 false (checksum clean)
[debug] (replica): 0: repair_header: op=32 false (checksum clean)
[debug] (replica): 0: repair_header: op=31 false (checksum clean)
[debug] (replica): 0: repair_header: op=30 false (checksum clean)
[debug] (replica): 0: repair_header: op=29 false (checksum clean)
[debug] (replica): 0: repair_header: op=28 false (checksum clean)
[debug] (replica): 0: repair_header: op=27 false (checksum clean)
[debug] (replica): 0: repair_header: op=26 false (checksum clean)
[debug] (replica): 0: repair_header: op=25 false (checksum clean)
[debug] (replica): 0: repair_header: op=24 false (checksum clean)
[debug] (replica): 0: repair_header: op=23 false (checksum clean)
[debug] (replica): 0: repair_header: op=22 false (checksum clean)
[debug] (replica): 0: repair_header: op=21 false (checksum clean)
[debug] (replica): 0: repair_header: op=20 false (checksum clean)
[debug] (replica): 0: repair_header: op=19 false (checksum clean)
[debug] (replica): 0: repair_header: op=18 false (checksum clean)
[debug] (replica): 0: repair_header: op=17 false (checksum clean)
[debug] (replica): 0: repair_header: op=16 false (checksum clean)
[debug] (replica): 0: repair_header: op=15 false (checksum clean)
[debug] (replica): 0: repair_header: op=14 false (checksum clean)
[debug] (replica): 0: repair_header: op=13 false (checksum clean)
[debug] (replica): 0: repair_header: op=12 false (checksum clean)
[debug] (replica): 0: repair_header: op=11 false (checksum clean)
[debug] (replica): 0: repair_header: op=10 false (checksum clean)
[debug] (replica): 0: repair_header: op=9 false (checksum clean)
[debug] (replica): 0: repair_header: op=8 false (checksum clean)
[debug] (replica): 0: repair_header: op=7 false (checksum clean)
[debug] (replica): 0: repair_header: op=6 false (checksum clean)
[debug] (replica): 0: repair_header: op=5 false (checksum clean)
[debug] (replica): 0: repair_header: op=4 false (checksum clean)
[debug] (vsr): 0: repair_timeout started
[debug] (vsr): 0: repair_timeout reset
[debug] (journal): 0: read_sectors: offset=36716544 len=1048576
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=10 clogged for ticks=429
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=16 clogged for ticks=1129
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=19 clogged for ticks=666
[debug] (vsr): 0: view_change_message_timeout fired
[debug] (vsr): 0: view_change_message_timeout reset
[debug] (vsr): 1: view_change_message_timeout fired
[debug] (vsr): 1: view_change_message_timeout reset
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 306862647723847229154315538209811092098, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 306862647723847229154315538209811092098, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 3: Header{ .checksum = 306862647723847229154315538209811092098, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 4: Header{ .checksum = 306862647723847229154315538209811092098, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 4 }: Command.start_view_change
[debug] (journal): 1: copy_latest_headers_between: op_min=0 op_max=38 dest.len=32 copied=32
[debug] (replica): 1: sending do_view_change to replica 0: Header{ .checksum = 295125833514046497477417016375813539493, .checksum_body = 43438521744508393877188308739636040781, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 38, .commit = 34, .timestamp = 82, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (vsr): 2: view_change_message_timeout fired
[debug] (vsr): 2: view_change_message_timeout reset
[debug] (replica): 2: sending start_view_change to replica 0: Header{ .checksum = 332506169712965800489683747143313501463, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 1: Header{ .checksum = 332506169712965800489683747143313501463, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 3: Header{ .checksum = 332506169712965800489683747143313501463, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (replica): 2: sending start_view_change to replica 4: Header{ .checksum = 332506169712965800489683747143313501463, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 4 }: Command.start_view_change
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=35 dest.len=32 copied=32
[debug] (replica): 2: sending do_view_change to replica 0: Header{ .checksum = 45196314851654425440015472743882102501, .checksum_body = 152253185890823699600189553543394328753, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 35, .commit = 35, .timestamp = 82, .size = 4224, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (vsr): 3: view_change_message_timeout fired
[debug] (vsr): 3: view_change_message_timeout reset
[debug] (replica): 3: sending start_view_change to replica 0: Header{ .checksum = 294687916571834667022761603202616457336, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 3: sending start_view_change to replica 1: Header{ .checksum = 294687916571834667022761603202616457336, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 3: sending start_view_change to replica 2: Header{ .checksum = 294687916571834667022761603202616457336, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 3: sending start_view_change to replica 4: Header{ .checksum = 294687916571834667022761603202616457336, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 4 }: Command.start_view_change
[debug] (journal): 3: copy_latest_headers_between: op_min=0 op_max=38 dest.len=32 copied=32
[debug] (replica): 3: sending do_view_change to replica 0: Header{ .checksum = 198928583514774852867833222939978396326, .checksum_body = 43438521744508393877188308739636040781, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 38, .commit = 34, .timestamp = 85, .size = 4224, .replica = 3, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (vsr): 4: view_change_message_timeout fired
[debug] (vsr): 4: view_change_message_timeout reset
[debug] (replica): 4: sending start_view_change to replica 0: Header{ .checksum = 319769534077860800428939586429613724616, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 4, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 4 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 4: sending start_view_change to replica 1: Header{ .checksum = 319769534077860800428939586429613724616, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 4, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 4 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 4: sending start_view_change to replica 2: Header{ .checksum = 319769534077860800428939586429613724616, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 4, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 4 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 4: sending start_view_change to replica 3: Header{ .checksum = 319769534077860800428939586429613724616, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 4, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 4 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (journal): 4: copy_latest_headers_between: op_min=0 op_max=38 dest.len=32 copied=32
[debug] (replica): 4: sending do_view_change to replica 0: Header{ .checksum = 245596492756088146394221149159536414346, .checksum_body = 43438521744508393877188308739636040781, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 38, .commit = 35, .timestamp = 85, .size = 4224, .replica = 4, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 4 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=5 clogged for ticks=71
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=6 clogged for ticks=356
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=8 clogged for ticks=551
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=31 path.target=14 clogged for ticks=109
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=21 clogged for ticks=1088
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=23 clogged for ticks=1077
[debug] (packet_simulator): delivering packet from=2 to=1
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=90 status=Status.view_change Header{ .checksum = 332506169712965800489683747143313501463, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: ignoring (duplicate message replica=2)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=5 clogged for ticks=383
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=30 path.target=5 clogged for ticks=972
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=10 clogged for ticks=136
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=14 clogged for ticks=1267
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=27 path.target=17 clogged for ticks=273
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=22 clogged for ticks=317
[debug] (packet_simulator): delivering packet from=4 to=2
[debug] (network): deliver_message: Process{ .replica = 4 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=90 status=Status.view_change Header{ .checksum = 319769534077860800428939586429613724616, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 4, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=4)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=15 clogged for ticks=60
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=19 clogged for ticks=1440
[err] (packet_simulator): dropped packet from=1 to=4.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=2 clogged for ticks=1631
[debug] (packet_simulator): delivering packet from=4 to=1
[debug] (network): deliver_message: Process{ .replica = 4 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=90 status=Status.view_change Header{ .checksum = 319769534077860800428939586429613724616, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 4, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: 4 message(s)
[debug] (replica): 1: on_start_view_change: ignoring (quorum received already)
[debug] (packet_simulator): delivering packet from=4 to=1
[debug] (network): deliver_message: Process{ .replica = 4 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=90 status=Status.view_change Header{ .checksum = 319769534077860800428939586429613724616, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 90, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 4, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: ignoring (duplicate message replica=4)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=7 clogged for ticks=59
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=12 clogged for ticks=392
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=13 clogged for ticks=46
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=27 path.target=24 clogged for ticks=493
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=29 clogged for ticks=448
[debug] (replica): 0: commit_op: executing view=90 true op=35 checksum=58910217213832069309723205167041263321 (hash)
[debug] (state_machine): state=149def99e7952df7ce6565055d61a5e2 input=980b72d2288ede33361ab2b0bb76c03d input.len=592300 new state=8c5f0d9a0888063fa79787199d73778
[err] (state_checker): state checker error: error.ReplicaTransitionedToInvalidState

Time:
2022-07-07 17:41:46.051698684 +0000 UTC

Currently when shutting down tigerbeetle in docker is a slow process. I think TB needs to better handle sigint/sigterm which could correspond to better shutdown modes. Here is an comment from the postgres docker image

We set the default STOPSIGNAL to SIGINT, which corresponds to what PostgreSQL
calls "Fast Shutdown mode" wherein new connections are disallowed and any
in-progress transactions are aborted, allowing PostgreSQL to stop cleanly and
flush tables to disk, which is the best compromise available to avoid data
corruption.
Users who know their applications do not keep open long-lived idle connections
may way to use a value of SIGTERM instead, which corresponds to "Smart
Shutdown mode" in which any existing sessions are allowed to finish and the
server stops when all sessions are terminated.
See https://www.postgresql.org/docs/12/server-shutdown.html for more details
about available PostgreSQL server shutdown signals.
See also https://www.postgresql.org/docs/12/server-start.html for further
justification of this as the default value, namely that the example (and
shipped) systemd service files use the "Fast Shutdown mode" for service
termination.

Bug Type:
crash

Seed:
16161173465443828016

Commit Hash:
01494c5

Parameters:
[info] (state_checker):
SEED=16161173465443828016

replicas=3
clients=21
request_probability=9%
idle_on_probability=16%
idle_off_probability=12%
one_way_delay_mean=5 ticks
one_way_delay_min=1 ticks
packet_loss_probability=8%
path_maximum_capacity=2 messages
path_clog_duration_mean=370 ticks
path_clog_probability=1%
packet_replay_probability=25%
partition_mode=PartitionMode.isolate_single
partition_probability=1%
unpartition_probability=2%
partition_stability=170 ticks
unpartition_stability=3 ticks
read_latency_min=2
read_latency_mean=8
write_latency_min=1
write_latency_mean=28
read_fault_probability=7%
write_fault_probability=9%
crash_probability=0.01%
crash_stability=652 ticks
restart_probability=1%
restart_stability=278 ticks

Stack Trace:

thread panic: reached unreachable code
zig/lib/std/debug.zig:225:14: in std.debug.assert (simulator)
if (!ok) unreachable; // assertion failure
^
src/vsr/replica.zig:2285:19: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).commit_op (simulator)
assert(self.state_machine.commit_timestamp < prepare.header.timestamp);
^
src/vsr/replica.zig:2247:27: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).commit_ops_commit (simulator)
self.commit_op(prepare.?);
^
src/vsr/journal.zig:717:25: in vsr.journal.Journal(vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time),test.storage.Storage).read_prepare (simulator)
callback(replica, message, destination_replica);
^
src/vsr/replica.zig:2185:42: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).commit_ops_read (simulator)
self.journal.read_prepare(commit_ops_commit, op, checksum, null);
^
src/vsr/replica.zig:2163:33: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).commit_ops (simulator)
self.commit_ops_read();
^
src/vsr/replica.zig:1145:28: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).on_start_view (simulator)
self.commit_ops(self.commit_max);
^
src/vsr/replica.zig:530:50: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).on_message (simulator)
.start_view => self.on_start_view(message),
^
src/test/message_bus.zig:57:27: in test.message_bus.struct:55:35.wrapper (simulator)
on_message(@intToPtr(Context, @ptrToInt(_context)), message);
^
src/test/network.zig:167:41: in test.network.Network.deliver_message (simulator)
target_bus.on_message_callback.?(target_bus.on_message_context, message);
^
src/test/packet_simulator.zig:342:42: in test.packet_simulator.PacketSimulator(test.network.Packet).tick (simulator)
data.callback(data.packet, path);
^
src/simulator.zig:277:46: in main (simulator)
cluster.network.packet_simulator.tick(cluster.health);
^
zig/lib/std/start.zig:561:37: in std.start.callMain (simulator)
const result = root.main() catch |err| {
^
zig/lib/std/start.zig:495:12: in std.start.callMainWithArgs (simulator)
return @call(.{ .modifier = .always_inline }, callMain, .{});
^
zig/lib/std/start.zig:409:17: in std.start.posixCallMainAndExit (simulator)
std.os.exit(@call(.{ .modifier = .always_inline }, callMainWithArgs, .{ argc, argv, envp }));
^
zig/lib/std/start.zig:322:5: in std.start._start (simulator)
@call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{});
^
exit with signal: 6. Indicates a crash bug.


Tail of Debug Logs:
ed for ticks=1264
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=12 clogged for ticks=101
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=12 clogged for ticks=487
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=17 clogged for ticks=182
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=19 clogged for ticks=1006
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=18 clogged for ticks=77
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=1 clogged for ticks=559
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=10 clogged for ticks=514
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=0 clogged for ticks=375
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=7 clogged for ticks=482
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=11 clogged for ticks=834
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=0 clogged for ticks=224
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=7 clogged for ticks=280
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=9 clogged for ticks=704
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=1 clogged for ticks=691
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=6 clogged for ticks=70
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=21 clogged for ticks=252
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=17 clogged for ticks=114
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=8 clogged for ticks=71
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=9 clogged for ticks=304
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=11 clogged for ticks=400
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=11 clogged for ticks=44
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=12 clogged for ticks=232
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=19 clogged for ticks=305
[debug] (vsr): 1: view_change_message_timeout fired
[debug] (vsr): 1: view_change_message_timeout reset
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 210585977827242776312673320260768662487, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 210585977827242776312673320260768662487, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[err] (packet_simulator): submit_packet: Path{ .source = 1, .target = 2 } reached capacity, dropped packet=6
[debug] (vsr): 2: view_change_message_timeout fired
[debug] (vsr): 2: view_change_message_timeout reset
[debug] (replica): 2: sending start_view_change to replica 0: Header{ .checksum = 171090930232257726822000413160432348086, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 0 }: Command.start_view_change
[err] (packet_simulator): submit_packet: Path{ .source = 2, .target = 0 } reached capacity, dropped packet=2
[debug] (replica): 2: sending start_view_change to replica 1: Header{ .checksum = 171090930232257726822000413160432348086, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=20 clogged for ticks=1389
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=4 clogged for ticks=71
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=22 clogged for ticks=56
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=18 clogged for ticks=24
[debug] (packet_simulator): delivering packet from=20 to=0
[debug] (network): deliver_message: Process{ .client = 287810089771532728846072714949622740831 } > Process{ .replica = 0 }: Command.request
[debug] (replica): 0: on_message: view=0 status=Status.recovering Header{ .checksum = 226383091331929378406098144783477121380, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 287810089771532728846072714949622740831, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (replica): 0: on_request: ignoring (Status.recovering)
[debug] (vsr): 153180148162341222020479798348339838575: request_timeout fired
[debug] (vsr): 153180148162341222020479798348339838575: request_timeout backing off
[debug] (vsr): 153180148162341222020479798348339838575: request_timeout after=1032..878 (rtt=30 min=10 max=1000 attempts=25)
[debug] (client): 153180148162341222020479798348339838575: on_request_timeout: resending request=0 checksum=311815149182673109280286132091072967031
[debug] (client): 153180148162341222020479798348339838575: sending request to replica 1: Header{ .checksum = 311815149182673109280286132091072967031, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 153180148162341222020479798348339838575, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.request, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .client = 153180148162341222020479798348339838575 } > Process{ .replica = 1 }: Command.request
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=4 clogged for ticks=16
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=8 clogged for ticks=40
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=13 clogged for ticks=77
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=15 clogged for ticks=177
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=21 clogged for ticks=439
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=4 clogged for ticks=163
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=5 clogged for ticks=228
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=13 clogged for ticks=191
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=15 clogged for ticks=296
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=11 clogged for ticks=164
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=12 clogged for ticks=201
[debug] (packet_simulator): delivering packet from=2 to=1
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=19 status=Status.view_change Header{ .checksum = 171090930232257726822000413160432348086, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 2, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: ignoring (duplicate message replica=2)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=8 clogged for ticks=65
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=12 clogged for ticks=7
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=14 clogged for ticks=185
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=20 clogged for ticks=93
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=4 clogged for ticks=473
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=13 clogged for ticks=285
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=0 clogged for ticks=753
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=9 clogged for ticks=679
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=17 clogged for ticks=356
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=4 clogged for ticks=1313
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=9 clogged for ticks=150
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=13 clogged for ticks=799
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=16 clogged for ticks=635
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=7 clogged for ticks=168
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=0 clogged for ticks=765
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=11 clogged for ticks=558
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=11 clogged for ticks=12
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=14 clogged for ticks=302
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=22 clogged for ticks=116
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=19 status=Status.view_change Header{ .checksum = 110198014001222544951758937179618828926, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (older view)
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=19 status=Status.view_change Header{ .checksum = 110198014001222544951758937179618828926, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (older view)
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=19 status=Status.view_change Header{ .checksum = 110198014001222544951758937179618828926, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 18, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (older view)
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=19 status=Status.view_change Header{ .checksum = 210585977827242776312673320260768662487, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: 1 message(s)
[debug] (replica): 2: on_start_view_change: view=19 quorum received
[debug] (journal): 2: copy_latest_headers_between: op_min=0 op_max=18 dest.len=32 copied=19
[debug] (replica): 2: sending do_view_change to replica 1: Header{ .checksum = 106359795531867278133445549438015879794, .checksum_body = 209786732256430492506904588436778875262, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 18, .commit = 9, .timestamp = 10, .size = 2560, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.do_view_change
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=19 status=Status.view_change Header{ .checksum = 210585977827242776312673320260768662487, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=19 status=Status.view_change Header{ .checksum = 210585977827242776312673320260768662487, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): delivering packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=19 status=Status.view_change Header{ .checksum = 210585977827242776312673320260768662487, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): replayed packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 2: on_message: view=19 status=Status.view_change Header{ .checksum = 210585977827242776312673320260768662487, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=4 clogged for ticks=1045
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=9 clogged for ticks=1501
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=16 clogged for ticks=352
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=18 clogged for ticks=133
[err] (packet_simulator): dropped packet from=1 to=2.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=4 clogged for ticks=315
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=10 clogged for ticks=1259
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=20 clogged for ticks=634
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=21 clogged for ticks=29
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=1 clogged for ticks=136
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=9 clogged for ticks=106
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=2 clogged for ticks=770
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=8 clogged for ticks=1006
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=14 clogged for ticks=445
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=4 clogged for ticks=832
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=10 clogged for ticks=1013
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=12 clogged for ticks=541
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=23 clogged for ticks=115
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=0 clogged for ticks=164
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=9 clogged for ticks=326
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=12 clogged for ticks=1429
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=22 clogged for ticks=92
[debug] (packet_simulator): delivering packet from=2 to=1
[debug] (network): deliver_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.do_view_change
[debug] (replica): 1: on_message: view=19 status=Status.view_change Header{ .checksum = 106359795531867278133445549438015879794, .checksum_body = 209786732256430492506904588436778875262, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 18, .commit = 9, .timestamp = 10, .size = 2560, .replica = 2, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_do_view_change: 2 message(s)
[debug] (replica): 1: on_do_view_change: view=19 quorum received
[debug] (replica): 1: on_do_view_change: replica=1 v'=10 op=24 commit=11 latest=Header{ .checksum = 62491947186845620388480043189392706627, .checksum_body = 142962943249812662729585695502793536359, .parent = 213746699062736194393763037979811490389, .client = 319900957435401496420456182321292294007, .context = 35283095150809060692789485478187725823, .request = 1, .cluster = 0, .epoch = 0, .view = 10, .op = 24, .commit = 11, .timestamp = 70500000000, .size = 642685, .replica = 1, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_do_view_change: replica=2 v'=10 op=18 commit=9 latest=Header{ .checksum = 2835632526009919174043886214336399373, .checksum_body = 98081761841810822994326224503485043631, .parent = 145808874704876854497554711267518515503, .client = 64208852036624404841057810865198101861, .context = 195897753792501674194391476476560355387, .request = 0, .cluster = 0, .epoch = 0, .view = 7, .op = 18, .commit = 9, .timestamp = 57170000000, .size = 128, .replica = 1, .command = Command.prepare, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_do_view_change: view=19 op=24..24 commit=11..11 checksum=62491947186845620388480043189392706627
[debug] (replica): 1: on_do_view_change: latest op exists exactly
[debug] (journal): 1: remove_entries_from: op_min=25
[debug] (replica): 1: repair_header: op=24 false (checksum clean)
[debug] (replica): 1: repair_header: op=23 false (checksum clean)
[debug] (replica): 1: repair_header: op=22 false (checksum clean)
[debug] (replica): 1: repair_header: op=21 false (checksum clean)
[debug] (replica): 1: repair_header: op=20 false (checksum clean)
[debug] (replica): 1: repair_header: op=19 false (checksum clean)
[debug] (replica): 1: repair_header: op=18 false (checksum clean)
[debug] (replica): 1: repair_header: op=17 false (checksum clean)
[debug] (replica): 1: repair_header: op=16 false (checksum clean)
[debug] (replica): 1: repair_header: op=15 false (checksum clean)
[debug] (replica): 1: repair_header: op=14 false (checksum clean)
[debug] (replica): 1: repair_header: op=13 false (checksum clean)
[debug] (replica): 1: repair_header: op=12 false (checksum clean)
[debug] (replica): 1: repair_header: op=11 false (checksum clean)
[debug] (replica): 1: repair_header: op=10 false (checksum clean)
[debug] (replica): 1: repair_header: op=9 false (checksum clean)
[debug] (replica): 1: repair_header: op=8 false (checksum clean)
[debug] (replica): 1: repair_header: op=7 false (checksum clean)
[debug] (replica): 1: repair_header: op=6 false (checksum clean)
[debug] (replica): 1: repair_header: op=5 false (checksum clean)
[debug] (replica): 1: repair_header: op=4 false (checksum clean)
[debug] (replica): 1: repair_header: op=3 false (checksum clean)
[debug] (replica): 1: repair_header: op=2 false (checksum clean)
[debug] (replica): 1: repair_header: op=1 false (checksum clean)
[debug] (replica): 1: repair_header: op=0 false (checksum clean)
[debug] (replica): 1: repair_header: op=18 false (checksum clean)
[debug] (replica): 1: repair_header: op=17 false (checksum clean)
[debug] (replica): 1: repair_header: op=16 false (checksum clean)
[debug] (replica): 1: repair_header: op=15 false (checksum clean)
[debug] (replica): 1: repair_header: op=14 false (checksum clean)
[debug] (replica): 1: repair_header: op=13 false (checksum clean)
[debug] (replica): 1: repair_header: op=12 false (checksum clean)
[debug] (replica): 1: repair_header: op=11 false (checksum clean)
[debug] (replica): 1: repair_header: op=10 false (checksum clean)
[debug] (replica): 1: repair_header: op=9 false (checksum clean)
[debug] (replica): 1: repair_header: op=8 false (checksum clean)
[debug] (replica): 1: repair_header: op=7 false (checksum clean)
[debug] (replica): 1: repair_header: op=6 false (checksum clean)
[debug] (replica): 1: repair_header: op=5 false (checksum clean)
[debug] (replica): 1: repair_header: op=4 false (checksum clean)
[debug] (replica): 1: repair_header: op=3 false (checksum clean)
[debug] (replica): 1: repair_header: op=2 false (checksum clean)
[debug] (replica): 1: repair_header: op=1 false (checksum clean)
[debug] (replica): 1: repair_header: op=0 false (checksum clean)
[debug] (vsr): 1: repair_timeout started
[debug] (vsr): 1: repair_timeout reset
[debug] (replica): 1: repair_pipeline_diff: 13 prepare(s)
[debug] (replica): 1: verify_pipeline: op=12 checksum=1820eba2b229193a28519631d5396acb parent=fbe9d7b03eadfbbefc98256a6fe8551a
[debug] (replica): 1: verify_pipeline: op=13 checksum=fb93354ac5f80617fc5223bd05ac1cf2 parent=1820eba2b229193a28519631d5396acb
[debug] (replica): 1: verify_pipeline: op=14 checksum=3cb7faa91ef38aae273bc1fd0148d41b parent=fb93354ac5f80617fc5223bd05ac1cf2
[debug] (replica): 1: verify_pipeline: op=15 checksum=bcb99b50bb514d92d8ddf51b1f780f59 parent=3cb7faa91ef38aae273bc1fd0148d41b
[debug] (replica): 1: verify_pipeline: op=16 checksum=4b43c446c3c26498271ccb86676853a parent=bcb99b50bb514d92d8ddf51b1f780f59
[debug] (replica): 1: verify_pipeline: op=17 checksum=6db1c48e391fb2e74815b5d1843d092f parent=4b43c446c3c26498271ccb86676853a
[debug] (replica): 1: verify_pipeline: op=18 checksum=2221f7a42f02c9354f22540d766c80d parent=6db1c48e391fb2e74815b5d1843d092f
[debug] (replica): 1: verify_pipeline: op=19 checksum=b1584ecc83228316ae875fc74c07528d parent=2221f7a42f02c9354f22540d766c80d
[debug] (replica): 1: verify_pipeline: op=20 checksum=d8863bfeb1e3824baa355c67b928d17e parent=b1584ecc83228316ae875fc74c07528d
[debug] (replica): 1: verify_pipeline: op=21 checksum=cad9c57e3f620f0f629a045d5636a00c parent=d8863bfeb1e3824baa355c67b928d17e
[debug] (replica): 1: verify_pipeline: op=22 checksum=4c89e6afc5398f78b913a1886a879025 parent=cad9c57e3f620f0f629a045d5636a00c
[debug] (replica): 1: verify_pipeline: op=23 checksum=a0ce1de82b8dcaaab51580be768faa55 parent=4c89e6afc5398f78b913a1886a879025
[debug] (replica): 1: verify_pipeline: op=24 checksum=2f0382e06dee6a35247d0c7360a90c43 parent=a0ce1de82b8dcaaab51580be768faa55
[debug] (replica): 1: repair_pipeline_diff: 13 prepare(s)
[debug] (replica): 1: verify_pipeline: op=12 checksum=1820eba2b229193a28519631d5396acb parent=fbe9d7b03eadfbbefc98256a6fe8551a
[debug] (replica): 1: verify_pipeline: op=13 checksum=fb93354ac5f80617fc5223bd05ac1cf2 parent=1820eba2b229193a28519631d5396acb
[debug] (replica): 1: verify_pipeline: op=14 checksum=3cb7faa91ef38aae273bc1fd0148d41b parent=fb93354ac5f80617fc5223bd05ac1cf2
[debug] (replica): 1: verify_pipeline: op=15 checksum=bcb99b50bb514d92d8ddf51b1f780f59 parent=3cb7faa91ef38aae273bc1fd0148d41b
[debug] (replica): 1: verify_pipeline: op=16 checksum=4b43c446c3c26498271ccb86676853a parent=bcb99b50bb514d92d8ddf51b1f780f59
[debug] (replica): 1: verify_pipeline: op=17 checksum=6db1c48e391fb2e74815b5d1843d092f parent=4b43c446c3c26498271ccb86676853a
[debug] (replica): 1: verify_pipeline: op=18 checksum=2221f7a42f02c9354f22540d766c80d parent=6db1c48e391fb2e74815b5d1843d092f
[debug] (replica): 1: verify_pipeline: op=19 checksum=b1584ecc83228316ae875fc74c07528d parent=2221f7a42f02c9354f22540d766c80d
[debug] (replica): 1: verify_pipeline: op=20 checksum=d8863bfeb1e3824baa355c67b928d17e parent=b1584ecc83228316ae875fc74c07528d
[debug] (replica): 1: verify_pipeline: op=21 checksum=cad9c57e3f620f0f629a045d5636a00c parent=d8863bfeb1e3824baa355c67b928d17e
[debug] (replica): 1: verify_pipeline: op=22 checksum=4c89e6afc5398f78b913a1886a879025 parent=cad9c57e3f620f0f629a045d5636a00c
[debug] (replica): 1: verify_pipeline: op=23 checksum=a0ce1de82b8dcaaab51580be768faa55 parent=4c89e6afc5398f78b913a1886a879025
[debug] (replica): 1: verify_pipeline: op=24 checksum=2f0382e06dee6a35247d0c7360a90c43 parent=a0ce1de82b8dcaaab51580be768faa55
[debug] (replica): 1: verify_pipeline: op=12 checksum=1820eba2b229193a28519631d5396acb parent=fbe9d7b03eadfbbefc98256a6fe8551a
[debug] (replica): 1: verify_pipeline: op=13 checksum=fb93354ac5f80617fc5223bd05ac1cf2 parent=1820eba2b229193a28519631d5396acb
[debug] (replica): 1: verify_pipeline: op=14 checksum=3cb7faa91ef38aae273bc1fd0148d41b parent=fb93354ac5f80617fc5223bd05ac1cf2
[debug] (replica): 1: verify_pipeline: op=15 checksum=bcb99b50bb514d92d8ddf51b1f780f59 parent=3cb7faa91ef38aae273bc1fd0148d41b
[debug] (replica): 1: verify_pipeline: op=16 checksum=4b43c446c3c26498271ccb86676853a parent=bcb99b50bb514d92d8ddf51b1f780f59
[debug] (replica): 1: verify_pipeline: op=17 checksum=6db1c48e391fb2e74815b5d1843d092f parent=4b43c446c3c26498271ccb86676853a
[debug] (replica): 1: verify_pipeline: op=18 checksum=2221f7a42f02c9354f22540d766c80d parent=6db1c48e391fb2e74815b5d1843d092f
[debug] (replica): 1: verify_pipeline: op=19 checksum=b1584ecc83228316ae875fc74c07528d parent=2221f7a42f02c9354f22540d766c80d
[debug] (replica): 1: verify_pipeline: op=20 checksum=d8863bfeb1e3824baa355c67b928d17e parent=b1584ecc83228316ae875fc74c07528d
[debug] (replica): 1: verify_pipeline: op=21 checksum=cad9c57e3f620f0f629a045d5636a00c parent=d8863bfeb1e3824baa355c67b928d17e
[debug] (replica): 1: verify_pipeline: op=22 checksum=4c89e6afc5398f78b913a1886a879025 parent=cad9c57e3f620f0f629a045d5636a00c
[debug] (replica): 1: verify_pipeline: op=23 checksum=a0ce1de82b8dcaaab51580be768faa55 parent=4c89e6afc5398f78b913a1886a879025
[debug] (replica): 1: verify_pipeline: op=24 checksum=2f0382e06dee6a35247d0c7360a90c43 parent=a0ce1de82b8dcaaab51580be768faa55
[debug] (journal): 1: copy_latest_headers_between: op_min=0 op_max=24 dest.len=32 copied=25
[debug] (replica): 1: transition_to_normal_from_view_change_status: view=19 leader
[debug] (vsr): 1: ping_timeout started
[debug] (vsr): 1: commit_timeout started
[debug] (vsr): 1: normal_status_timeout stopped
[debug] (vsr): 1: view_change_status_timeout stopped
[debug] (vsr): 1: view_change_message_timeout stopped
[debug] (vsr): 1: repair_timeout started
[debug] (vsr): 1: prepare_timeout started
[debug] (replica): 1: reset 2 do_view_change message(s) from view=19
[debug] (replica): 1: send_prepare_ok: op=12 checksum=32072404654668297725958781508299352779
[debug] (replica): 1: sending prepare_ok to replica 1: Header{ .checksum = 112879901744255822730454138432158268714, .checksum_body = 98081761841810822994326224503485043631, .parent = 334850406791672290168754381324852679962, .client = 308226792076776665654055406093519386056, .context = 32072404654668297725958781508299352779, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 12, .commit = 3, .timestamp = 39170000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_message: view=19 status=Status.normal Header{ .checksum = 112879901744255822730454138432158268714, .checksum_body = 98081761841810822994326224503485043631, .parent = 334850406791672290168754381324852679962, .client = 308226792076776665654055406093519386056, .context = 32072404654668297725958781508299352779, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 12, .commit = 3, .timestamp = 39170000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_prepare_ok: ignoring (duplicate message replica=1)
[debug] (replica): 1: send_prepare_ok: op=13 checksum=334400575472079912669068057645031365874
[debug] (replica): 1: sending prepare_ok to replica 1: Header{ .checksum = 160070992163244516453393629884299695804, .checksum_body = 98081761841810822994326224503485043631, .parent = 32072404654668297725958781508299352779, .client = 85476675160049341928679601485089171373, .context = 334400575472079912669068057645031365874, .request = 1, .cluster = 0, .epoch = 0, .view = 19, .op = 13, .commit = 3, .timestamp = 40040000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_message: view=19 status=Status.normal Header{ .checksum = 160070992163244516453393629884299695804, .checksum_body = 98081761841810822994326224503485043631, .parent = 32072404654668297725958781508299352779, .client = 85476675160049341928679601485089171373, .context = 334400575472079912669068057645031365874, .request = 1, .cluster = 0, .epoch = 0, .view = 19, .op = 13, .commit = 3, .timestamp = 40040000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_prepare_ok: ignoring (duplicate message replica=1)
[debug] (replica): 1: send_prepare_ok: op=14 checksum=80708954073746177826470422838013121563
[debug] (replica): 1: sending prepare_ok to replica 1: Header{ .checksum = 312598948524789804734387498886364225372, .checksum_body = 98081761841810822994326224503485043631, .parent = 334400575472079912669068057645031365874, .client = 185329393834824597269743093564245359584, .context = 80708954073746177826470422838013121563, .request = 1, .cluster = 0, .epoch = 0, .view = 19, .op = 14, .commit = 8, .timestamp = 56290000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_message: view=19 status=Status.normal Header{ .checksum = 312598948524789804734387498886364225372, .checksum_body = 98081761841810822994326224503485043631, .parent = 334400575472079912669068057645031365874, .client = 185329393834824597269743093564245359584, .context = 80708954073746177826470422838013121563, .request = 1, .cluster = 0, .epoch = 0, .view = 19, .op = 14, .commit = 8, .timestamp = 56290000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_prepare_ok: ignoring (duplicate message replica=1)
[debug] (replica): 1: send_prepare_ok: op=15 checksum=250858588296106680526307632778923478873
[debug] (replica): 1: sending prepare_ok to replica 1: Header{ .checksum = 169954443024623636058199536936080794915, .checksum_body = 98081761841810822994326224503485043631, .parent = 80708954073746177826470422838013121563, .client = 153180148162341222020479798348339838575, .context = 250858588296106680526307632778923478873, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 15, .commit = 8, .timestamp = 56490000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_message: view=19 status=Status.normal Header{ .checksum = 169954443024623636058199536936080794915, .checksum_body = 98081761841810822994326224503485043631, .parent = 80708954073746177826470422838013121563, .client = 153180148162341222020479798348339838575, .context = 250858588296106680526307632778923478873, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 15, .commit = 8, .timestamp = 56490000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_prepare_ok: ignoring (duplicate message replica=1)
[debug] (replica): 1: send_prepare_ok: op=16 checksum=6252747783264299950771441366184920378
[debug] (replica): 1: sending prepare_ok to replica 1: Header{ .checksum = 28646806433723719079821166958713493469, .checksum_body = 98081761841810822994326224503485043631, .parent = 250858588296106680526307632778923478873, .client = 247491742881365980942049954145171637731, .context = 6252747783264299950771441366184920378, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 16, .commit = 9, .timestamp = 56870000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_message: view=19 status=Status.normal Header{ .checksum = 28646806433723719079821166958713493469, .checksum_body = 98081761841810822994326224503485043631, .parent = 250858588296106680526307632778923478873, .client = 247491742881365980942049954145171637731, .context = 6252747783264299950771441366184920378, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 16, .commit = 9, .timestamp = 56870000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_prepare_ok: ignoring (duplicate message replica=1)
[debug] (replica): 1: send_prepare_ok: op=17 checksum=145808874704876854497554711267518515503
[debug] (replica): 1: sending prepare_ok to replica 1: Header{ .checksum = 237998811366100125583772710095787723230, .checksum_body = 98081761841810822994326224503485043631, .parent = 6252747783264299950771441366184920378, .client = 5714407116126894641878461151385727860, .context = 145808874704876854497554711267518515503, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 17, .commit = 9, .timestamp = 57110000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_message: view=19 status=Status.normal Header{ .checksum = 237998811366100125583772710095787723230, .checksum_body = 98081761841810822994326224503485043631, .parent = 6252747783264299950771441366184920378, .client = 5714407116126894641878461151385727860, .context = 145808874704876854497554711267518515503, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 17, .commit = 9, .timestamp = 57110000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_prepare_ok: ignoring (duplicate message replica=1)
[debug] (replica): 1: send_prepare_ok: op=18 checksum=2835632526009919174043886214336399373
[debug] (replica): 1: sending prepare_ok to replica 1: Header{ .checksum = 83846620146112459663859484777475739940, .checksum_body = 98081761841810822994326224503485043631, .parent = 145808874704876854497554711267518515503, .client = 64208852036624404841057810865198101861, .context = 2835632526009919174043886214336399373, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 18, .commit = 9, .timestamp = 57170000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_message: view=19 status=Status.normal Header{ .checksum = 83846620146112459663859484777475739940, .checksum_body = 98081761841810822994326224503485043631, .parent = 145808874704876854497554711267518515503, .client = 64208852036624404841057810865198101861, .context = 2835632526009919174043886214336399373, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 18, .commit = 9, .timestamp = 57170000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_prepare_ok: ignoring (duplicate message replica=1)
[debug] (replica): 1: send_prepare_ok: op=19 checksum=235731875608559670888379323049212072589
[debug] (replica): 1: sending prepare_ok to replica 1: Header{ .checksum = 18598987504659116454914756252508659710, .checksum_body = 98081761841810822994326224503485043631, .parent = 2835632526009919174043886214336399373, .client = 287810089771532728846072714949622740831, .context = 235731875608559670888379323049212072589, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 19, .commit = 9, .timestamp = 66720000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_message: view=19 status=Status.normal Header{ .checksum = 18598987504659116454914756252508659710, .checksum_body = 98081761841810822994326224503485043631, .parent = 2835632526009919174043886214336399373, .client = 287810089771532728846072714949622740831, .context = 235731875608559670888379323049212072589, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 19, .commit = 9, .timestamp = 66720000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_prepare_ok: ignoring (duplicate message replica=1)
[debug] (replica): 1: send_prepare_ok: op=20 checksum=287810231709759277723986160506469142910
[debug] (replica): 1: sending prepare_ok to replica 1: Header{ .checksum = 207689714288631293051488299698320841517, .checksum_body = 98081761841810822994326224503485043631, .parent = 235731875608559670888379323049212072589, .client = 266468236836559685217778096289618480169, .context = 287810231709759277723986160506469142910, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 20, .commit = 11, .timestamp = 67870000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_message: view=19 status=Status.normal Header{ .checksum = 207689714288631293051488299698320841517, .checksum_body = 98081761841810822994326224503485043631, .parent = 235731875608559670888379323049212072589, .client = 266468236836559685217778096289618480169, .context = 287810231709759277723986160506469142910, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 20, .commit = 11, .timestamp = 67870000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (replica): 1: on_prepare_ok: ignoring (duplicate message replica=1)
[debug] (replica): 1: send_prepare_ok: op=21 checksum=269634789203911561566710571575270613004
[debug] (replica): 1: sending prepare_ok to replica 1: Header{ .checksum = 64321702136058733927926776245758612199, .checksum_body = 98081761841810822994326224503485043631, .parent = 287810231709759277723986160506469142910, .client = 99628049574114710307046079389893566763, .context = 269634789203911561566710571575270613004, .request = 1, .cluster = 0, .epoch = 0, .view = 19, .op = 21, .commit = 11, .timestamp = 68440000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_message: view=19 status=Status.normal Header{ .checksum = 64321702136058733927926776245758612199, .checksum_body = 98081761841810822994326224503485043631, .parent = 287810231709759277723986160506469142910, .client = 99628049574114710307046079389893566763, .context = 269634789203911561566710571575270613004, .request = 1, .cluster = 0, .epoch = 0, .view = 19, .op = 21, .commit = 11, .timestamp = 68440000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_prepare_ok: ignoring (duplicate message replica=1)
[debug] (replica): 1: send_prepare_ok: op=22 checksum=101737351229448290838130133451397369893
[debug] (replica): 1: sending prepare_ok to replica 1: Header{ .checksum = 208285275274616563484949968015463780785, .checksum_body = 98081761841810822994326224503485043631, .parent = 269634789203911561566710571575270613004, .client = 328207285438033872769143267419582945940, .context = 101737351229448290838130133451397369893, .request = 2, .cluster = 0, .epoch = 0, .view = 19, .op = 22, .commit = 11, .timestamp = 69710000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_message: view=19 status=Status.normal Header{ .checksum = 208285275274616563484949968015463780785, .checksum_body = 98081761841810822994326224503485043631, .parent = 269634789203911561566710571575270613004, .client = 328207285438033872769143267419582945940, .context = 101737351229448290838130133451397369893, .request = 2, .cluster = 0, .epoch = 0, .view = 19, .op = 22, .commit = 11, .timestamp = 69710000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_prepare_ok: ignoring (duplicate message replica=1)
[debug] (replica): 1: send_prepare_ok: op=23 checksum=213746699062736194393763037979811490389
[debug] (replica): 1: sending prepare_ok to replica 1: Header{ .checksum = 235990326384905310778478080803387699798, .checksum_body = 98081761841810822994326224503485043631, .parent = 101737351229448290838130133451397369893, .client = 252270329856794713538497583429063534266, .context = 213746699062736194393763037979811490389, .request = 1, .cluster = 0, .epoch = 0, .view = 19, .op = 23, .commit = 11, .timestamp = 70050000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_message: view=19 status=Status.normal Header{ .checksum = 235990326384905310778478080803387699798, .checksum_body = 98081761841810822994326224503485043631, .parent = 101737351229448290838130133451397369893, .client = 252270329856794713538497583429063534266, .context = 213746699062736194393763037979811490389, .request = 1, .cluster = 0, .epoch = 0, .view = 19, .op = 23, .commit = 11, .timestamp = 70050000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_prepare_ok: ignoring (duplicate message replica=1)
[debug] (replica): 1: send_prepare_ok: op=24 checksum=62491947186845620388480043189392706627
[debug] (replica): 1: sending prepare_ok to replica 1: Header{ .checksum = 235000252070316255854308906061088985192, .checksum_body = 98081761841810822994326224503485043631, .parent = 213746699062736194393763037979811490389, .client = 319900957435401496420456182321292294007, .context = 62491947186845620388480043189392706627, .request = 1, .cluster = 0, .epoch = 0, .view = 19, .op = 24, .commit = 11, .timestamp = 70500000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_message: view=19 status=Status.normal Header{ .checksum = 235000252070316255854308906061088985192, .checksum_body = 98081761841810822994326224503485043631, .parent = 213746699062736194393763037979811490389, .client = 319900957435401496420456182321292294007, .context = 62491947186845620388480043189392706627, .request = 1, .cluster = 0, .epoch = 0, .view = 19, .op = 24, .commit = 11, .timestamp = 70500000000, .size = 128, .replica = 1, .command = Command.prepare_ok, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_prepare_ok: ignoring (duplicate message replica=1)
[debug] (replica): 1: sending start_view to replica 0: Header{ .checksum = 137895789246117398132372792080014765714, .checksum_body = 19290122793229230828952911761826243802, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 24, .commit = 11, .timestamp = 0, .size = 3328, .replica = 1, .command = Command.start_view, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view
[err] (packet_simulator): submit_packet: Path{ .source = 1, .target = 0 } reached capacity, dropped packet=7
[debug] (replica): 1: sending start_view to replica 2: Header{ .checksum = 137895789246117398132372792080014765714, .checksum_body = 19290122793229230828952911761826243802, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 24, .commit = 11, .timestamp = 0, .size = 3328, .replica = 1, .command = Command.start_view, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=4 clogged for ticks=459
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=14 clogged for ticks=361
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=21 clogged for ticks=1083
[debug] (packet_simulator): replayed packet from=1 to=2
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view
[debug] (replica): 2: on_message: view=19 status=Status.view_change Header{ .checksum = 137895789246117398132372792080014765714, .checksum_body = 19290122793229230828952911761826243802, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 24, .commit = 11, .timestamp = 0, .size = 3328, .replica = 1, .command = Command.start_view, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 2: on_start_view: view=19 op=18..24 commit=9..11 checksum=62491947186845620388480043189392706627
[debug] (journal): 2: set_header_as_dirty: op=24 checksum=62491947186845620388480043189392706627
[debug] (journal): 2: remove_entries_from: op_min=25
[debug] (replica): 2: repair_header: op=24 exists, checksum dirty
[debug] (journal): 2: set_header_as_dirty: op=24 checksum=62491947186845620388480043189392706627
[debug] (replica): 2: repair_header: op=23 gap
[debug] (journal): 2: set_header_as_dirty: op=23 checksum=213746699062736194393763037979811490389
[debug] (replica): 2: repair_header: op=22 gap
[debug] (journal): 2: set_header_as_dirty: op=22 checksum=101737351229448290838130133451397369893
[debug] (replica): 2: repair_header: op=21 gap
[debug] (journal): 2: set_header_as_dirty: op=21 checksum=269634789203911561566710571575270613004
[debug] (replica): 2: repair_header: op=20 gap
[debug] (journal): 2: set_header_as_dirty: op=20 checksum=287810231709759277723986160506469142910
[debug] (replica): 2: repair_header: op=19 gap
[debug] (journal): 2: set_header_as_dirty: op=19 checksum=235731875608559670888379323049212072589
[debug] (replica): 2: repair_header: op=18 exists, checksum dirty
[debug] (journal): 2: set_header_as_dirty: op=18 checksum=2835632526009919174043886214336399373
[debug] (replica): 2: repair_header: op=17 false (checksum clean)
[debug] (replica): 2: repair_header: op=16 exists, checksum dirty
[debug] (journal): 2: set_header_as_dirty: op=16 checksum=6252747783264299950771441366184920378
[debug] (replica): 2: repair_header: op=15 exists, checksum dirty
[debug] (journal): 2: set_header_as_dirty: op=15 checksum=250858588296106680526307632778923478873
[debug] (replica): 2: repair_header: op=14 exists, checksum dirty
[debug] (journal): 2: set_header_as_dirty: op=14 checksum=80708954073746177826470422838013121563
[debug] (replica): 2: repair_header: op=13 exists, checksum dirty
[debug] (journal): 2: set_header_as_dirty: op=13 checksum=334400575472079912669068057645031365874
[debug] (replica): 2: repair_header: op=12 exists, checksum dirty
[debug] (journal): 2: set_header_as_dirty: op=12 checksum=32072404654668297725958781508299352779
[debug] (replica): 2: repair_header: op=11 false (checksum clean)
[debug] (replica): 2: repair_header: op=10 false (checksum clean)
[debug] (replica): 2: repair_header: op=9 exists, checksum dirty
[debug] (journal): 2: set_header_as_dirty: op=9 checksum=125142628637491456972972696345912815133
[debug] (replica): 2: repair_header: op=8 false (checksum clean)
[debug] (replica): 2: repair_header: op=7 false (checksum clean)
[debug] (replica): 2: repair_header: op=6 false (checksum clean)
[debug] (replica): 2: repair_header: op=5 false (checksum clean)
[debug] (replica): 2: repair_header: op=4 false (checksum clean)
[debug] (replica): 2: repair_header: op=3 false (checksum clean)
[debug] (replica): 2: repair_header: op=2 false (checksum clean)
[debug] (replica): 2: repair_header: op=1 false (checksum clean)
[debug] (replica): 2: repair_header: op=0 false (checksum clean)
[debug] (replica): 2: transition_to_normal_from_view_change_status: view=19 follower
[debug] (vsr): 2: ping_timeout started
[debug] (vsr): 2: commit_timeout stopped
[debug] (vsr): 2: normal_status_timeout started
[debug] (vsr): 2: view_change_status_timeout stopped
[debug] (vsr): 2: view_change_message_timeout stopped
[debug] (vsr): 2: repair_timeout started
[debug] (replica): 2: reset 0 do_view_change message(s) from view=null
[debug] (replica): 2: send_prepare_ok: not sending (dirty)
[debug] (replica): 2: send_prepare_ok: not sending (dirty)
[debug] (replica): 2: send_prepare_ok: not sending (dirty)
[debug] (replica): 2: send_prepare_ok: not sending (dirty)
[debug] (replica): 2: send_prepare_ok: not sending (dirty)
[debug] (replica): 2: send_prepare_ok: op=17 checksum=145808874704876854497554711267518515503
[debug] (replica): 2: sending prepare_ok to replica 1: Header{ .checksum = 153994284209715364942958607157331886504, .checksum_body = 98081761841810822994326224503485043631, .parent = 6252747783264299950771441366184920378, .client = 5714407116126894641878461151385727860, .context = 145808874704876854497554711267518515503, .request = 0, .cluster = 0, .epoch = 0, .view = 19, .op = 17, .commit = 9, .timestamp = 57110000000, .size = 128, .replica = 2, .command = Command.prepare_ok, .operation = Operation.register, .version = 0 }
[debug] (network): send_message: Process{ .replica = 2 } > Process{ .replica = 1 }: Command.prepare_ok
[debug] (replica): 2: send_prepare_ok: not sending (dirty)
[debug] (replica): 2: send_prepare_ok: not sending (dirty)
[debug] (replica): 2: send_prepare_ok: not sending (dirty)
[debug] (replica): 2: send_prepare_ok: not sending (dirty)
[debug] (replica): 2: send_prepare_ok: not sending (dirty)
[debug] (replica): 2: send_prepare_ok: not sending (dirty)
[debug] (replica): 2: send_prepare_ok: not sending (dirty)
[debug] (replica): 2: commit_op: executing view=19 false op=10 checksum=170012022214129691811322329883558384076 (register)

Time:
2022-07-13 09:21:13.886565352 +0000 UTC

Imagine I have a multicurrency account and my balances are 100 USD and 100 EUR.
If I try to pay 150 USD, most systems will be smart enough to withdraw 100 USD and 42 EUR (~50USD) leaving me with 0 USD and 58 EUR balance. On the other hand if try to pay 150 EUR, most systems will be smart enough to withdraw 100 EUR and 59 USD (~50EUR) leaving me with 41 USD and 0 EUR balance.

How can I implement this logic with tigerbettle in an atomic manner?

Bug Type:
crash

Seed:
2307603236927025621

Commit Hash:
b09db37

Parameters:
[info] (state_checker):
SEED=2307603236927025621

replicas=4
clients=32
request_probability=55%
idle_on_probability=2%
idle_off_probability=16%
one_way_delay_mean=11 ticks
one_way_delay_min=0 ticks
packet_loss_probability=8%
path_maximum_capacity=6 messages
path_clog_duration_mean=171 ticks
path_clog_probability=1%
packet_replay_probability=19%
partition_mode=PartitionMode.isolate_single
partition_probability=1%
unpartition_probability=2%
partition_stability=178 ticks
unpartition_stability=8 ticks
read_latency_min=0
read_latency_mean=10
write_latency_min=2
write_latency_mean=53
read_fault_probability=8%
write_fault_probability=8%
crash_probability=0.01%
crash_stability=787 ticks
restart_probability=1%
restart_stability=818 ticks

Stack Trace:

thread panic: reached unreachable code
zig/lib/std/debug.zig:225:14: in std.debug.assert (simulator)
if (!ok) unreachable; // assertion failure
^
src/vsr/journal.zig:623:35: in vsr.journal.Journal(vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time),test.storage.Storage).find_latest_headers_break_between (simulator)
assert(a.op > op_min);
^
src/vsr/replica.zig:3422:72: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).repair (simulator)
var broken = self.journal.find_latest_headers_break_between(self.commit_min, self.op);
^
src/vsr/replica.zig:1099:24: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).on_do_view_change (simulator)
self.repair();
^
src/vsr/replica.zig:529:58: in vsr.replica.Replica(test.state_machine.StateMachine,test.message_bus.MessageBus,test.storage.Storage,test.time.Time).on_message (simulator)
.do_view_change => self.on_do_view_change(message),
^
src/test/message_bus.zig:57:27: in test.message_bus.struct:55:35.wrapper (simulator)
on_message(@intToPtr(Context, @ptrToInt(_context)), message);
^
src/test/network.zig:167:41: in test.network.Network.deliver_message (simulator)
target_bus.on_message_callback.?(target_bus.on_message_context, message);
^
src/test/packet_simulator.zig:345:42: in test.packet_simulator.PacketSimulator(test.network.Packet).tick (simulator)
data.callback(data.packet, path);
^
src/simulator.zig:277:46: in main (simulator)
cluster.network.packet_simulator.tick(cluster.health);
^
zig/lib/std/start.zig:561:37: in std.start.callMain (simulator)
const result = root.main() catch |err| {
^
zig/lib/std/start.zig:495:12: in std.start.callMainWithArgs (simulator)
return @call(.{ .modifier = .always_inline }, callMain, .{});
^
zig/lib/std/start.zig:409:17: in std.start.posixCallMainAndExit (simulator)
std.os.exit(@call(.{ .modifier = .always_inline }, callMainWithArgs, .{ argc, argv, envp }));
^
zig/lib/std/start.zig:322:5: in std.start._start (simulator)
@call(.{ .modifier = .never_inline }, posixCallMainAndExit, .{});
^
exit with signal: 6. Indicates a crash bug.


Tail of Debug Logs:
.version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=19 clogged for ticks=341
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=25 clogged for ticks=28
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=29 clogged for ticks=265
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=29 clogged for ticks=574
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=2 clogged for ticks=212
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=5 clogged for ticks=287
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=33 path.target=10 clogged for ticks=186
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=33 path.target=19 clogged for ticks=180
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=35 path.target=27 clogged for ticks=412
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=2 path.target=28 clogged for ticks=17
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=35 clogged for ticks=550
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=15 clogged for ticks=121
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=29 clogged for ticks=738
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=29 clogged for ticks=16
[debug] (packet_simulator): delivering packet from=35 to=1
[debug] (network): deliver_message: Process{ .client = 313899296432599841036734661714799091527 } > Process{ .replica = 1 }: Command.ping
[debug] (replica): 1: on_message: view=116 status=Status.view_change Header{ .checksum = 319284032531657572463224707906488312365, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 313899296432599841036734661714799091527, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=2 clogged for ticks=137
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=9 clogged for ticks=198
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=11 clogged for ticks=279
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=23 clogged for ticks=201
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=33 path.target=24 clogged for ticks=44
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=34 path.target=32 clogged for ticks=355
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=35 clogged for ticks=917
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=35 clogged for ticks=5
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=16 clogged for ticks=88
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=32 path.target=19 clogged for ticks=20
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=23 clogged for ticks=17
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=26 clogged for ticks=142
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=27 path.target=2 clogged for ticks=9
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=30 path.target=9 clogged for ticks=404
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=35 path.target=9 clogged for ticks=815
[debug] (packet_simulator): replayed packet from=15 to=2
[debug] (network): deliver_message: Process{ .client = 276863211189640008083582293853128025033 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=116 status=Status.view_change Header{ .checksum = 157188295985695757420265524025443168197, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 276863211189640008083582293853128025033, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=35 path.target=20 clogged for ticks=5
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=31 path.target=6 clogged for ticks=420
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=34 path.target=8 clogged for ticks=18
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=10 clogged for ticks=142
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=13 clogged for ticks=162
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=14 clogged for ticks=172
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=17 clogged for ticks=213
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=20 clogged for ticks=5
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=22 clogged for ticks=445
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=23 clogged for ticks=184
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=25 clogged for ticks=54
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=25 clogged for ticks=475
[err] (packet_simulator): dropped packet from=26 to=1.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=26 clogged for ticks=83
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=26 clogged for ticks=217
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=26 clogged for ticks=37
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=33 path.target=26 clogged for ticks=223
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=31 path.target=31 clogged for ticks=54
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=34 path.target=2 clogged for ticks=51
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=13 clogged for ticks=44
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=16 clogged for ticks=336
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=19 clogged for ticks=79
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=20 clogged for ticks=75
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=26 path.target=20 clogged for ticks=163
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=27 clogged for ticks=223
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=27 clogged for ticks=4
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=29 clogged for ticks=19
[debug] (packet_simulator): delivering packet from=30 to=2
[debug] (network): deliver_message: Process{ .client = 96961538920279659129136396673597551601 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=116 status=Status.view_change Header{ .checksum = 89672263535331463933878366220832331518, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 96961538920279659129136396673597551601, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=0 clogged for ticks=323
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=19 clogged for ticks=587
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=25 clogged for ticks=49
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=25 clogged for ticks=76
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=34 path.target=31 clogged for ticks=306
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=32 clogged for ticks=45
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=5 clogged for ticks=119
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=7 clogged for ticks=265
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=8 clogged for ticks=74
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=8 path.target=9 clogged for ticks=144
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=15 clogged for ticks=44
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=15 clogged for ticks=331
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=34 path.target=21 clogged for ticks=28
[debug] (packet_simulator): delivering packet from=25 to=2
[debug] (network): deliver_message: Process{ .client = 252290228160185086500343102085482571948 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=116 status=Status.view_change Header{ .checksum = 339671455449395084294305386662891822012, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 252290228160185086500343102085482571948, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=26 clogged for ticks=113
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=30 clogged for ticks=43
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=31 clogged for ticks=223
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=31 clogged for ticks=110
[debug] (vsr): 1: view_change_message_timeout fired
[debug] (vsr): 1: view_change_message_timeout reset
[debug] (replica): 1: sending start_view_change to replica 0: Header{ .checksum = 141600800721974632213432197773493852912, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 2: Header{ .checksum = 141600800721974632213432197773493852912, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 1: sending start_view_change to replica 3: Header{ .checksum = 141600800721974632213432197773493852912, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=5 clogged for ticks=0
[debug] (packet_simulator): replayed packet from=9 to=2
[debug] (network): deliver_message: Process{ .client = 8692090197721442754428111418984585964 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=116 status=Status.view_change Header{ .checksum = 15951159401213846548239697836166243408, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 8692090197721442754428111418984585964, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): delivering packet from=9 to=2
[debug] (network): deliver_message: Process{ .client = 8692090197721442754428111418984585964 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=116 status=Status.view_change Header{ .checksum = 15951159401213846548239697836166243408, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 8692090197721442754428111418984585964, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=9 clogged for ticks=22
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=17 clogged for ticks=224
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=32 path.target=26 clogged for ticks=613
[debug] (packet_simulator): delivering packet from=28 to=0
[debug] (network): deliver_message: Process{ .client = 164019838610497512882647126067879499889 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 0: on_message: view=116 status=Status.view_change Header{ .checksum = 237042542613339430792824983921234893990, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 164019838610497512882647126067879499889, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=29 clogged for ticks=141
[debug] (packet_simulator): delivering packet from=1 to=3
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (replica): 3: on_message: view=116 status=Status.view_change Header{ .checksum = 141600800721974632213432197773493852912, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 3: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=4 clogged for ticks=16
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=20 clogged for ticks=58
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=116 status=Status.view_change Header{ .checksum = 141600800721974632213432197773493852912, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 1, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=1)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=31 path.target=2 clogged for ticks=227
[err] (packet_simulator): dropped packet from=5 to=3.
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=14 path.target=9 clogged for ticks=18
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=13 path.target=11 clogged for ticks=60
[debug] (packet_simulator): delivering packet from=15 to=2
[debug] (network): deliver_message: Process{ .client = 276863211189640008083582293853128025033 } > Process{ .replica = 2 }: Command.ping
[debug] (replica): 2: on_message: view=116 status=Status.view_change Header{ .checksum = 157188295985695757420265524025443168197, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 276863211189640008083582293853128025033, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): delivering packet from=24 to=1
[debug] (network): deliver_message: Process{ .client = 171068750686591771117146550268163403062 } > Process{ .replica = 1 }: Command.request
[debug] (replica): 1: on_message: view=116 status=Status.view_change Header{ .checksum = 204405746842745400663901889773471347409, .checksum_body = 5925322286383647777600919495303806592, .parent = 96228287639278778394096242146966163699, .client = 171068750686591771117146550268163403062, .context = 4, .request = 3, .cluster = 0, .epoch = 0, .view = 113, .op = 0, .commit = 0, .timestamp = 0, .size = 1048576, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (replica): 1: on_request: ignoring (Status.view_change)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=24 clogged for ticks=80
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=29 clogged for ticks=0
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=16 path.target=35 clogged for ticks=77
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=3 clogged for ticks=22
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=5 clogged for ticks=103
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=5 clogged for ticks=244
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=8 clogged for ticks=228
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=17 clogged for ticks=505
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=20 clogged for ticks=418
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=32 path.target=33 clogged for ticks=61
[debug] (vsr): 3: view_change_message_timeout fired
[debug] (vsr): 3: view_change_message_timeout reset
[debug] (replica): 3: sending start_view_change to replica 0: Header{ .checksum = 309602676064189718266491026628088251141, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 3: sending start_view_change to replica 1: Header{ .checksum = 309602676064189718266491026628088251141, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 3: sending start_view_change to replica 2: Header{ .checksum = 309602676064189718266491026628088251141, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (journal): 3: copy_latest_headers_between: op_min=0 op_max=46 dest.len=32 copied=32
[debug] (replica): 3: sending do_view_change to replica 0: Header{ .checksum = 272984488358666824910809625651570325506, .checksum_body = 50921966306137435114496407384486497024, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 46, .commit = 45, .timestamp = 114, .size = 4224, .replica = 3, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 3 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=6 clogged for ticks=28
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=10 path.target=10 clogged for ticks=149
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=17 clogged for ticks=615
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=21 clogged for ticks=115
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=25 path.target=32 clogged for ticks=79
[debug] (vsr): 0: view_change_message_timeout fired
[debug] (vsr): 0: view_change_message_timeout reset
[debug] (replica): 0: sending start_view_change to replica 1: Header{ .checksum = 104295809099253968360005005645289834032, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 2: Header{ .checksum = 104295809099253968360005005645289834032, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 2 }: Command.start_view_change
[debug] (replica): 0: sending start_view_change to replica 3: Header{ .checksum = 104295809099253968360005005645289834032, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 0 } > Process{ .replica = 3 }: Command.start_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=19 path.target=1 clogged for ticks=1
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=2 clogged for ticks=181
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=34 path.target=12 clogged for ticks=65
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=15 clogged for ticks=265
[debug] (vsr): 242575447354266710814428897160766561769: request_timeout fired
[debug] (vsr): 242575447354266710814428897160766561769: request_timeout backing off
[debug] (vsr): 242575447354266710814428897160766561769: request_timeout after=83..167 (rtt=30 min=10 max=1000 attempts=5)
[debug] (client): 242575447354266710814428897160766561769: on_request_timeout: resending request=2 checksum=230410473249947775202815328177430305930
[debug] (client): 242575447354266710814428897160766561769: sending request to replica 3: Header{ .checksum = 230410473249947775202815328177430305930, .checksum_body = 258699285477501539301258299190605143151, .parent = 96176801642227922961130270533557646696, .client = 242575447354266710814428897160766561769, .context = 21, .request = 2, .cluster = 0, .epoch = 0, .view = 114, .op = 0, .commit = 0, .timestamp = 0, .size = 223870, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (network): send_message: Process{ .client = 242575447354266710814428897160766561769 } > Process{ .replica = 3 }: Command.request
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=32 path.target=2 clogged for ticks=17
[debug] (packet_simulator): delivering packet from=3 to=0
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 0 }: Command.start_view_change
[debug] (replica): 0: on_message: view=116 status=Status.view_change Header{ .checksum = 309602676064189718266491026628088251141, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_start_view_change: ignoring (duplicate message replica=3)
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=35 path.target=3 clogged for ticks=244
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=4 clogged for ticks=56
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=3 path.target=5 clogged for ticks=95
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=26 path.target=8 clogged for ticks=82
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=27 path.target=11 clogged for ticks=340
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=15 clogged for ticks=710
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=26 path.target=18 clogged for ticks=338
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=32 path.target=20 clogged for ticks=523
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=35 path.target=21 clogged for ticks=37
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=24 clogged for ticks=629
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=15 path.target=27 clogged for ticks=161
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=30 path.target=33 clogged for ticks=6
[debug] (packet_simulator): delivering packet from=8 to=0
[debug] (network): deliver_message: Process{ .client = 55495239005329570407968135538252453127 } > Process{ .replica = 0 }: Command.request
[debug] (replica): 0: on_message: view=116 status=Status.view_change Header{ .checksum = 206291105709023876229511004357442726385, .checksum_body = 40716651581382621803163745678768260898, .parent = 42177954695316304485313342374138644234, .client = 55495239005329570407968135538252453127, .context = 42, .request = 1, .cluster = 0, .epoch = 0, .view = 113, .op = 0, .commit = 0, .timestamp = 0, .size = 824074, .replica = 0, .command = Command.request, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_request: ignoring (Status.view_change)
[debug] (packet_simulator): delivering packet from=8 to=0
[debug] (network): deliver_message: Process{ .client = 55495239005329570407968135538252453127 } > Process{ .replica = 0 }: Command.ping
[debug] (replica): 0: on_message: view=116 status=Status.view_change Header{ .checksum = 14306441989831204653399616571415728156, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 55495239005329570407968135538252453127, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 0, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 0, .command = Command.ping, .operation = Operation.reserved, .version = 0 }
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=35 clogged for ticks=65
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=3 clogged for ticks=24
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=3 clogged for ticks=29
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=4 clogged for ticks=286
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=8 clogged for ticks=826
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=9 clogged for ticks=87
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=33 path.target=10 clogged for ticks=102
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=31 clogged for ticks=109
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=31 clogged for ticks=195
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=34 clogged for ticks=327
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=1 clogged for ticks=173
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=1 clogged for ticks=167
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=23 path.target=16 clogged for ticks=1
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=17 clogged for ticks=808
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=33 path.target=22 clogged for ticks=31
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=6 path.target=24 clogged for ticks=46
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=21 path.target=26 clogged for ticks=387
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=34 path.target=26 clogged for ticks=156
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=31 clogged for ticks=41
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=31 clogged for ticks=6
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=32 path.target=1 clogged for ticks=214
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=10 clogged for ticks=406
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=12 path.target=14 clogged for ticks=545
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=16 clogged for ticks=59
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=24 clogged for ticks=27
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=0 path.target=26 clogged for ticks=102
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=27 clogged for ticks=280
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=9 path.target=29 clogged for ticks=29
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=33 path.target=29 clogged for ticks=5
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=7 path.target=5 clogged for ticks=114
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=17 path.target=5 clogged for ticks=113
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=33 path.target=5 clogged for ticks=128
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=7 clogged for ticks=83
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=1 path.target=10 clogged for ticks=46
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=19 clogged for ticks=248
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=28 path.target=19 clogged for ticks=209
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=29 path.target=23 clogged for ticks=19
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=24 path.target=25 clogged for ticks=82
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=0 clogged for ticks=523
[debug] (packet_simulator): replayed packet from=3 to=0
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (replica): 0: on_message: view=116 status=Status.view_change Header{ .checksum = 272984488358666824910809625651570325506, .checksum_body = 50921966306137435114496407384486497024, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 46, .commit = 45, .timestamp = 114, .size = 4224, .replica = 3, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: ignoring (duplicate message)
[debug] (packet_simulator): delivering packet from=3 to=1
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 1 }: Command.request_headers
[debug] (replica): 1: on_message: view=116 status=Status.view_change Header{ .checksum = 2007897910343056829332569572942988081, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 115, .op = 45, .commit = 45, .timestamp = 0, .size = 128, .replica = 3, .command = Command.request_headers, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_request_headers: ignoring (older view)
[debug] (packet_simulator): delivering packet from=3 to=1
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 1 }: Command.request_headers
[debug] (replica): 1: on_message: view=116 status=Status.view_change Header{ .checksum = 2007897910343056829332569572942988081, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 115, .op = 45, .commit = 45, .timestamp = 0, .size = 128, .replica = 3, .command = Command.request_headers, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_request_headers: ignoring (older view)
[err] (packet_simulator): dropped packet from=3 to=1.
[debug] (packet_simulator): delivering packet from=3 to=1
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 1 }: Command.request_headers
[debug] (replica): 1: on_message: view=116 status=Status.view_change Header{ .checksum = 2007897910343056829332569572942988081, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 115, .op = 45, .commit = 45, .timestamp = 0, .size = 128, .replica = 3, .command = Command.request_headers, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_request_headers: ignoring (older view)
[debug] (packet_simulator): delivering packet from=3 to=1
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 1 }: Command.request_headers
[debug] (replica): 1: on_message: view=116 status=Status.view_change Header{ .checksum = 2007897910343056829332569572942988081, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 115, .op = 45, .commit = 45, .timestamp = 0, .size = 128, .replica = 3, .command = Command.request_headers, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_request_headers: ignoring (older view)
[debug] (packet_simulator): delivering packet from=3 to=1
[debug] (network): deliver_message: Process{ .replica = 3 } > Process{ .replica = 1 }: Command.start_view_change
[debug] (replica): 1: on_message: view=116 status=Status.view_change Header{ .checksum = 309602676064189718266491026628088251141, .checksum_body = 98081761841810822994326224503485043631, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 0, .commit = 0, .timestamp = 0, .size = 128, .replica = 3, .command = Command.start_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 1: on_start_view_change: 2 message(s)
[debug] (replica): 1: on_start_view_change: view=116 quorum received
[debug] (journal): 1: copy_latest_headers_between: op_min=0 op_max=45 dest.len=32 copied=32
[debug] (replica): 1: sending do_view_change to replica 0: Header{ .checksum = 236818326394235417573925765988003848801, .checksum_body = 318113801950903108920132916189221361120, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 45, .commit = 45, .timestamp = 114, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (network): send_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=30 path.target=4 clogged for ticks=67
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=32 path.target=6 clogged for ticks=519
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=8 clogged for ticks=1
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=18 path.target=18 clogged for ticks=380
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=4 path.target=19 clogged for ticks=355
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=20 path.target=28 clogged for ticks=192
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=5 path.target=30 clogged for ticks=138
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=11 path.target=32 clogged for ticks=207
[debug] (packet_simulator): reverse path clogged
[debug] (packet_simulator): Path path.source=22 path.target=0 clogged for ticks=72
[debug] (packet_simulator): delivering packet from=1 to=0
[debug] (network): deliver_message: Process{ .replica = 1 } > Process{ .replica = 0 }: Command.do_view_change
[debug] (replica): 0: on_message: view=116 status=Status.view_change Header{ .checksum = 236818326394235417573925765988003848801, .checksum_body = 318113801950903108920132916189221361120, .parent = 0, .client = 0, .context = 0, .request = 0, .cluster = 0, .epoch = 0, .view = 116, .op = 45, .commit = 45, .timestamp = 114, .size = 4224, .replica = 1, .command = Command.do_view_change, .operation = Operation.reserved, .version = 0 }
[debug] (replica): 0: on_do_view_change: 3 message(s)
[debug] (replica): 0: on_do_view_change: view=116 quorum received
[debug] (replica): 0: on_do_view_change: replica=0 v'=114 op=67 commit=45 latest=Header{ .checksum = 225466488880791551215778493962948705990, .checksum_body = 292953914588440935460132200950529731783, .parent = 237427123328675127808447509399122880956, .client = 276863211189640008083582293853128025033, .context = 65871882543657280361916130310319582953, .request = 1, .cluster = 0, .epoch = 0, .view = 114, .op = 67, .commit = 45, .timestamp = 772180000000, .size = 973116, .replica = 2, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_do_view_change: replica=1 v'=114 op=45 commit=45 latest=Header{ .checksum = 158563123423669088107546308079571285642, .checksum_body = 272252433209143431060000921687982464653, .parent = 16613873452230041296368681780580484624, .client = 242575447354266710814428897160766561769, .context = 9478713457137123902632669967354423488, .request = 1, .cluster = 0, .epoch = 0, .view = 114, .op = 45, .commit = 28, .timestamp = 756170000000, .size = 333708, .replica = 2, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_do_view_change: replica=3 v'=114 op=46 commit=45 latest=Header{ .checksum = 157646444844000058396358105497054208294, .checksum_body = 82126605047714642513600041249309184494, .parent = 100742081376031504748368503220766505612, .client = 324034001925819384619529401651962979440, .context = 117347012834845637349767723642385569062, .request = 1, .cluster = 0, .epoch = 0, .view = 102, .op = 46, .commit = 23, .timestamp = 688570000000, .size = 1034221, .replica = 2, .command = Command.prepare, .operation = Operation(3), .version = 0 }
[debug] (replica): 0: on_do_view_change: view=116 op=67..67 commit=45..45 checksum=225466488880791551215778493962948705990
[debug] (replica): 0: on_do_view_change: latest op exists exactly
[debug] (journal): 0: remove_entries_from: op_min=68
[debug] (replica): 0: repair_header: op=67 false (checksum clean)
[debug] (replica): 0: repair_header: op=66 exists, checksum dirty
[debug] (journal): 0: set_header_as_dirty: op=66 checksum=237427123328675127808447509399122880956
[debug] (replica): 0: repair_header: op=65 false (checksum clean)
[debug] (replica): 0: repair_header: op=61 exists, checksum dirty
[debug] (journal): 0: set_header_as_dirty: op=61 checksum=96069576566591067725212675252770325025
[debug] (replica): 0: repair_header: op=60 exists, checksum dirty
[debug] (journal): 0: set_header_as_dirty: op=60 checksum=174924177162624348330388921621929850626
[debug] (replica): 0: repair_header: op=59 exists, checksum dirty
[debug] (journal): 0: set_header_as_dirty: op=59 checksum=259721155533355521706651232068105675945
[debug] (replica): 0: repair_header: op=58 exists, checksum dirty
[debug] (journal): 0: set_header_as_dirty: op=58 checksum=215609949523286285945119070648942162601
[debug] (replica): 0: repair_header: op=57 exists, checksum dirty
[debug] (journal): 0: set_header_as_dirty: op=57 checksum=103829886747030490564832598892504552012
[debug] (replica): 0: repair_header: op=56 exists, checksum dirty
[debug] (journal): 0: set_header_as_dirty: op=56 checksum=169219917899093443535076331034706083119
[debug] (replica): 0: repair_header: op=55 exists, checksum dirty
[debug] (journal): 0: set_header_as_dirty: op=55 checksum=216543863690832227323803821463276937035
[debug] (replica): 0: repair_header: op=54 exists, checksum dirty
[debug] (journal): 0: set_header_as_dirty: op=54 checksum=177376274815217231599754200724963219351
[debug] (replica): 0: repair_header: op=46 exists, checksum dirty
[debug] (journal): 0: set_header_as_dirty: op=46 checksum=157646444844000058396358105497054208294
[debug] (replica): 0: repair_header: op=45 false (checksum clean)
[debug] (replica): 0: repair_header: op=44 false (checksum clean)
[debug] (replica): 0: repair_header: op=43 false (checksum clean)
[debug] (replica): 0: repair_header: op=42 false (checksum clean)
[debug] (replica): 0: repair_header: op=41 false (checksum clean)
[debug] (replica): 0: repair_header: op=40 false (checksum clean)
[debug] (replica): 0: repair_header: op=39 false (checksum clean)
[debug] (replica): 0: repair_header: op=38 false (checksum clean)
[debug] (replica): 0: repair_header: op=37 false (checksum clean)
[debug] (replica): 0: repair_header: op=36 false (checksum clean)
[debug] (replica): 0: repair_header: op=35 false (checksum clean)
[debug] (replica): 0: repair_header: op=34 false (checksum clean)
[debug] (replica): 0: repair_header: op=33 false (checksum clean)
[debug] (replica): 0: repair_header: op=32 false (checksum clean)
[debug] (replica): 0: repair_header: op=31 false (checksum clean)
[debug] (replica): 0: repair_header: op=30 false (checksum clean)
[debug] (replica): 0: repair_header: op=29 false (checksum clean)
[debug] (replica): 0: repair_header: op=28 false (checksum clean)
[debug] (replica): 0: repair_header: op=27 false (checksum clean)
[debug] (replica): 0: repair_header: op=26 false (checksum clean)
[debug] (replica): 0: repair_header: op=45 false (checksum clean)
[debug] (replica): 0: repair_header: op=44 false (checksum clean)
[debug] (replica): 0: repair_header: op=43 false (checksum clean)
[debug] (replica): 0: repair_header: op=42 false (checksum clean)
[debug] (replica): 0: repair_header: op=41 false (checksum clean)
[debug] (replica): 0: repair_header: op=40 false (checksum clean)
[debug] (replica): 0: repair_header: op=39 false (checksum clean)
[debug] (replica): 0: repair_header: op=38 false (checksum clean)
[debug] (replica): 0: repair_header: op=37 false (checksum clean)
[debug] (replica): 0: repair_header: op=36 false (checksum clean)
[debug] (replica): 0: repair_header: op=35 false (checksum clean)
[debug] (replica): 0: repair_header: op=34 false (checksum clean)
[debug] (replica): 0: repair_header: op=33 false (checksum clean)
[debug] (replica): 0: repair_header: op=32 false (checksum clean)
[debug] (replica): 0: repair_header: op=31 false (checksum clean)
[debug] (replica): 0: repair_header: op=30 false (checksum clean)
[debug] (replica): 0: repair_header: op=29 false (checksum clean)
[debug] (replica): 0: repair_header: op=28 false (checksum clean)
[debug] (replica): 0: repair_header: op=27 false (checksum clean)
[debug] (replica): 0: repair_header: op=26 false (checksum clean)
[debug] (replica): 0: repair_header: op=25 false (checksum clean)
[debug] (replica): 0: repair_header: op=24 false (checksum clean)
[debug] (replica): 0: repair_header: op=23 false (checksum clean)
[debug] (replica): 0: repair_header: op=22 false (checksum clean)
[debug] (replica): 0: repair_header: op=21 false (checksum clean)
[debug] (replica): 0: repair_header: op=20 false (checksum clean)
[debug] (replica): 0: repair_header: op=19 false (checksum clean)
[debug] (replica): 0: repair_header: op=18 false (checksum clean)
[debug] (replica): 0: repair_header: op=17 false (checksum clean)
[debug] (replica): 0: repair_header: op=16 false (checksum clean)
[debug] (replica): 0: repair_header: op=15 false (checksum clean)
[debug] (replica): 0: repair_header: op=14 false (checksum clean)
[debug] (replica): 0: repair_header: op=46 exists, checksum dirty
[debug] (journal): 0: set_header_as_dirty: op=46 checksum=157646444844000058396358105497054208294
[debug] (replica): 0: repair_header: op=44 false (checksum clean)
[debug] (replica): 0: repair_header: op=43 false (checksum clean)
[debug] (replica): 0: repair_header: op=42 false (checksum clean)
[debug] (replica): 0: repair_header: op=41 false (checksum clean)
[debug] (replica): 0: repair_header: op=40 false (checksum clean)
[debug] (replica): 0: repair_header: op=39 false (checksum clean)
[debug] (replica): 0: repair_header: op=38 false (checksum clean)
[debug] (replica): 0: repair_header: op=37 false (checksum clean)
[debug] (replica): 0: repair_header: op=36 false (checksum clean)
[debug] (replica): 0: repair_header: op=35 false (checksum clean)
[debug] (replica): 0: repair_header: op=34 false (checksum clean)
[debug] (replica): 0: repair_header: op=33 false (checksum clean)
[debug] (replica): 0: repair_header: op=32 false (checksum clean)
[debug] (replica): 0: repair_header: op=31 false (checksum clean)
[debug] (replica): 0: repair_header: op=30 false (checksum clean)
[debug] (replica): 0: repair_header: op=29 false (checksum clean)
[debug] (replica): 0: repair_header: op=28 false (checksum clean)
[debug] (replica): 0: repair_header: op=27 false (checksum clean)
[debug] (replica): 0: repair_header: op=26 false (checksum clean)
[debug] (replica): 0: repair_header: op=25 false (checksum clean)
[debug] (replica): 0: repair_header: op=24 false (checksum clean)
[debug] (replica): 0: repair_header: op=23 false (checksum clean)
[debug] (replica): 0: repair_header: op=22 false (checksum clean)
[debug] (replica): 0: repair_header: op=21 false (checksum clean)
[debug] (replica): 0: repair_header: op=20 false (checksum clean)
[debug] (replica): 0: repair_header: op=19 false (checksum clean)
[debug] (replica): 0: repair_header: op=18 false (checksum clean)
[debug] (replica): 0: repair_header: op=17 false (checksum clean)
[debug] (replica): 0: repair_header: op=16 false (checksum clean)
[debug] (replica): 0: repair_header: op=15 false (checksum clean)
[debug] (replica): 0: repair_header: op=14 false (checksum clean)
[debug] (replica): 0: discard_uncommitted_headers: op=64 gap
[debug] (replica): 0: discard_uncommitted_headers: replica=1 op=64 nack=true
[debug] (replica): 0: discard_uncommitted_headers: replica=3 op=64 nack=true
[debug] (replica): 0: discard_uncommitted_headers: op=64 nacks=2 threshold=1
[debug] (journal): 0: remove_entries_from: op_min=64
[debug] (replica): 0: discard_uncommitted_headers: op=63 gap
[debug] (replica): 0: discard_uncommitted_headers: replica=1 op=63 nack=true
[debug] (replica): 0: discard_uncommitted_headers: replica=3 op=63 nack=true
[debug] (replica): 0: discard_uncommitted_headers: op=63 nacks=2 threshold=1
[debug] (journal): 0: remove_entries_from: op_min=63
[debug] (replica): 0: discard_uncommitted_headers: op=62 gap
[debug] (replica): 0: discard_uncommitted_headers: replica=1 op=62 nack=true
[debug] (replica): 0: discard_uncommitted_headers: replica=3 op=62 nack=true
[debug] (replica): 0: discard_uncommitted_headers: op=62 nacks=2 threshold=1
[debug] (journal): 0: remove_entries_from: op_min=62
[debug] (replica): 0: discard_uncommitted_headers: op=53 gap
[debug] (replica): 0: discard_uncommitted_headers: replica=1 op=53 nack=true
[debug] (replica): 0: discard_uncommitted_headers: replica=3 op=53 nack=true
[debug] (replica): 0: discard_uncommitted_headers: op=53 nacks=2 threshold=1
[debug] (journal): 0: remove_entries_from: op_min=53
[debug] (replica): 0: discard_uncommitted_headers: op=52 gap
[debug] (replica): 0: discard_uncommitted_headers: replica=1 op=52 nack=true
[debug] (replica): 0: discard_uncommitted_headers: replica=3 op=52 nack=true
[debug] (replica): 0: discard_uncommitted_headers: op=52 nacks=2 threshold=1
[debug] (journal): 0: remove_entries_from: op_min=52
[debug] (replica): 0: discard_uncommitted_headers: op=51 gap
[debug] (replica): 0: discard_uncommitted_headers: replica=1 op=51 nack=true
[debug] (replica): 0: discard_uncommitted_headers: replica=3 op=51 nack=true
[debug] (replica): 0: discard_uncommitted_headers: op=51 nacks=2 threshold=1
[debug] (journal): 0: remove_entries_from: op_min=51
[debug] (replica): 0: discard_uncommitted_headers: op=50 gap
[debug] (replica): 0: discard_uncommitted_headers: replica=1 op=50 nack=true
[debug] (replica): 0: discard_uncommitted_headers: replica=3 op=50 nack=true
[debug] (replica): 0: discard_uncommitted_headers: op=50 nacks=2 threshold=1
[debug] (journal): 0: remove_entries_from: op_min=50
[debug] (replica): 0: discard_uncommitted_headers: op=49 gap
[debug] (replica): 0: discard_uncommitted_headers: replica=1 op=49 nack=true
[debug] (replica): 0: discard_uncommitted_headers: replica=3 op=49 nack=true
[debug] (replica): 0: discard_uncommitted_headers: op=49 nacks=2 threshold=1
[debug] (journal): 0: remove_entries_from: op_min=49
[debug] (replica): 0: discard_uncommitted_headers: op=48 gap
[debug] (replica): 0: discard_uncommitted_headers: replica=1 op=48 nack=true
[debug] (replica): 0: discard_uncommitted_headers: replica=3 op=48 nack=true
[debug] (replica): 0: discard_uncommitted_headers: op=48 nacks=2 threshold=1
[debug] (journal): 0: remove_entries_from: op_min=48
[debug] (replica): 0: discard_uncommitted_headers: op=47 gap
[debug] (replica): 0: discard_uncommitted_headers: replica=1 op=47 nack=true
[debug] (replica): 0: discard_uncommitted_headers: replica=3 op=47 nack=true
[debug] (replica): 0: discard_uncommitted_headers: op=47 nacks=2 threshold=1
[debug] (journal): 0: remove_entries_from: op_min=47
[debug] (vsr): 0: repair_timeout started
[debug] (vsr): 0: repair_timeout reset

Time:
2022-07-10 09:35:05.669378029 +0000 UTC