k3s etcd restore fails on host with IPv6 only about k3s HOT 2 CLOSED

root@test:~# kubectl get no
NAME   STATUS   ROLES                       AGE   VERSION
test   Ready    control-plane,etcd,master   10m   v1.28.7+k3s1
root@test:~#
root@test:~#
root@test:~# ls /var/lib/rancher/k3s/server/db/snapshots/
on-demand-test-1709966626
root@test:~#
root@test:~# systemctl stop k3s
root@test:~# systemctl is-active k3s
inactive
root@test:~#
root@test:~#
root@test:~# k3s server --cluster-reset --cluster-reset-restore-path=/var/lib/rancher/k3s/server/db/snapshots/on-demand-test-1709966626
WARN[0000] remove /var/lib/rancher/k3s/agent/etc/k3s-agent-load-balancer.json: no such file or directory
WARN[0000] remove /var/lib/rancher/k3s/agent/etc/k3s-api-server-agent-load-balancer.json: no such file or directory
INFO[0000] Starting k3s v1.28.7+k3s1 (051b14b2)
INFO[0000] Managed etcd cluster bootstrap already complete and initialized
INFO[0000] Pre-restore etcd database moved to /var/lib/rancher/k3s/server/db/etcd-old-1709967040
{"level":"info","ts":"2024-03-09T06:50:40.925499Z","caller":"snapshot/v3_snapshot.go:248","msg":"restoring snapshot","path":"/var/lib/rancher/k3s/server/db/snapshots/on-demand-test-1709966626","wal-dir":"/var/lib/rancher/k3s/server/db/etcd/member/wal","data-dir":"/var/lib/rancher/k3s/server/db/etcd","snap-dir":"/var/lib/rancher/k3s/server/db/etcd/member/snap","stack":"go.etcd.io/etcd/etcdutl/v3/snapshot.(*v3Manager).Restore\n\t/go/pkg/mod/github.com/k3s-io/etcd/etcdutl/[email protected]/snapshot/v3_snapshot.go:254\ngithub.com/k3s-io/k3s/pkg/etcd.(*ETCD).Restore\n\t/go/src/github.com/k3s-io/k3s/pkg/etcd/etcd.go:1440\ngithub.com/k3s-io/k3s/pkg/etcd.(*ETCD).Reset\n\t/go/src/github.com/k3s-io/k3s/pkg/etcd/etcd.go:391\ngithub.com/k3s-io/k3s/pkg/cluster.(*Cluster).start\n\t/go/src/github.com/k3s-io/k3s/pkg/cluster/managed.go:70\ngithub.com/k3s-io/k3s/pkg/cluster.(*Cluster).Start\n\t/go/src/github.com/k3s-io/k3s/pkg/cluster/cluster.go:77\ngithub.com/k3s-io/k3s/pkg/daemons/control.prepare\n\t/go/src/github.com/k3s-io/k3s/pkg/daemons/control/server.go:262\ngithub.com/k3s-io/k3s/pkg/daemons/control.Server\n\t/go/src/github.com/k3s-io/k3s/pkg/daemons/control/server.go:35\ngithub.com/k3s-io/k3s/pkg/server.StartServer\n\t/go/src/github.com/k3s-io/k3s/pkg/server/server.go:56\ngithub.com/k3s-io/k3s/pkg/cli/server.run\n\t/go/src/github.com/k3s-io/k3s/pkg/cli/server/server.go:490\ngithub.com/k3s-io/k3s/pkg/cli/server.Run\n\t/go/src/github.com/k3s-io/k3s/pkg/cli/server/server.go:44\ngithub.com/urfave/cli.HandleAction\n\t/go/pkg/mod/github.com/urfave/[email protected]/app.go:524\ngithub.com/urfave/cli.Command.Run\n\t/go/pkg/mod/github.com/urfave/[email protected]/command.go:175\ngithub.com/urfave/cli.(*App).Run\n\t/go/pkg/mod/github.com/urfave/[email protected]/app.go:277\nmain.main\n\t/go/src/github.com/k3s-io/k3s/cmd/server/main.go:81\nruntime.main\n\t/usr/local/go/src/runtime/proc.go:267"}
{"level":"info","ts":"2024-03-09T06:50:40.929964Z","caller":"membership/store.go:141","msg":"Trimming membership information from the backend..."}
{"level":"info","ts":"2024-03-09T06:50:40.966477Z","caller":"membership/cluster.go:421","msg":"added member","cluster-id":"214b29c52e234c97","local-member-id":"0","added-peer-id":"f7a7f54728b9f0da","added-peer-peer-urls":["https://[2a05:d01c:2c7:d803:c903:e274:5215:df54]:2380"]}
{"level":"info","ts":"2024-03-09T06:50:41.037269Z","caller":"snapshot/v3_snapshot.go:269","msg":"restored snapshot","path":"/var/lib/rancher/k3s/server/db/snapshots/on-demand-test-1709966626","wal-dir":"/var/lib/rancher/k3s/server/db/etcd/member/wal","data-dir":"/var/lib/rancher/k3s/server/db/etcd","snap-dir":"/var/lib/rancher/k3s/server/db/etcd/member/snap"}
INFO[0000] Starting etcd for new cluster, cluster-reset=true
{"level":"info","ts":"2024-03-09T06:50:41.043756Z","caller":"embed/etcd.go:127","msg":"configuring peer listeners","listen-peer-urls":["https://[::1]:2380"]}
{"level":"info","ts":"2024-03-09T06:50:41.044055Z","caller":"embed/etcd.go:495","msg":"starting with peer TLS","tls-info":"cert = /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.crt, key = /var/lib/rancher/k3s/server/tls/etcd/peer-server-client.key, client-cert=, client-key=, trusted-ca = /var/lib/rancher/k3s/server/tls/etcd/peer-ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]}
{"level":"info","ts":"2024-03-09T06:50:41.045421Z","caller":"embed/etcd.go:135","msg":"configuring client listeners","listen-client-urls":["https://[::1]:2379"]}
{"level":"info","ts":"2024-03-09T06:50:41.046325Z","caller":"embed/etcd.go:309","msg":"starting an etcd server","etcd-version":"3.5.9","git-sha":"Not provided (use ./build instead of go build)","go-version":"go1.21.7","go-os":"linux","go-arch":"amd64","max-cpu-set":2,"max-cpu-available":2,"member-initialized":true,"name":"test-89299818","data-dir":"/var/lib/rancher/k3s/server/db/etcd","wal-dir":"","wal-dir-dedicated":"","member-dir":"/var/lib/rancher/k3s/server/db/etcd/member","force-new-cluster":true,"heartbeat-interval":"500ms","election-timeout":"5s","initial-election-tick-advance":true,"snapshot-count":10000,"max-wals":5,"max-snapshots":5,"snapshot-catchup-entries":5000,"initial-advertise-peer-urls":["https://[2a05:d01c:2c7:d803:c903:e274:5215:df54]:2380"],"listen-peer-urls":["https://[::1]:2380"],"advertise-client-urls":["https://[::1]:2379"],"listen-client-urls":["https://[::1]:2379"],"listen-metrics-urls":["http://[::1]:2381"],"cors":["*"],"host-whitelist":["*"],"initial-cluster":"","initial-cluster-state":"new","initial-cluster-token":"","quota-backend-bytes":2147483648,"max-request-bytes":1572864,"max-concurrent-streams":4294967295,"pre-vote":true,"initial-corrupt-check":true,"corrupt-check-time-interval":"0s","compact-check-time-enabled":false,"compact-check-time-interval":"1m0s","auto-compaction-mode":"","auto-compaction-retention":"0s","auto-compaction-interval":"0s","discovery-url":"","discovery-proxy":"","downgrade-check-interval":"5s"}
{"level":"info","ts":"2024-03-09T06:50:41.048754Z","caller":"etcdserver/backend.go:81","msg":"opened backend db","path":"/var/lib/rancher/k3s/server/db/etcd/member/snap/db","took":"1.930509ms"}
{"level":"info","ts":"2024-03-09T06:50:41.050109Z","caller":"etcdserver/server.go:509","msg":"recovered v2 store from snapshot","snapshot-index":1,"snapshot-size":"6.5 kB"}
{"level":"info","ts":"2024-03-09T06:50:41.050275Z","caller":"etcdserver/server.go:522","msg":"recovered v3 backend from snapshot","backend-size-bytes":1437696,"backend-size":"1.4 MB","backend-size-in-use-bytes":1417216,"backend-size-in-use":"1.4 MB"}
{"level":"info","ts":"2024-03-09T06:50:41.0507Z","caller":"etcdserver/raft.go:603","msg":"forcing restart member","cluster-id":"214b29c52e234c97","local-member-id":"f7a7f54728b9f0da","commit-index":1}
{"level":"info","ts":"2024-03-09T06:50:41.050894Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"f7a7f54728b9f0da switched to configuration voters=(17845501734453637338)"}
{"level":"info","ts":"2024-03-09T06:50:41.051009Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"f7a7f54728b9f0da became follower at term 1"}
{"level":"info","ts":"2024-03-09T06:50:41.051127Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"newRaft f7a7f54728b9f0da [peers: [f7a7f54728b9f0da], term: 1, commit: 1, applied: 1, lastindex: 1, lastterm: 1]"}
{"level":"info","ts":"2024-03-09T06:50:41.051295Z","caller":"membership/cluster.go:278","msg":"recovered/added member from store","cluster-id":"214b29c52e234c97","local-member-id":"f7a7f54728b9f0da","recovered-remote-peer-id":"f7a7f54728b9f0da","recovered-remote-peer-urls":["https://[2a05:d01c:2c7:d803:c903:e274:5215:df54]:2380"]}
{"level":"warn","ts":"2024-03-09T06:50:41.056213Z","caller":"auth/store.go:1238","msg":"simple token is not cryptographically signed"}
{"level":"info","ts":"2024-03-09T06:50:41.063281Z","caller":"mvcc/kvstore.go:393","msg":"kvstore restored","current-rev":1120}
{"level":"info","ts":"2024-03-09T06:50:41.07441Z","caller":"etcdserver/quota.go:94","msg":"enabled backend quota with default value","quota-name":"v3-applier","quota-size-bytes":2147483648,"quota-size":"2.1 GB"}
{"level":"info","ts":"2024-03-09T06:50:41.085Z","caller":"etcdserver/corrupt.go:95","msg":"starting initial corruption check","local-member-id":"f7a7f54728b9f0da","timeout":"15s"}
{"level":"info","ts":"2024-03-09T06:50:41.085822Z","caller":"etcdserver/corrupt.go:165","msg":"initial corruption checking passed; no corruption","local-member-id":"f7a7f54728b9f0da"}
{"level":"info","ts":"2024-03-09T06:50:41.08589Z","caller":"etcdserver/server.go:854","msg":"starting etcd server","local-member-id":"f7a7f54728b9f0da","local-server-version":"3.5.9","cluster-version":"to_be_decided"}
{"level":"info","ts":"2024-03-09T06:50:41.086266Z","caller":"etcdserver/server.go:738","msg":"started as single-node; fast-forwarding election ticks","local-member-id":"f7a7f54728b9f0da","forward-ticks":9,"forward-duration":"4.5s","election-ticks":10,"election-timeout":"5s"}
{"level":"info","ts":"2024-03-09T06:50:41.086666Z","caller":"fileutil/purge.go:44","msg":"started to purge file","dir":"/var/lib/rancher/k3s/server/db/etcd/member/snap","suffix":"snap.db","max":5,"interval":"30s"}
{"level":"info","ts":"2024-03-09T06:50:41.086948Z","caller":"fileutil/purge.go:44","msg":"started to purge file","dir":"/var/lib/rancher/k3s/server/db/etcd/member/snap","suffix":"snap","max":5,"interval":"30s"}
{"level":"info","ts":"2024-03-09T06:50:41.087067Z","caller":"fileutil/purge.go:44","msg":"started to purge file","dir":"/var/lib/rancher/k3s/server/db/etcd/member/wal","suffix":"wal","max":5,"interval":"30s"}
{"level":"info","ts":"2024-03-09T06:50:41.089618Z","caller":"embed/etcd.go:726","msg":"starting with client TLS","tls-info":"cert = /var/lib/rancher/k3s/server/tls/etcd/server-client.crt, key = /var/lib/rancher/k3s/server/tls/etcd/server-client.key, client-cert=, client-key=, trusted-ca = /var/lib/rancher/k3s/server/tls/etcd/server-ca.crt, client-cert-auth = true, crl-file = ","cipher-suites":[]}
{"level":"info","ts":"2024-03-09T06:50:41.089818Z","caller":"embed/etcd.go:597","msg":"serving peer traffic","address":"[::1]:2380"}
{"level":"info","ts":"2024-03-09T06:50:41.089956Z","caller":"embed/etcd.go:569","msg":"cmux::serve","address":"[::1]:2380"}
{"level":"info","ts":"2024-03-09T06:50:41.089941Z","caller":"embed/etcd.go:855","msg":"serving metrics","address":"http://[::1]:2381"}
{"level":"info","ts":"2024-03-09T06:50:41.089902Z","caller":"embed/etcd.go:278","msg":"now serving peer/client/metrics","local-member-id":"f7a7f54728b9f0da","initial-advertise-peer-urls":["https://[2a05:d01c:2c7:d803:c903:e274:5215:df54]:2380"],"listen-peer-urls":["https://[::1]:2380"],"advertise-client-urls":["https://[::1]:2379"],"listen-client-urls":["https://[::1]:2379"],"listen-metrics-urls":["http://[::1]:2381"]}
INFO[0000] Server node token is available at /var/lib/rancher/k3s/server/token
INFO[0000] To join server node to cluster: k3s server -s https://[2a05:d01c:2c7:d803:c903:e274:5215:df54]:6443 -t ${SERVER_NODE_TOKEN}
INFO[0000] Agent node token is available at /var/lib/rancher/k3s/server/agent-token
INFO[0000] To join agent node to cluster: k3s agent -s https://[2a05:d01c:2c7:d803:c903:e274:5215:df54]:6443 -t ${AGENT_NODE_TOKEN}
INFO[0000] Wrote kubeconfig /etc/rancher/k3s/k3s.yaml
INFO[0000] Run: k3s kubectl
WARN[0000] remove /var/lib/rancher/k3s/agent/etc/k3s-agent-load-balancer.json: no such file or directory
INFO[0000] Adding server to load balancer k3s-agent-load-balancer: [::1]:6443
INFO[0000] Running load balancer k3s-agent-load-balancer 127.0.0.1:6444 -> [[::1]:6443] [default: [::1]:6443]
INFO[0002] Password verified locally for node test
INFO[0002] certificate CN=test signed by CN=k3s-server-ca@1709966379: notBefore=2024-03-09 06:39:39 +0000 UTC notAfter=2025-03-09 06:50:42 +0000 UTC
INFO[0002] certificate CN=system:node:test,O=system:nodes signed by CN=k3s-client-ca@1709966379: notBefore=2024-03-09 06:39:39 +0000 UTC notAfter=2025-03-09 06:50:43 +0000 UTC
{"level":"info","ts":"2024-03-09T06:50:45.55213Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"f7a7f54728b9f0da is starting a new election at term 1"}
{"level":"info","ts":"2024-03-09T06:50:45.552334Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"f7a7f54728b9f0da became pre-candidate at term 1"}
{"level":"info","ts":"2024-03-09T06:50:45.552439Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"f7a7f54728b9f0da received MsgPreVoteResp from f7a7f54728b9f0da at term 1"}
{"level":"info","ts":"2024-03-09T06:50:45.552503Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"f7a7f54728b9f0da became candidate at term 2"}
{"level":"info","ts":"2024-03-09T06:50:45.552558Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"f7a7f54728b9f0da received MsgVoteResp from f7a7f54728b9f0da at term 2"}
{"level":"info","ts":"2024-03-09T06:50:45.552616Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"f7a7f54728b9f0da became leader at term 2"}
{"level":"info","ts":"2024-03-09T06:50:45.552669Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"raft.node: f7a7f54728b9f0da elected leader f7a7f54728b9f0da at term 2"}
{"level":"info","ts":"2024-03-09T06:50:45.569873Z","caller":"etcdserver/server.go:2573","msg":"setting up initial cluster version using v2 API","cluster-version":"3.5"}
{"level":"info","ts":"2024-03-09T06:50:45.575674Z","caller":"etcdserver/server.go:2064","msg":"published local member to cluster through raft","local-member-id":"f7a7f54728b9f0da","local-member-attributes":"{Name:test-89299818 ClientURLs:[https://[::1]:2379]}","request-path":"/0/members/f7a7f54728b9f0da/attributes","cluster-id":"214b29c52e234c97","publish-timeout":"15s"}
{"level":"info","ts":"2024-03-09T06:50:45.575792Z","caller":"embed/serve.go:103","msg":"ready to serve client requests"}
{"level":"info","ts":"2024-03-09T06:50:45.576187Z","caller":"embed/serve.go:103","msg":"ready to serve client requests"}
{"level":"info","ts":"2024-03-09T06:50:45.577618Z","caller":"embed/serve.go:250","msg":"serving client traffic securely","traffic":"grpc","address":"[::1]:2379"}
{"level":"info","ts":"2024-03-09T06:50:45.579716Z","caller":"embed/serve.go:250","msg":"serving client traffic securely","traffic":"http","address":"[::1]:2382"}
{"level":"info","ts":"2024-03-09T06:50:45.588479Z","caller":"membership/cluster.go:584","msg":"set initial cluster version","cluster-id":"214b29c52e234c97","local-member-id":"f7a7f54728b9f0da","cluster-version":"3.5"}
{"level":"info","ts":"2024-03-09T06:50:45.58863Z","caller":"api/capability.go:75","msg":"enabled capabilities for version","cluster-version":"3.5"}
{"level":"info","ts":"2024-03-09T06:50:45.588771Z","caller":"etcdserver/server.go:2597","msg":"cluster version is updated","cluster-version":"3.5"}
INFO[0004] Defragmenting etcd database
{"level":"info","ts":"2024-03-09T06:50:45.597707Z","caller":"v3rpc/maintenance.go:90","msg":"starting defragment"}
{"level":"info","ts":"2024-03-09T06:50:45.623971Z","caller":"backend/backend.go:497","msg":"defragmenting","path":"/var/lib/rancher/k3s/server/db/etcd/member/snap/db","current-db-size-bytes":1437696,"current-db-size":"1.4 MB","current-db-size-in-use-bytes":1421312,"current-db-size-in-use":"1.4 MB"}
{"level":"info","ts":"2024-03-09T06:50:45.678731Z","caller":"backend/backend.go:549","msg":"finished defragmenting directory","path":"/var/lib/rancher/k3s/server/db/etcd/member/snap/db","current-db-size-bytes-diff":-114688,"current-db-size-bytes":1323008,"current-db-size":"1.3 MB","current-db-size-in-use-bytes-diff":-106496,"current-db-size-in-use-bytes":1314816,"current-db-size-in-use":"1.3 MB","took":"80.941229ms"}
{"level":"info","ts":"2024-03-09T06:50:45.678822Z","caller":"v3rpc/maintenance.go:96","msg":"finished defragment"}
INFO[0004] etcd data store connection OK
INFO[0004] Waiting for API server to become available
INFO[0004] Saving cluster bootstrap data to datastore
INFO[0004] ETCD server is now running
INFO[0004] k3s is up and running
WARN[0004] Bootstrap key already exists
INFO[0008] Defragmenting etcd database
{"level":"info","ts":"2024-03-09T06:50:48.949296Z","caller":"v3rpc/maintenance.go:90","msg":"starting defragment"}
{"level":"info","ts":"2024-03-09T06:50:48.976038Z","caller":"backend/backend.go:497","msg":"defragmenting","path":"/var/lib/rancher/k3s/server/db/etcd/member/snap/db","current-db-size-bytes":1323008,"current-db-size":"1.3 MB","current-db-size-in-use-bytes":1314816,"current-db-size-in-use":"1.3 MB"}
{"level":"info","ts":"2024-03-09T06:50:49.032395Z","caller":"backend/backend.go:549","msg":"finished defragmenting directory","path":"/var/lib/rancher/k3s/server/db/etcd/member/snap/db","current-db-size-bytes-diff":0,"current-db-size-bytes":1323008,"current-db-size":"1.3 MB","current-db-size-in-use-bytes-diff":0,"current-db-size-in-use-bytes":1314816,"current-db-size-in-use":"1.3 MB","took":"82.865858ms"}
{"level":"info","ts":"2024-03-09T06:50:49.032634Z","caller":"v3rpc/maintenance.go:96","msg":"finished defragment"}
INFO[0008] Reconciling bootstrap data between datastore and disk
INFO[0008] Cluster reset: backing up certificates directory to /var/lib/rancher/k3s/server/tls-1709967049
WARN[0008] Updating bootstrap data on disk from datastore
INFO[0008] stopping etcd
{"level":"info","ts":"2024-03-09T06:50:49.087385Z","caller":"embed/etcd.go:376","msg":"closing etcd server","name":"test-89299818","data-dir":"/var/lib/rancher/k3s/server/db/etcd","advertise-peer-urls":["https://[2a05:d01c:2c7:d803:c903:e274:5215:df54]:2380"],"advertise-client-urls":["https://[::1]:2379"]}
{"level":"info","ts":"2024-03-09T06:50:49.088657Z","caller":"etcdserver/server.go:1465","msg":"skipped leadership transfer for single voting member cluster","local-member-id":"f7a7f54728b9f0da","current-leader-member-id":"f7a7f54728b9f0da"}
{"level":"info","ts":"2024-03-09T06:50:49.122426Z","caller":"embed/etcd.go:579","msg":"stopping serving peer traffic","address":"[::1]:2380"}
{"level":"info","ts":"2024-03-09T06:50:49.122692Z","caller":"embed/etcd.go:584","msg":"stopped serving peer traffic","address":"[::1]:2380"}
{"level":"info","ts":"2024-03-09T06:50:49.122807Z","caller":"embed/etcd.go:378","msg":"closed etcd server","name":"test-89299818","data-dir":"/var/lib/rancher/k3s/server/db/etcd","advertise-peer-urls":["https://[2a05:d01c:2c7:d803:c903:e274:5215:df54]:2380"],"advertise-client-urls":["https://[::1]:2379"]}
INFO[0013] Managed etcd cluster membership has been reset, restart without --cluster-reset flag now. Backup and delete ${datadir}/server/db on each peer etcd server and rejoin the nodes
root@test:~#
root@test:~#
root@test:~# systemctl start k3s
root@test:~#
root@test:~#
root@test:~# kubectl get no -o wide
NAME   STATUS   ROLES                       AGE   VERSION        INTERNAL-IP                              EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION    CONTAINER-RUNTIME
test   Ready    control-plane,etcd,master   14m   v1.28.7+k3s1   2a05:d01c:2c7:d803:c903:e274:5215:df54   <none>        Ubuntu 20.04.6 LTS   5.15.0-1055-aws   containerd://1.7.11-k3s2
root@test:~#