etcd snapshot controller thrashes on etcdsnapshotfile management when server is run with `--disable-agent` about k3s HOT 1 CLOSED

$ cat /etc/os-release
PRETTY_NAME="Ubuntu 22.04.2 LTS"

$ uname -m
x86_64

HA: 3 server/ 1 agent

token: xxxx
cluster-init: true
disable-agent: true
write-kubeconfig-mode: "0644"
node-external-ip: 1.1.1.1
node-label:
- k3s-upgrade=server
etcd-snapshot-retention: 2
etcd-snapshot-schedule-cron: "* * * * *"
etcd-s3: true
etcd-s3-access-key: xxxx
etcd-s3-secret-key: xxxx
etcd-s3-bucket: xxxx
etcd-s3-folder: xxxx
etcd-s3-region: xxxx

$ sudo mkdir -p /etc/rancher/k3s && sudo cp config.yaml /etc/rancher/k3s

curl -sfL https://get.k3s.io | sudo INSTALL_K3S_COMMIT='3f906bee79cb32072bea061cbbf4e38142343c66' sh -s - server

$ sudo journalctl -xeu k3s | grep 'Failed to reconcile snapshot ConfigMap: no nodes have reconciled ETCDSnapshotFile resources, requeuing'
$ sudo journalctl -xeu k3s | grep error | grep snapshot

$ k3s -v
k3s version v1.29.3+k3s1 (8aecc26b)
go version go1.21.8

 $ sudo journalctl -xeu k3s | grep 'Failed to reconcile snapshot ConfigMap: no nodes have reconciled ETCDSnapshotFile resources, requeuing' 
Apr 11 21:56:25 ip-172-31-16-97 k3s[8451]: time="2024-04-11T21:56:25Z" level=debug msg="Failed to reconcile snapshot ConfigMap: no nodes have reconciled ETCDSnapshotFile resources, requeuing"
Apr 11 21:56:56 ip-172-31-16-97 k3s[8451]: time="2024-04-11T21:56:56Z" level=debug msg="Failed to reconcile snapshot ConfigMap: no nodes have reconciled ETCDSnapshotFile resources, requeuing"
Apr 11 21:57:28 ip-172-31-16-97 k3s[8451]: time="2024-04-11T21:57:28Z" level=debug msg="Failed to reconcile snapshot ConfigMap: no nodes have reconciled ETCDSnapshotFile resources, requeuing"

 $ sudo journalctl -xeu k3s | grep error | grep snapshot 
Apr 11 21:56:04 ip-172-31-16-97 k3s[8451]: time="2024-04-11T21:56:04Z" level=error msg="error syncing 'local-etcd-snapshot-ip-172-31-16-97-1712872505-428a6c': handler managed-etcd-snapshots-controller: Operation cannot be fulfilled on etcdsnapshotfiles.k3s.cattle.io \"local-etcd-snapshot-ip-172-31-16-97-1712872505-428a6c\": StorageError: invalid object, Code: 4, Key: /registry/k3s.cattle.io/etcdsnapshotfiles/local-etcd-snapshot-ip-172-31-16-97-1712872505-428a6c, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: be05a51b-714d-4a39-a8e6-571917f60f8f, UID in object meta: , requeuing"
Apr 11 21:56:04 ip-172-31-16-97 k3s[8451]: time="2024-04-11T21:56:04Z" level=error msg="Failed to record snapshots for cluster: nodes \"ip-172-31-16-97\" not found"
Apr 11 21:56:06 ip-172-31-16-97 k3s[8451]: time="2024-04-11T21:56:06Z" level=error msg="error syncing 'local-etcd-snapshot-ip-172-31-16-97-1712872562-4d0b13': handler managed-etcd-snapshots-controller: Operation cannot be fulfilled on etcdsnapshotfiles.k3s.cattle.io \"local-etcd-snapshot-ip-172-31-16-97-1712872562-4d0b13\": StorageError: invalid object, Code: 4, Key: /registry/k3s.cattle.io/etcdsnapshotfiles/local-etcd-snapshot-ip-172-31-16-97-1712872562-4d0b13, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: 8e45277b-8e84-4e8b-ac44-5984e7d340a9, UID in object meta: , requeuing"
Apr 11 21:56:07 ip-172-31-16-97 k3s[8451]: time="2024-04-11T21:56:07Z" level=error msg="error syncing 'local-on-demand-ip-172-31-16-97-1712872526-9574ea': handler managed-etcd-snapshots-controller: Operation cannot be fulfilled on etcdsnapshotfiles.k3s.cattle.io \"local-on-demand-ip-172-31-16-97-1712872526-9574ea\": StorageError: invalid object, Code: 4, Key: /registry/k3s.cattle.io/etcdsnapshotfiles/local-on-demand-ip-172-31-16-97-1712872526-9574ea, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: 66960485-96e8-4068-ad73-295a95a48b8d, UID in object meta: , requeuing"
Apr 11 21:56:07 ip-172-31-16-97 k3s[8451]: time="2024-04-11T21:56:07Z" level=error msg="error syncing 'local-on-demand-ip-172-31-16-97-1712872518-9651cf': handler managed-etcd-snapshots-controller: Operation cannot be fulfilled on etcdsnapshotfiles.k3s.cattle.io \"local-on-demand-ip-172-31-16-97-1712872518-9651cf\": StorageError: invalid object, Code: 4, Key: /registry/k3s.cattle.io/etcdsnapshotfiles/local-on-demand-ip-172-31-16-97-1712872518-9651cf, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: 85ce3d94-aae9-400d-9598-3a1dad6f2c23, UID in object meta: , requeuing"
Apr 11 21:56:08 ip-172-31-16-97 k3s[8451]: time="2024-04-11T21:56:08Z" level=error msg="error syncing 'local-on-demand-ip-172-31-16-97-1712872534-fe3ecf': handler managed-etcd-snapshots-controller: Operation cannot be fulfilled on etcdsnapshotfiles.k3s.cattle.io \"local-on-demand-ip-172-31-16-97-1712872534-fe3ecf\": StorageError: invalid object, Code: 4, Key: /registry/k3s.cattle.io/etcdsnapshotfiles/local-on-demand-ip-172-31-16-97-1712872534-fe3ecf, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: b5a8ef96-195f-44fd-9c22-7ac9d5624e6a, UID in object meta: , requeuing"
Apr 11 21:56:08 ip-172-31-16-97 k3s[8451]: time="2024-04-11T21:56:08Z" level=error msg="error syncing 'local-etcd-snapshot-ip-172-31-16-97-1712872505-428a6c': handler managed-etcd-snapshots-controller: Operation cannot be fulfilled on etcdsnapshotfiles.k3s.cattle.io \"local-etcd-snapshot-ip-172-31-16-97-1712872505-428a6c\": StorageError: invalid object, Code: 4, Key: /registry/k3s.cattle.io/etcdsnapshotfiles/local-etcd-snapshot-ip-172-31-16-97-1712872505-428a6c, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: d8816b7c-55bf-44d1-a6bd-7d2738a635d7, UID in object meta: , requeuing"
Apr 11 21:56:09 ip-172-31-16-97 k3s[8451]: time="2024-04-11T21:56:09Z" level=error msg="error syncing 'local-on-demand-ip-172-31-16-97-1712872526-9574ea': handler managed-etcd-snapshots-controller: Operation cannot be fulfilled on etcdsnapshotfiles.k3s.cattle.io \"local-on-demand-ip-172-31-16-97-1712872526-9574ea\": StorageError: invalid object, Code: 4, Key: /registry/k3s.cattle.io/etcdsnapshotfiles/local-on-demand-ip-172-31-16-97-1712872526-9574ea, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: af610bcb-f848-4552-b4d3-f769871b5648, UID in object meta: , requeuing"
Apr 11 21:56:10 ip-172-31-16-97 k3s[8451]: time="2024-04-11T21:56:10Z" level=error msg="error syncing 'local-on-demand-ip-172-31-16-97-1712872518-9651cf': handler managed-etcd-snapshots-controller: Operation cannot be fulfilled on etcdsnapshotfiles.k3s.cattle.io \"local-on-demand-ip-172-31-16-97-1712872518-9651cf\": StorageError: invalid object, Code: 4, Key: /registry/k3s.cattle.io/etcdsnapshotfiles/local-on-demand-ip-172-31-16-97-1712872518-9651cf, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: 258dd772-7245-4fbf-ab23-faf18ca557c8, UID in object meta: , requeuing"
Apr 11 21:56:10 ip-172-31-16-97 k3s[8451]: time="2024-04-11T21:56:10Z" level=error msg="Failed to take scheduled snapshot: nodes \"ip-172-31-16-97\" not found"

$ k3s -v
k3s version v1.29.3+k3s-3f906bee (3f906bee)
go version go1.21.8

 $ sudo journalctl -xeu k3s | grep 'Failed to reconcile snapshot ConfigMap: no nodes have reconciled ETCDSnapshotFile resources, requeuing' 

 $ sudo journalctl -xeu k3s | grep error | grep snapshot 
Apr 11 22:21:27 ip-172-31-18-168 k3s[24196]: time="2024-04-11T22:21:27Z" level=debug msg="Error encountered attempting to retrieve extra metadata from k3s-etcd-snapshot-extra-metadata ConfigMap, error: configmaps \"k3s-etcd-snapshot-extra-metadata\" not found"