guettli@p15$ k get events -A --sort-by=.metadata.creationTimestamp 

default     8m14s       Warning   ReconcileError                machinehealthcheck/my-cluster-control-plane-unhealthy-5m   error creating client and cache for remote cluster: error fetching REST client config for remote cluster "default/my-cluster": failed to retrieve kubeconfig secret for Cluster default/my-cluster: secrets "my-cluster-kubeconfig" not found
default     8m14s       Warning   ReconcileError                machinehealthcheck/my-cluster-md-0-unhealthy-5m            error creating client and cache for remote cluster: error fetching REST client config for remote cluster "default/my-cluster": failed to retrieve kubeconfig secret for Cluster default/my-cluster: secrets "my-cluster-kubeconfig" not found
default     8m35s       Warning   ReconcileError                machinedeployment/my-cluster-md-0                          failed to retrieve HCloudMachineTemplate external object "default"/"my-cluster-md-0": hcloudmachinetemplates.infrastructure.cluster.x-k8s.io "my-cluster-md-0" not found
default     8m35s       Normal    SuccessfulCreate              machinedeployment/my-cluster-md-0                          Created MachineSet "my-cluster-md-0-7465476f6d"
default     8m4s        Normal    ChangeLoadBalancerAlgorithm   hetznercluster/my-cluster                                  Changed load balancer algorithm
default     8m4s        Normal    CreateLoadBalancer            hetznercluster/my-cluster                                  Created load balancer
default     8m          Normal    SuccessfulCreate              hcloudmachine/my-cluster-control-plane-72v5w               Created new server with id 20435964
default     7m56s       Warning   ReconcileError                machinehealthcheck/my-cluster-control-plane-unhealthy-5m   error creating client and cache for remote cluster: error creating dynamic rest mapper for remote cluster "default/my-cluster": Get "https://142.132.242.98:443/api?timeout=10s": dial tcp 142.132.242.98:443: connect: connection refused
default     7m45s       Normal    AddedAsTargetToLoadBalancer   hetznercluster/my-cluster                                  Added new server with id 20435964 to the loadbalancer 719385
default     6m52s       Warning   ReconcileError                machinehealthcheck/my-cluster-control-plane-unhealthy-5m   error creating client and cache for remote cluster: error creating dynamic rest mapper for remote cluster "default/my-cluster": context deadline exceeded
default     6m31s       Warning   ReconcileError                machinehealthcheck/my-cluster-md-0-unhealthy-5m            error creating client and cache for remote cluster: error creating dynamic rest mapper for remote cluster "default/my-cluster": context deadline exceeded
default     66s         Normal    DetectedUnhealthy             machine/my-cluster-control-plane-rb8qb                     Machine default/my-cluster-control-plane-unhealthy-5m/my-cluster-control-plane-rb8qb/ has unhealthy node

kubectl create secret generic my-cluster-kubeconfig --from-literal=hcloud=$HCLOUD_TOKEN
kubectl patch secret my-cluster-kubeconfig -p '{"metadata":{"labels":{"clusterctl.cluster.x-k8s.io/move":""}}}'

default     29s         Warning   ReconcileError                machinehealthcheck/my-cluster-md-0-unhealthy-5m            error creating client and cache for remote cluster: error fetching REST client config for remote cluster "default/my-cluster": failed to retrieve kubeconfig secret for Cluster default/my-cluster: secrets "my-cluster-kubeconfig" not found
default     29s         Warning   ReconcileError                machinehealthcheck/my-cluster-control-plane-unhealthy-5m   error creating client and cache for remote cluster: error fetching REST client config for remote cluster "default/my-cluster": failed to retrieve kubeconfig secret for Cluster default/my-cluster: secrets "my-cluster-kubeconfig" not found

[...]
spec:
  controlPlaneEndpoint:
    host: ""
    port: 6443
[...]

kubectl --context kind-test get secret -l cluster.x-k8s.io/cluster-name=test -o json | jq -r '.items[] | select(.metadata.name | endswith("bootstrap-data")) | [.metadata.name, .data.value] | join(" ")' | while read name data; do echo -n "$name: "; base64 -d <<<$data | yq -r .cluster.controlPlane.endpoint; done
test-controlplane-6v8gn-bootstrap-data: https://XXX.XXX.XXX.XXX:6443
test-controlplane-qw2h6-bootstrap-data: https://XXX.XXX.XXX.XXX:6443
test-controlplane-tb9dz-bootstrap-data: https://XXX.XXX.XXX.XXX:6443
test-md-0-5f6c9b9447-gj6gt-bootstrap-data: https://:6443

$ clusterctl describe cluster g-work
NAME                                                                       READY  SEVERITY  REASON                       SINCE  MESSAGE                                                       
Cluster/g-work                                                             True                                          51m                                                                   
├─ClusterInfrastructure - HetznerCluster/g-work                                                                                                                                                
├─ControlPlane - KubeadmControlPlane/g-work-control-plane                  True                                          51m                                                                   
│ └─Machine/g-work-control-plane-4gvd4                                     True                                          52m                                                                   
│   └─MachineInfrastructure - HCloudMachine/g-work-control-plane-4fxsc                                                                                                                         
└─Workers                                                                                                                                                                                      
  ├─MachineDeployment/g-work-md-0                                          True                                          54m                                                                   
  └─MachineDeployment/g-work-md-1                                          False  Warning   WaitingForAvailableMachines  54m    Minimum availability requires 1 replicas, current 0 available  
    └─Machine/g-work-md-1-5bd54bfd6c-tzxql                                 True                                          45m                                                                   
      └─MachineInfrastructure - HetznerBareMetalMachine/g-work-md-1-gh6ks                                                                             

$ kubectl describe MachineHealthCheck g-work-md-1-unhealthy-5m
...
Status:
  Conditions:
    Last Transition Time:  2022-11-03T07:49:57Z
    Status:                True
    Type:                  RemediationAllowed
  Expected Machines:       1
  Observed Generation:     1
  Targets:
    g-work-md-1-5bd54bfd6c-tzxql
Events:
  Type     Reason          Age                 From                           Message
  ----     ------          ----                ----                           -------
  Warning  ReconcileError  60m (x16 over 60m)  machinehealthcheck-controller  error creating client and cache for remote cluster: error fetching REST client config for remote cluster "default/g-work": failed to retrieve kubeconfig secret for Cluster default/g-work: secrets "g-work-kubeconfig" not found
  Warning  ReconcileError  59m (x2 over 59m)   machinehealthcheck-controller  error creating client and cache for remote cluster: error creating dynamic rest mapper for remote cluster "default/g-work": client rate limiter Wait returned an error: context deadline exceeded - error from a previous attempt: EOF

$ kubectl describe secrets g-work-kubeconfig
Name:         g-work-kubeconfig
Namespace:    default
Labels:       caph.environment=owned
              cluster.x-k8s.io/cluster-name=g-work
Annotations:  <none>

Type:  cluster.x-k8s.io/secret

Data
====
value:  5535 bytes

$ kubectl describe hetznerbaremetalmachines g-work-md-1-gh6ks
Name:         g-work-md-1-gh6ks
Namespace:    default
Labels:       cluster.x-k8s.io/cluster-name=g-work
              cluster.x-k8s.io/deployment-name=g-work-md-1
              machine-template-hash=1681069827
              nodepool=g-work-md-1
Annotations:  cluster.x-k8s.io/cloned-from-groupkind: HetznerBareMetalMachineTemplate.infrastructure.cluster.x-k8s.io
              cluster.x-k8s.io/cloned-from-name: g-work-md-1
              infrastructure.cluster.x-k8s.io/HetznerBareMetalHost: default/bm-0
API Version:  infrastructure.cluster.x-k8s.io/v1beta1
Kind:         HetznerBareMetalMachine
...
Status:
  Addresses:
    Address:  46.4.66.173/26
    Type:     InternalIP
    Address:  2a01:4f8:140:2484::2/64
    Type:     InternalIP
    Address:  g-work-md-1-gh6ks
    Type:     Hostname
    Address:  g-work-md-1-gh6ks
    Type:     InternalDNS
  Conditions:
    Last Transition Time:  2022-11-03T07:49:58Z
    Status:                True
    Type:                  AssociateBMHCondition
    Last Transition Time:  2022-11-03T07:49:58Z
    Status:                True
    Type:                  InstanceBootstrapReady
    Last Transition Time:  2022-11-03T07:55:34Z
    Status:                True
    Type:                  InstanceReady
  Last Updated:            2022-11-03T07:51:34Z
  Ready:                   true
Events:                    <none>

---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: HetznerBareMetalMachineTemplate
metadata:
  name: g-work-md-1
  namespace: default
spec:
  template:
    spec:
      installImage:
        image:
          path: /root/images/Ubuntu-2204-jammy-amd64-base.tar.gz
          # path: /root/.oldroot/nfs/install/../images/Ubuntu-2004-focal-64-minimal-hwe.tar.gz
        partitions:
        - fileSystem: ext4
          mount: /boot
          size: 1024M
        - fileSystem: ext4
          mount: /
          size: all
        postInstallScript: |
          #!/bin/bash
          apt-get update && apt-get install -y cloud-init apparmor apparmor-utils
      hostSelector:
        matchExpressions:
        - key: "geneea/name"
          operator: "in"
          values:
          - "sigma"
          - "theta"
      sshSpec:
        portAfterCloudInit: 22
        portAfterInstallImage: 22
        secretRef:
          key:
            name: sshkey-name
            privateKey: ssh-privatekey
            publicKey: ssh-publickey
          name: robot-ssh
---
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
kind: HetznerBareMetalHost
metadata:
  name: "bm-0"
  labels:
    geneea/name: "sigma"
spec:
  serverID: 1702753
  rootDeviceHints:
    raid:
      wwn:
        - "0x500a07511320714e"
        - "0x5002538d41d5de7c"
  maintenanceMode: false
  description: "sigma"

clusterctl generate provider --infrastructure hetzner:v1.0.0-alpha.20 > hetzner-capi.yml
kubectl apply -f hetzner-capi.yml

export HCLOUD_TOKEN=xxxxxxxxxxxx
export HCLOUD_SSH_KEY="mykey"
export CLUSTER_NAME="mycluster"
export HCLOUD_REGION="fsn1"
export CONTROL_PLANE_MACHINE_COUNT=3
export WORKER_MACHINE_COUNT=3
export KUBERNETES_VERSION=1.24.1
export HCLOUD_CONTROL_PLANE_MACHINE_TYPE=cpx31
export HCLOUD_WORKER_MACHINE_TYPE=cpx31

kubectl create secret generic hetzner --from-literal=hcloud=${HCLOUD_TOKEN}
kubectl patch secret hetzner-p '{"metadata":{"labels":{"clusterctl.cluster.x-k8s.io/move":""}}}'
clusterctl generate cluster --infrastructure hetzner:v1.0.0-alpha.20 ${CLUSTER_NAME} > ${CLUSTER_NAME}.yaml
kubectl apply -f ${CLUSTER_NAME}.yaml

$ clusterctl describe cluster ${CLUSTER_NAME}
NAME                                                                       READY  SEVERITY  REASON                       SINCE  MESSAGE                                                                  
Cluster/mycluster                                                          False  Warning   ScalingUp                    71m    Scaling up control plane to 3 replicas (actual 1)                        
├─ClusterInfrastructure - HetznerCluster/mycluster                                                                                                                                                       
├─ControlPlane - KubeadmControlPlane/mycluster-control-plane               False  Warning   ScalingUp                    71m    Scaling up control plane to 3 replicas (actual 1)                        
│ └─Machine/mycluster-control-plane-x5jb5                                  False  Warning   NodeStartupTimeout           49m    Node failed to report startup in &Duration{Duration:20m0s,}              
│   └─MachineInfrastructure - HCloudMachine/mycluster-control-plane-prxwq                                                                                                                                
└─Workers                                                                                                                                                                                                
  └─MachineDeployment/mycluster-md-0                                       False  Warning   WaitingForAvailableMachines  72m    Minimum availability requires 3 replicas, current 0 available            
    └─3 Machines...                                                        True                                          7m53s  See mycluster-md-0-59f5696b48-khjkp, mycluster-md-0-59f5696b48-v57kg, ...

$ kubectl get KubeadmControlPlane
NAME                          CLUSTER         INITIALIZED   API SERVER AVAILABLE   REPLICAS   READY   UPDATED   UNAVAILABLE   AGE   VERSION
mycluster-control-plane       mycluster       true                                 1                  1         1             73m   v1.24.1

$ kubectl describe KubeadmControlPlane mycluster-control-plane
.....
Events:
  Type     Reason                 Age                    From                              Message
  ----     ------                 ----                   ----                              -------
  Warning  ControlPlaneUnhealthy  3m28s (x285 over 73m)  kubeadm-control-plane-controller  Waiting for control plane to pass preflight checks to continue reconciliation: [machine mycluster-control-plane-x5jb5 does not have APIServerPodHealthy condition, machine mycluster-control-plane-x5jb5 does not have ControllerManagerPodHealthy condition, machine mycluster-control-plane-x5jb5 does not have SchedulerPodHealthy condition, machine mycluster-control-plane-x5jb5 does not have EtcdPodHealthy condition, machine mycluster-control-plane-x5jb5 does not have EtcdMemberHealthy condition]

$ kubectl get MachineHealthCheck
NAME                                       CLUSTER         EXPECTEDMACHINES   MAXUNHEALTHY   CURRENTHEALTHY   AGE
mycluster-control-plane-unhealthy-5m       mycluster       1                  100%                            74m
mycluster-md-0-unhealthy-5m                mycluster       3                  100%                            74m

$ kubectl describe MachineHealthCheck mycluster-control-plane-unhealthy-5m
Events:
  Type     Reason          Age                 From                           Message
  ----     ------          ----                ----                           -------
  Warning  ReconcileError  75m (x13 over 75m)  machinehealthcheck-controller  error creating client and cache for remote cluster: error fetching REST client config for remote cluster "default/mycluster": failed to retrieve kubeconfig secret for Cluster default/mycluster: secrets "mycluster-kubeconfig" not found
  Warning  ReconcileError  74m                 machinehealthcheck-controller  error creating client and cache for remote cluster: error creating dynamic rest mapper for remote cluster "default/mycluster": Get "https://142.132.240.114:443/api?timeout=10s": dial tcp 142.132.240.114:443: i/o timeout
  Warning  ReconcileError  73m (x4 over 74m)   machinehealthcheck-controller  error creating client and cache for remote cluster: error creating dynamic rest mapper for remote cluster "default/mycluster": context deadline exceeded

$ kubectl get secrets
NAME                                TYPE                                  DATA   AGE
default-token-vcccj                 kubernetes.io/service-account-token   3      5h35m
hetzner                             Opaque                                1      78m
mycluster-ca                        cluster.x-k8s.io/secret               2      76m
mycluster-control-plane-xkvnf       cluster.x-k8s.io/secret               2      76m
mycluster-etcd                      cluster.x-k8s.io/secret               2      76m
mycluster-kubeconfig                cluster.x-k8s.io/secret               1      76m
mycluster-md-0-48wdl                cluster.x-k8s.io/secret               2      12m
mycluster-md-0-rlccb                cluster.x-k8s.io/secret               2      13m
mycluster-md-0-x2q8s                cluster.x-k8s.io/secret               2      12m
mycluster-proxy                     cluster.x-k8s.io/secret               2      76m
mycluster-sa                        cluster.x-k8s.io/secret               2      76m

# Get the node status of the deployed cluster
$ kubectl get no --kubeconfig mycluster
NAME                            STATUS   ROLES              AGE    VERSION
mycluster-control-plane-x5jb5   NotReady    control-plane   74m    v1.24.1
mycluster-md-0-khjkp            NotReady    <none>          72m    v1.24.1
mycluster-md-0-v57kg            NotReady    <none>          72m    v1.24.1
mycluster-md-0-x5d32            NotReady    <none>          72m    v1.24.1

# Try to fetch API - possible
$ curl https://142.132.240.114:443/api?timeout=10s -k
{
  "kind": "Status",
  "apiVersion": "v1",
  "metadata": {},
  "status": "Failure",
  "message": "forbidden: User \"system:anonymous\" cannot get path \"/api\"",
  "reason": "Forbidden",
  "details": {},
  "code": 403
}

Client Version: version.Info{Major:"1", Minor:"22", GitVersion:"v1.22.4", GitCommit:"b695d79d4f967c403a96986f1750a35eb75e75f1", GitTreeState:"clean", BuildDate:"2021-11-17T15:48:33Z", GoVersion:"go1.16.10", Compiler:"gc", Platform:"darwin/amd64"}
Server Version: version.Info{Major:"1", Minor:"22", GitVersion:"v1.22.6", GitCommit:"42a9a90338d705a1650fb68b7891f84b62adb0b0", GitTreeState:"clean", BuildDate:"2022-06-15T04:25:21Z", GoVersion:"go1.16.12", Compiler:"gc", Platform:"linux/amd64"}

{"level":"ERROR","time":"2022-02-22T10:12:50.710Z","logger":"controller.hcloudmachine","file":"controller/controller.go:317","message":"Reconciler error","reconciler group":"infrastructure.cluster.x-k8s.io","reconciler kind":"HCloudMachine","name":"test-control-plane-b8a01-f65ww","namespace":"cluster","error":"failed to reconcile server for HCloudMachine cluster/test-control-plane-b8a01-f65ww: failed to create server: error while creating HCloud server test-control-plane-b8a01-f65ww: error during placement (resource_unavailable)","errorVerbose":"error while creating HCloud server test-control-plane-b8a01-f65ww: error during placement (resource_unavailable)\nfailed to create server\ngithub.com/syself/cluster-api-provider-hetzner/pkg/services/hcloud/server.(Service).Reconcile\n\t/workspace/pkg/services/hcloud/server/server.go:78\ngithub.com/syself/cluster-api-provider-hetzner/controllers.(HCloudMachineReconciler).reconcileNormal\n\t/workspace/controllers/hcloudmachine_controller.go:180\ngithub.com/syself/cluster-api-provider-hetzner/controllers.(HCloudMachineReconciler).Reconcile\n\t/workspace/controllers/hcloudmachine_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:227\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1581\nfailed to reconcile server for HCloudMachine cluster/test-control-plane-b8a01-f65ww\ngithub.com/syself/cluster-api-provider-hetzner/controllers.(HCloudMachineReconciler).reconcileNormal\n\t/workspace/controllers/hcloudmachine_controller.go:181\ngithub.com/syself/cluster-api-provider-hetzner/controllers.(HCloudMachineReconciler).Reconcile\n\t/workspace/controllers/hcloudmachine_controller.go:141\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).Reconcile\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:114\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:311\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:227\nruntime.goexit\n\t/usr/local/go/src/runtime/asm_amd64.s:1581","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).reconcileHandler\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:317\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).processNextWorkItem\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:266\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(Controller).Start.func2.2\n\t/go/pkg/mod/sigs.k8s.io/[email protected]/pkg/internal/controller/controller.go:227"}