k8s: v1.17.17 on rancher
root@10-102-35-35:~# kubectl version
Client Version: version.Info{Major:"1", Minor:"17", GitVersion:"v1.17.17", GitCommit:"f3abc15296f3a3f54e4ee42e830c61047b13895f", GitTreeState:"clean", BuildDate:"2021-01-13T13:21:12Z", GoVersion:"go1.13.15", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"17", GitVersion:"v1.17.17", GitCommit:"f3abc15296f3a3f54e4ee42e830c61047b13895f", GitTreeState:"clean", BuildDate:"2021-01-13T13:13:00Z", GoVersion:"go1.13.15", Compiler:"gc", Platform:"linux/amd64"}
root@10-102-35-35:~# helm list
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
my-release default 3 2021-11-04 11:10:39.708777253 +0800 CST deployed milvus-2.3.1 2.0.0-rc.8
root@10-102-35-35:~# helm version
WARNING: Kubernetes configuration file is group-readable. This is insecure. Location: /root/.kube/config
WARNING: Kubernetes configuration file is world-readable. This is insecure. Location: /root/.kube/config
version.BuildInfo{Version:"v3.7.0", GitCommit:"eeac83883cb4014fe60267ec6373570374ce770b", GitTreeState:"clean", GoVersion:"go1.16.8"}
root@10-102-35-35:~# kubectl get sc
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
local-path (default) rancher.io/local-path Delete WaitForFirstConsumer false 4d16h
root@10-102-35-35:~# kubectl get pods -A
NAMESPACE NAME READY STATUS RESTARTS AGE
cattle-system cattle-cluster-agent-568889bd6c-62flb 1/1 Running 0 4d16h
cattle-system cattle-node-agent-h725h 1/1 Running 0 3d22h
cattle-system cattle-node-agent-xnzwp 1/1 Running 0 4d16h
cattle-system kube-api-auth-wpk9l 1/1 Running 0 4d16h
default my-release-etcd-0 0/1 CrashLoopBackOff 1108 3d22h
default my-release-etcd-1 1/1 Running 0 3d22h
default my-release-etcd-2 1/1 Running 0 3d22h
default my-release-milvus-datacoord-6765bcffdf-hqknk 1/1 Running 0 4d16h
default my-release-milvus-datanode-699f55794d-5skbk 1/1 Running 0 4d16h
default my-release-milvus-indexcoord-f75f84786-8vm44 1/1 Running 0 3d22h
default my-release-milvus-indexnode-7fdb75f647-nhbkd 1/1 Running 0 3d22h
default my-release-milvus-proxy-6d4d594875-crzb6 1/1 Running 0 4d16h
default my-release-milvus-querycoord-55dd8dbfd5-8qjqp 1/1 Running 0 3d22h
default my-release-milvus-querynode-65c577b69-lhlqf 1/1 Running 0 4d16h
default my-release-milvus-rootcoord-59545d48f7-9xmx5 1/1 Running 4 3d22h
default my-release-minio-0 1/1 Running 0 3d22h
default my-release-minio-1 1/1 Running 0 4d16h
default my-release-minio-2 1/1 Running 0 3d22h
default my-release-minio-3 1/1 Running 0 4d16h
default my-release-pulsar-autorecovery-7cbfd6ccc-mcwlf 1/1 Running 0 3d22h
default my-release-pulsar-bastion-85886c49b7-4hp9t 1/1 Running 0 4d16h
default my-release-pulsar-bookkeeper-0 1/1 Running 0 3d22h
default my-release-pulsar-bookkeeper-1 1/1 Running 1 4d16h
default my-release-pulsar-broker-bcf858d9c-blz9c 1/1 Running 6 4d16h
default my-release-pulsar-proxy-678998cb5f-d88dm 2/2 Running 0 4d16h
default my-release-pulsar-zookeeper-0 1/1 Running 0 3d22h
default my-release-pulsar-zookeeper-1 1/1 Running 0 4d16h
default my-release-pulsar-zookeeper-2 1/1 Running 0 3d22h
kube-system coredns-6b84d75d99-94mwh 1/1 Running 0 3d22h
kube-system coredns-6b84d75d99-b2p77 1/1 Running 0 4d16h
kube-system coredns-autoscaler-5c4b6999d9-zwdvm 1/1 Running 0 3d22h
kube-system kube-flannel-6c795 2/2 Running 0 4d16h
kube-system kube-flannel-8jfkx 2/2 Running 0 3d22h
kube-system metrics-server-7579449c57-t6n7l 1/1 Running 0 4d16h
kube-system rke-coredns-addon-deploy-job-hdc62 0/1 Completed 0 3d22h
kube-system rke-metrics-addon-deploy-job-mmkzs 0/1 Completed 0 4d16h
kube-system rke-network-plugin-deploy-job-wc6mc 0/1 Completed 0 4d16h
local-path-storage local-path-provisioner-85cff57c57-2d9mh 1/1 Running 0 3d22h
root@10-102-35-35:~# kubectl describe pod my-release-etcd-0
Name: my-release-etcd-0
Namespace: default
Priority: 0
Node: 10-102-35-36/10.102.35.36
Start Time: Thu, 04 Nov 2021 11:57:34 +0800
Labels: app.kubernetes.io/instance=my-release
app.kubernetes.io/managed-by=Helm
app.kubernetes.io/name=etcd
controller-revision-hash=my-release-etcd-666797d59
helm.sh/chart=etcd-6.3.3
statefulset.kubernetes.io/pod-name=my-release-etcd-0
Annotations: <none>
Status: Running
IP: 10.42.1.4
IPs:
IP: 10.42.1.4
Controlled By: StatefulSet/my-release-etcd
Containers:
etcd:
Container ID: docker://824f1259aec2dc36b0837c2bd52be9944604cde6df0463dabdded445c47b9ef7
Image: docker.io/bitnami/etcd:3.5.0-debian-10-r24
Image ID: docker-pullable://bitnami/etcd@sha256:914039ec8f4ba2c53580195bb21f487a1a86f6c3cd7275a1ec451e03c6c52dd1
Ports: 2379/TCP, 2380/TCP
Host Ports: 0/TCP, 0/TCP
State: Waiting
Reason: CrashLoopBackOff
Last State: Terminated
Reason: Error
Exit Code: 1
Started: Mon, 08 Nov 2021 09:56:43 +0800
Finished: Mon, 08 Nov 2021 09:56:43 +0800
Ready: False
Restart Count: 1107
Liveness: exec [/opt/bitnami/scripts/etcd/healthcheck.sh] delay=60s timeout=5s period=30s #success=1 #failure=5
Readiness: exec [/opt/bitnami/scripts/etcd/healthcheck.sh] delay=60s timeout=5s period=10s #success=1 #failure=5
Environment:
BITNAMI_DEBUG: false
MY_POD_IP: (v1:status.podIP)
MY_POD_NAME: my-release-etcd-0 (v1:metadata.name)
ETCDCTL_API: 3
ETCD_ON_K8S: yes
ETCD_START_FROM_SNAPSHOT: no
ETCD_DISASTER_RECOVERY: no
ETCD_NAME: $(MY_POD_NAME)
ETCD_DATA_DIR: /bitnami/etcd/data
ETCD_LOG_LEVEL: info
ALLOW_NONE_AUTHENTICATION: yes
ETCD_ADVERTISE_CLIENT_URLS: http://$(MY_POD_NAME).my-release-etcd-headless.default.svc.cluster.local:2379
ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379
ETCD_INITIAL_ADVERTISE_PEER_URLS: http://$(MY_POD_NAME).my-release-etcd-headless.default.svc.cluster.local:2380
ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380
ETCD_AUTO_COMPACTION_MODE: revision
ETCD_AUTO_COMPACTION_RETENTION: 1000
ETCD_INITIAL_CLUSTER_TOKEN: etcd-cluster-k8s
ETCD_INITIAL_CLUSTER_STATE: existing
ETCD_INITIAL_CLUSTER: my-release-etcd-0=http://my-release-etcd-0.my-release-etcd-headless.default.svc.cluster.local:2380,my-release-etcd-1=http://my-release-etcd-1.my-release-etcd-headless.default.svc.cluster.local:2380,my-release-etcd-2=http://my-release-etcd-2.my-release-etcd-headless.default.svc.cluster.local:2380
ETCD_CLUSTER_DOMAIN: my-release-etcd-headless.default.svc.cluster.local
ETCD_QUOTA_BACKEND_BYTES: 4294967296
Mounts:
/bitnami/etcd from data (rw)
/var/run/secrets/kubernetes.io/serviceaccount from default-token-tfhpq (ro)
Conditions:
Type Status
Initialized True
Ready False
ContainersReady False
PodScheduled True
Volumes:
data:
Type: PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
ClaimName: data-my-release-etcd-0
ReadOnly: false
default-token-tfhpq:
Type: Secret (a volume populated by a Secret)
SecretName: default-token-tfhpq
Optional: false
QoS Class: BestEffort
Node-Selectors: <none>
Tolerations: node.kubernetes.io/not-ready:NoExecute for 300s
node.kubernetes.io/unreachable:NoExecute for 300s
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Warning BackOff 115s (x26604 over 3d22h) kubelet, 10-102-35-36 Back-off restarting failed container
root@10-102-35-35:~# kubectl logs my-release-etcd-0
etcd 01:56:43.05
etcd 01:56:43.06 Welcome to the Bitnami etcd container
etcd 01:56:43.06 Subscribe to project updates by watching https://github.com/bitnami/bitnami-docker-etcd
etcd 01:56:43.06 Submit issues and feature requests at https://github.com/bitnami/bitnami-docker-etcd/issues
etcd 01:56:43.07
etcd 01:56:43.07 INFO ==> ** Starting etcd setup **
etcd 01:56:43.09 INFO ==> Validating settings in ETCD_* env vars..
etcd 01:56:43.10 WARN ==> You set the environment variable ALLOW_NONE_AUTHENTICATION=yes. For safety reasons, do not use this flag in a production environment.
etcd 01:56:43.10 INFO ==> Initializing etcd
etcd 01:56:43.12 INFO ==> Detected data from previous deployments
etcd 01:56:43.26 INFO ==> Updating member in existing cluster
{"level":"warn","ts":"2021-11-08T01:56:43.331Z","logger":"etcd-client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000230000/#initially=[my-release-etcd-2.my-release-etcd-headless.default.svc.cluster.local:2379;my-release-etcd-1.my-release-etcd-headless.default.svc.cluster.local:2379]","attempt":0,"error":"rpc error: code = NotFound desc = etcdserver: member not found"}
Error: etcdserver: member not found
There is a pointer memory address like string in the url part, I think it a bug related to golang.