--- apiVersion: v1 items: - apiVersion: v1 kind: Pod metadata: annotations: k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.128.0.41/23"],"mac_address":"0a:58:0a:80:00:29","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/16","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.41/23","gateway_ip":"10.128.0.1","role":"primary"}}' k8s.v1.cni.cncf.io/network-status: |- [{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.41" ], "mac": "0a:58:0a:80:00:29", "default": true, "dns": {} }] creationTimestamp: "2025-12-03T21:50:33Z" labels: app: installer managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:labels: .: {} f:app: {} f:ownerReferences: .: {} k:{"uid":"cb97f9f0-ad46-4d93-bb6b-22ff4a4d93d6"}: {} f:spec: f:automountServiceAccountToken: {} f:containers: k:{"name":"installer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"NODE_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:limits: .: {} f:cpu: {} f:memory: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:runAsUser: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/lock"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}: .: {} f:mountPath: {} f:name: {} f:readOnly: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:nodeName: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: .: {} f:runAsUser: {} f:serviceAccount: {} f:serviceAccountName: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"kube-api-access"}: .: {} f:name: {} f:projected: .: {} f:defaultMode: {} f:sources: {} k:{"name":"kubelet-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"var-lock"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: cluster-kube-apiserver-operator operation: Update time: "2025-12-03T21:50:33Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:k8s.ovn.org/pod-networks: {} manager: master-0 operation: Update subresource: status time: "2025-12-03T21:50:33Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: f:k8s.v1.cni.cncf.io/network-status: {} manager: multus-daemon operation: Update subresource: status time: "2025-12-03T21:50:39Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:phase: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2025-12-03T21:54:13Z" name: installer-1-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 kind: ConfigMap name: revision-status-1 uid: cb97f9f0-ad46-4d93-bb6b-22ff4a4d93d6 resourceVersion: "13879" uid: 4c2364d3-47b2-4784-9c42-76bf2547b797 spec: automountServiceAccountToken: false containers: - args: - -v=2 - --revision=1 - --namespace=openshift-kube-apiserver - --pod=kube-apiserver-pod - --resource-dir=/etc/kubernetes/static-pod-resources - --pod-manifest-dir=/etc/kubernetes/manifests - --configmaps=kube-apiserver-pod - --configmaps=config - --configmaps=kube-apiserver-cert-syncer-kubeconfig - --optional-configmaps=oauth-metadata - --optional-configmaps=cloud-config - --configmaps=bound-sa-token-signing-certs - --configmaps=etcd-serving-ca - --optional-configmaps=kube-apiserver-server-ca - --configmaps=kubelet-serving-ca - --configmaps=sa-token-signing-certs - --configmaps=kube-apiserver-audit-policies - --secrets=etcd-client - --optional-secrets=encryption-config - --secrets=localhost-recovery-serving-certkey - --secrets=localhost-recovery-client-token - --optional-secrets=webhook-authenticator - --cert-dir=/etc/kubernetes/static-pod-resources/kube-apiserver-certs - --cert-configmaps=aggregator-client-ca - --cert-configmaps=client-ca - --optional-cert-configmaps=trusted-ca-bundle - --cert-configmaps=control-plane-node-kubeconfig - --cert-configmaps=check-endpoints-kubeconfig - --cert-secrets=aggregator-client - --cert-secrets=localhost-serving-cert-certkey - --cert-secrets=service-network-serving-certkey - --cert-secrets=external-loadbalancer-serving-certkey - --cert-secrets=internal-loadbalancer-serving-certkey - --cert-secrets=bound-service-account-signing-key - --cert-secrets=control-plane-node-admin-client-cert-key - --cert-secrets=check-endpoints-client-cert-key - --cert-secrets=kubelet-client - --cert-secrets=node-kubeconfigs - --optional-cert-secrets=user-serving-cert - --optional-cert-secrets=user-serving-cert-000 - --optional-cert-secrets=user-serving-cert-001 - --optional-cert-secrets=user-serving-cert-002 - --optional-cert-secrets=user-serving-cert-003 - --optional-cert-secrets=user-serving-cert-004 - --optional-cert-secrets=user-serving-cert-005 - --optional-cert-secrets=user-serving-cert-006 - --optional-cert-secrets=user-serving-cert-007 - --optional-cert-secrets=user-serving-cert-008 - --optional-cert-secrets=user-serving-cert-009 command: - cluster-kube-apiserver-operator - installer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imagePullPolicy: IfNotPresent name: installer resources: limits: cpu: 150m memory: 200M requests: cpu: 150m memory: 200M securityContext: privileged: true runAsUser: 0 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true - mountPath: /var/lock name: var-lock dnsPolicy: ClusterFirst enableServiceLinks: true nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Never schedulerName: default-scheduler securityContext: runAsUser: 0 serviceAccount: installer-sa serviceAccountName: installer-sa terminationGracePeriodSeconds: 30 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/ type: "" name: kubelet-dir - hostPath: path: /var/lock type: "" name: var-lock - name: kube-api-access projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3600 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace status: conditions: - lastProbeTime: null lastTransitionTime: "2025-12-03T21:51:44Z" status: "False" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2025-12-03T21:50:33Z" status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2025-12-03T21:51:26Z" reason: PodFailed status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2025-12-03T21:51:26Z" reason: PodFailed status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2025-12-03T21:50:33Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://5850f56760e2a61c527c84ff5f17d82ffc3198cbe6b6606b3d9f5f38cfe114d6 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef lastState: {} name: installer ready: false restartCount: 0 started: false state: terminated: containerID: cri-o://5850f56760e2a61c527c84ff5f17d82ffc3198cbe6b6606b3d9f5f38cfe114d6 exitCode: 1 finishedAt: "2025-12-03T21:51:25Z" message: | ving-cert", (string) (len=21) "user-serving-cert-000", (string) (len=21) "user-serving-cert-001", (string) (len=21) "user-serving-cert-002", (string) (len=21) "user-serving-cert-003", (string) (len=21) "user-serving-cert-004", (string) (len=21) "user-serving-cert-005", (string) (len=21) "user-serving-cert-006", (string) (len=21) "user-serving-cert-007", (string) (len=21) "user-serving-cert-008", (string) (len=21) "user-serving-cert-009" }, CertConfigMapNamePrefixes: ([]string) (len=4 cap=4) { (string) (len=20) "aggregator-client-ca", (string) (len=9) "client-ca", (string) (len=29) "control-plane-node-kubeconfig", (string) (len=26) "check-endpoints-kubeconfig" }, OptionalCertConfigMapNamePrefixes: ([]string) (len=1 cap=1) { (string) (len=17) "trusted-ca-bundle" }, CertDir: (string) (len=57) "/etc/kubernetes/static-pod-resources/kube-apiserver-certs", ResourceDir: (string) (len=36) "/etc/kubernetes/static-pod-resources", PodManifestDir: (string) (len=25) "/etc/kubernetes/manifests", Timeout: (time.Duration) 2m0s, StaticPodManifestsLockFile: (string) "", PodMutationFns: ([]installerpod.PodMutationFunc) , KubeletVersion: (string) "" }) I1203 21:50:40.966860 1 cmd.go:413] Getting controller reference for node master-0 I1203 21:50:41.072490 1 cmd.go:426] Waiting for installer revisions to settle for node master-0 I1203 21:50:41.072561 1 envvar.go:172] "Feature gate default state" feature="WatchListClient" enabled=false I1203 21:50:41.072576 1 envvar.go:172] "Feature gate default state" feature="InformerResourceVersion" enabled=false I1203 21:50:41.110561 1 cmd.go:518] Waiting additional period after revisions have settled for node master-0 I1203 21:51:11.110899 1 cmd.go:524] Getting installer pods for node master-0 F1203 21:51:25.115236 1 cmd.go:109] Get "https://172.30.0.1:443/api/v1/namespaces/openshift-kube-apiserver/pods?labelSelector=app%3Dinstaller": net/http: request canceled (Client.Timeout exceeded while awaiting headers) reason: Error startedAt: "2025-12-03T21:50:40Z" volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true recursiveReadOnly: Disabled - mountPath: /var/lock name: var-lock hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 phase: Failed qosClass: Guaranteed startTime: "2025-12-03T21:50:33Z" - apiVersion: v1 kind: Pod metadata: annotations: k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.128.0.81/23"],"mac_address":"0a:58:0a:80:00:51","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/16","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.81/23","gateway_ip":"10.128.0.1","role":"primary"}}' k8s.v1.cni.cncf.io/network-status: |- [{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.81" ], "mac": "0a:58:0a:80:00:51", "default": true, "dns": {} }] creationTimestamp: "2025-12-03T22:00:08Z" labels: app: installer managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:labels: .: {} f:app: {} f:ownerReferences: .: {} k:{"uid":"a7febcb0-fe4c-4e7f-a9f8-48c96500f06b"}: {} f:spec: f:automountServiceAccountToken: {} f:containers: k:{"name":"installer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"NODE_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:limits: .: {} f:cpu: {} f:memory: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:runAsUser: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/lock"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}: .: {} f:mountPath: {} f:name: {} f:readOnly: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:nodeName: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: .: {} f:runAsUser: {} f:serviceAccount: {} f:serviceAccountName: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"kube-api-access"}: .: {} f:name: {} f:projected: .: {} f:defaultMode: {} f:sources: {} k:{"name":"kubelet-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"var-lock"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: cluster-kube-apiserver-operator operation: Update time: "2025-12-03T22:00:08Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:k8s.ovn.org/pod-networks: {} manager: master-0 operation: Update subresource: status time: "2025-12-03T22:00:08Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: f:k8s.v1.cni.cncf.io/network-status: {} manager: multus-daemon operation: Update subresource: status time: "2025-12-03T22:00:09Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:phase: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2025-12-03T22:05:27Z" name: installer-2-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 kind: ConfigMap name: revision-status-2 uid: a7febcb0-fe4c-4e7f-a9f8-48c96500f06b resourceVersion: "13852" uid: 8d60f02e-1803-461e-9606-667d91fcae14 spec: automountServiceAccountToken: false containers: - args: - -v=2 - --revision=2 - --namespace=openshift-kube-apiserver - --pod=kube-apiserver-pod - --resource-dir=/etc/kubernetes/static-pod-resources - --pod-manifest-dir=/etc/kubernetes/manifests - --configmaps=kube-apiserver-pod - --configmaps=config - --configmaps=kube-apiserver-cert-syncer-kubeconfig - --optional-configmaps=oauth-metadata - --optional-configmaps=cloud-config - --configmaps=bound-sa-token-signing-certs - --configmaps=etcd-serving-ca - --optional-configmaps=kube-apiserver-server-ca - --configmaps=kubelet-serving-ca - --configmaps=sa-token-signing-certs - --configmaps=kube-apiserver-audit-policies - --secrets=etcd-client - --optional-secrets=encryption-config - --secrets=localhost-recovery-serving-certkey - --secrets=localhost-recovery-client-token - --optional-secrets=webhook-authenticator - --cert-dir=/etc/kubernetes/static-pod-resources/kube-apiserver-certs - --cert-configmaps=aggregator-client-ca - --cert-configmaps=client-ca - --optional-cert-configmaps=trusted-ca-bundle - --cert-configmaps=control-plane-node-kubeconfig - --cert-configmaps=check-endpoints-kubeconfig - --cert-secrets=aggregator-client - --cert-secrets=localhost-serving-cert-certkey - --cert-secrets=service-network-serving-certkey - --cert-secrets=external-loadbalancer-serving-certkey - --cert-secrets=internal-loadbalancer-serving-certkey - --cert-secrets=bound-service-account-signing-key - --cert-secrets=control-plane-node-admin-client-cert-key - --cert-secrets=check-endpoints-client-cert-key - --cert-secrets=kubelet-client - --cert-secrets=node-kubeconfigs - --optional-cert-secrets=user-serving-cert - --optional-cert-secrets=user-serving-cert-000 - --optional-cert-secrets=user-serving-cert-001 - --optional-cert-secrets=user-serving-cert-002 - --optional-cert-secrets=user-serving-cert-003 - --optional-cert-secrets=user-serving-cert-004 - --optional-cert-secrets=user-serving-cert-005 - --optional-cert-secrets=user-serving-cert-006 - --optional-cert-secrets=user-serving-cert-007 - --optional-cert-secrets=user-serving-cert-008 - --optional-cert-secrets=user-serving-cert-009 command: - cluster-kube-apiserver-operator - installer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imagePullPolicy: IfNotPresent name: installer resources: limits: cpu: 150m memory: 200M requests: cpu: 150m memory: 200M securityContext: privileged: true runAsUser: 0 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true - mountPath: /var/lock name: var-lock dnsPolicy: ClusterFirst enableServiceLinks: true imagePullSecrets: - name: installer-sa-dockercfg-wkdbt nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Never schedulerName: default-scheduler securityContext: runAsUser: 0 serviceAccount: installer-sa serviceAccountName: installer-sa terminationGracePeriodSeconds: 30 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/ type: "" name: kubelet-dir - hostPath: path: /var/lock type: "" name: var-lock - name: kube-api-access projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3600 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace status: conditions: - lastProbeTime: null lastTransitionTime: "2025-12-03T22:00:57Z" status: "False" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2025-12-03T22:00:08Z" status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2025-12-03T22:00:55Z" reason: PodFailed status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2025-12-03T22:00:55Z" reason: PodFailed status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2025-12-03T22:00:08Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://a31f9ca2a3872e9cae07acaea514d81ec85d80124c14419e3e1663f38e942380 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef lastState: {} name: installer ready: false restartCount: 0 started: false state: terminated: containerID: cri-o://a31f9ca2a3872e9cae07acaea514d81ec85d80124c14419e3e1663f38e942380 exitCode: 1 finishedAt: "2025-12-03T22:00:54Z" message: | ving-cert", (string) (len=21) "user-serving-cert-000", (string) (len=21) "user-serving-cert-001", (string) (len=21) "user-serving-cert-002", (string) (len=21) "user-serving-cert-003", (string) (len=21) "user-serving-cert-004", (string) (len=21) "user-serving-cert-005", (string) (len=21) "user-serving-cert-006", (string) (len=21) "user-serving-cert-007", (string) (len=21) "user-serving-cert-008", (string) (len=21) "user-serving-cert-009" }, CertConfigMapNamePrefixes: ([]string) (len=4 cap=4) { (string) (len=20) "aggregator-client-ca", (string) (len=9) "client-ca", (string) (len=29) "control-plane-node-kubeconfig", (string) (len=26) "check-endpoints-kubeconfig" }, OptionalCertConfigMapNamePrefixes: ([]string) (len=1 cap=1) { (string) (len=17) "trusted-ca-bundle" }, CertDir: (string) (len=57) "/etc/kubernetes/static-pod-resources/kube-apiserver-certs", ResourceDir: (string) (len=36) "/etc/kubernetes/static-pod-resources", PodManifestDir: (string) (len=25) "/etc/kubernetes/manifests", Timeout: (time.Duration) 2m0s, StaticPodManifestsLockFile: (string) "", PodMutationFns: ([]installerpod.PodMutationFunc) , KubeletVersion: (string) "" }) I1203 22:00:10.745469 1 cmd.go:413] Getting controller reference for node master-0 I1203 22:00:10.755759 1 cmd.go:426] Waiting for installer revisions to settle for node master-0 I1203 22:00:10.755890 1 envvar.go:172] "Feature gate default state" feature="WatchListClient" enabled=false I1203 22:00:10.755912 1 envvar.go:172] "Feature gate default state" feature="InformerResourceVersion" enabled=false I1203 22:00:10.759314 1 cmd.go:518] Waiting additional period after revisions have settled for node master-0 I1203 22:00:40.759357 1 cmd.go:524] Getting installer pods for node master-0 F1203 22:00:54.762806 1 cmd.go:109] Get "https://172.30.0.1:443/api/v1/namespaces/openshift-kube-apiserver/pods?labelSelector=app%3Dinstaller": net/http: request canceled (Client.Timeout exceeded while awaiting headers) reason: Error startedAt: "2025-12-03T22:00:09Z" volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true recursiveReadOnly: Disabled - mountPath: /var/lock name: var-lock hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 phase: Failed qosClass: Guaranteed startTime: "2025-12-03T22:00:08Z" - apiVersion: v1 kind: Pod metadata: annotations: k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.128.0.91/23"],"mac_address":"0a:58:0a:80:00:5b","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/16","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.91/23","gateway_ip":"10.128.0.1","role":"primary"}}' k8s.v1.cni.cncf.io/network-status: |- [{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.91" ], "mac": "0a:58:0a:80:00:5b", "default": true, "dns": {} }] creationTimestamp: "2025-12-03T22:09:37Z" labels: app: installer managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:labels: .: {} f:app: {} f:ownerReferences: .: {} k:{"uid":"2ed59807-c2e6-41a9-87e7-0b922a3b2a0c"}: {} f:spec: f:automountServiceAccountToken: {} f:containers: k:{"name":"installer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"NODE_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:limits: .: {} f:cpu: {} f:memory: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:runAsUser: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/lock"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}: .: {} f:mountPath: {} f:name: {} f:readOnly: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:nodeName: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: .: {} f:runAsUser: {} f:serviceAccount: {} f:serviceAccountName: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"kube-api-access"}: .: {} f:name: {} f:projected: .: {} f:defaultMode: {} f:sources: {} k:{"name":"kubelet-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"var-lock"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: cluster-kube-apiserver-operator operation: Update time: "2025-12-03T22:09:37Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:k8s.ovn.org/pod-networks: {} manager: master-0 operation: Update subresource: status time: "2025-12-03T22:09:37Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: f:k8s.v1.cni.cncf.io/network-status: {} manager: multus-daemon operation: Update subresource: status time: "2025-12-03T22:09:38Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:phase: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2025-12-03T22:10:41Z" name: installer-5-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 kind: ConfigMap name: revision-status-5 uid: 2ed59807-c2e6-41a9-87e7-0b922a3b2a0c resourceVersion: "13857" uid: 5c8c7291-3150-46a5-9d14-57a23bb51cc0 spec: automountServiceAccountToken: false containers: - args: - -v=2 - --revision=5 - --namespace=openshift-kube-apiserver - --pod=kube-apiserver-pod - --resource-dir=/etc/kubernetes/static-pod-resources - --pod-manifest-dir=/etc/kubernetes/manifests - --configmaps=kube-apiserver-pod - --configmaps=config - --configmaps=kube-apiserver-cert-syncer-kubeconfig - --optional-configmaps=oauth-metadata - --optional-configmaps=cloud-config - --configmaps=bound-sa-token-signing-certs - --configmaps=etcd-serving-ca - --optional-configmaps=kube-apiserver-server-ca - --configmaps=kubelet-serving-ca - --configmaps=sa-token-signing-certs - --configmaps=kube-apiserver-audit-policies - --secrets=etcd-client - --optional-secrets=encryption-config - --secrets=localhost-recovery-serving-certkey - --secrets=localhost-recovery-client-token - --optional-secrets=webhook-authenticator - --cert-dir=/etc/kubernetes/static-pod-resources/kube-apiserver-certs - --cert-configmaps=aggregator-client-ca - --cert-configmaps=client-ca - --optional-cert-configmaps=trusted-ca-bundle - --cert-configmaps=control-plane-node-kubeconfig - --cert-configmaps=check-endpoints-kubeconfig - --cert-secrets=aggregator-client - --cert-secrets=localhost-serving-cert-certkey - --cert-secrets=service-network-serving-certkey - --cert-secrets=external-loadbalancer-serving-certkey - --cert-secrets=internal-loadbalancer-serving-certkey - --cert-secrets=bound-service-account-signing-key - --cert-secrets=control-plane-node-admin-client-cert-key - --cert-secrets=check-endpoints-client-cert-key - --cert-secrets=kubelet-client - --cert-secrets=node-kubeconfigs - --optional-cert-secrets=user-serving-cert - --optional-cert-secrets=user-serving-cert-000 - --optional-cert-secrets=user-serving-cert-001 - --optional-cert-secrets=user-serving-cert-002 - --optional-cert-secrets=user-serving-cert-003 - --optional-cert-secrets=user-serving-cert-004 - --optional-cert-secrets=user-serving-cert-005 - --optional-cert-secrets=user-serving-cert-006 - --optional-cert-secrets=user-serving-cert-007 - --optional-cert-secrets=user-serving-cert-008 - --optional-cert-secrets=user-serving-cert-009 command: - cluster-kube-apiserver-operator - installer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imagePullPolicy: IfNotPresent name: installer resources: limits: cpu: 150m memory: 200M requests: cpu: 150m memory: 200M securityContext: privileged: true runAsUser: 0 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true - mountPath: /var/lock name: var-lock dnsPolicy: ClusterFirst enableServiceLinks: true imagePullSecrets: - name: installer-sa-dockercfg-wkdbt nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Never schedulerName: default-scheduler securityContext: runAsUser: 0 serviceAccount: installer-sa serviceAccountName: installer-sa terminationGracePeriodSeconds: 30 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/ type: "" name: kubelet-dir - hostPath: path: /var/lock type: "" name: var-lock - name: kube-api-access projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3600 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace status: conditions: - lastProbeTime: null lastTransitionTime: "2025-12-03T22:10:36Z" status: "False" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2025-12-03T22:09:37Z" reason: PodCompleted status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2025-12-03T22:10:36Z" reason: PodCompleted status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2025-12-03T22:10:36Z" reason: PodCompleted status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2025-12-03T22:09:37Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://404a35f8de80b669ffc0285b3b2f8a18d00355d93f5f0568bb71370a69a93877 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef lastState: {} name: installer ready: false restartCount: 0 started: false state: terminated: containerID: cri-o://404a35f8de80b669ffc0285b3b2f8a18d00355d93f5f0568bb71370a69a93877 exitCode: 0 finishedAt: "2025-12-03T22:10:27Z" reason: Completed startedAt: "2025-12-03T22:09:39Z" volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true recursiveReadOnly: Disabled - mountPath: /var/lock name: var-lock hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 phase: Succeeded qosClass: Guaranteed startTime: "2025-12-03T22:09:37Z" - apiVersion: v1 kind: Pod metadata: annotations: k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.128.0.108/23"],"mac_address":"0a:58:0a:80:00:6c","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/16","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.108/23","gateway_ip":"10.128.0.1","role":"primary"}}' k8s.v1.cni.cncf.io/network-status: |- [{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.108" ], "mac": "0a:58:0a:80:00:6c", "default": true, "dns": {} }] creationTimestamp: "2025-12-03T22:11:58Z" labels: app: installer managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:labels: .: {} f:app: {} f:ownerReferences: .: {} k:{"uid":"aa86d0f5-68f1-476b-9194-1d1860298658"}: {} f:spec: f:automountServiceAccountToken: {} f:containers: k:{"name":"installer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"NODE_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:limits: .: {} f:cpu: {} f:memory: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:runAsUser: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/lock"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}: .: {} f:mountPath: {} f:name: {} f:readOnly: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:nodeName: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: .: {} f:runAsUser: {} f:serviceAccount: {} f:serviceAccountName: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"kube-api-access"}: .: {} f:name: {} f:projected: .: {} f:defaultMode: {} f:sources: {} k:{"name":"kubelet-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"var-lock"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: cluster-kube-apiserver-operator operation: Update time: "2025-12-03T22:11:58Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:k8s.ovn.org/pod-networks: {} manager: master-0 operation: Update subresource: status time: "2025-12-03T22:11:58Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: f:k8s.v1.cni.cncf.io/network-status: {} manager: multus-daemon operation: Update subresource: status time: "2025-12-03T22:11:59Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:phase: {} f:podIP: {} f:podIPs: .: {} k:{"ip":"10.128.0.108"}: .: {} f:ip: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2025-12-03T22:12:59Z" name: installer-7-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 kind: ConfigMap name: revision-status-7 uid: aa86d0f5-68f1-476b-9194-1d1860298658 resourceVersion: "17789" uid: 0e0ffab3-4f70-4bc8-9464-598c0668a4b8 spec: automountServiceAccountToken: false containers: - args: - -v=2 - --revision=7 - --namespace=openshift-kube-apiserver - --pod=kube-apiserver-pod - --resource-dir=/etc/kubernetes/static-pod-resources - --pod-manifest-dir=/etc/kubernetes/manifests - --configmaps=kube-apiserver-pod - --configmaps=config - --configmaps=kube-apiserver-cert-syncer-kubeconfig - --optional-configmaps=oauth-metadata - --optional-configmaps=cloud-config - --configmaps=bound-sa-token-signing-certs - --configmaps=etcd-serving-ca - --optional-configmaps=kube-apiserver-server-ca - --configmaps=kubelet-serving-ca - --configmaps=sa-token-signing-certs - --configmaps=kube-apiserver-audit-policies - --secrets=etcd-client - --optional-secrets=encryption-config - --secrets=localhost-recovery-serving-certkey - --secrets=localhost-recovery-client-token - --optional-secrets=webhook-authenticator - --cert-dir=/etc/kubernetes/static-pod-resources/kube-apiserver-certs - --cert-configmaps=aggregator-client-ca - --cert-configmaps=client-ca - --optional-cert-configmaps=trusted-ca-bundle - --cert-configmaps=control-plane-node-kubeconfig - --cert-configmaps=check-endpoints-kubeconfig - --cert-secrets=aggregator-client - --cert-secrets=localhost-serving-cert-certkey - --cert-secrets=service-network-serving-certkey - --cert-secrets=external-loadbalancer-serving-certkey - --cert-secrets=internal-loadbalancer-serving-certkey - --cert-secrets=bound-service-account-signing-key - --cert-secrets=control-plane-node-admin-client-cert-key - --cert-secrets=check-endpoints-client-cert-key - --cert-secrets=kubelet-client - --cert-secrets=node-kubeconfigs - --optional-cert-secrets=user-serving-cert - --optional-cert-secrets=user-serving-cert-000 - --optional-cert-secrets=user-serving-cert-001 - --optional-cert-secrets=user-serving-cert-002 - --optional-cert-secrets=user-serving-cert-003 - --optional-cert-secrets=user-serving-cert-004 - --optional-cert-secrets=user-serving-cert-005 - --optional-cert-secrets=user-serving-cert-006 - --optional-cert-secrets=user-serving-cert-007 - --optional-cert-secrets=user-serving-cert-008 - --optional-cert-secrets=user-serving-cert-009 command: - cluster-kube-apiserver-operator - installer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imagePullPolicy: IfNotPresent name: installer resources: limits: cpu: 150m memory: 200M requests: cpu: 150m memory: 200M securityContext: privileged: true runAsUser: 0 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true - mountPath: /var/lock name: var-lock dnsPolicy: ClusterFirst enableServiceLinks: true imagePullSecrets: - name: installer-sa-dockercfg-wkdbt nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Never schedulerName: default-scheduler securityContext: runAsUser: 0 serviceAccount: installer-sa serviceAccountName: installer-sa terminationGracePeriodSeconds: 30 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/ type: "" name: kubelet-dir - hostPath: path: /var/lock type: "" name: var-lock - name: kube-api-access projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3600 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace status: conditions: - lastProbeTime: null lastTransitionTime: "2025-12-03T22:12:41Z" status: "False" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2025-12-03T22:11:58Z" reason: PodCompleted status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2025-12-03T22:12:39Z" reason: PodCompleted status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2025-12-03T22:12:39Z" reason: PodCompleted status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2025-12-03T22:11:58Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://f475b284f57f52fc0c7eb702e9328890537fd74a9dd50bbac04f195c30c41055 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef lastState: {} name: installer ready: false restartCount: 0 started: false state: terminated: containerID: cri-o://f475b284f57f52fc0c7eb702e9328890537fd74a9dd50bbac04f195c30c41055 exitCode: 0 finishedAt: "2025-12-03T22:12:39Z" reason: Completed startedAt: "2025-12-03T22:12:00Z" volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true recursiveReadOnly: Disabled - mountPath: /var/lock name: var-lock hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 phase: Succeeded podIP: 10.128.0.108 podIPs: - ip: 10.128.0.108 qosClass: Guaranteed startTime: "2025-12-03T22:11:58Z" - apiVersion: v1 kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: kube-apiserver kubernetes.io/config.hash: 7035982c3bafe5c31d1c2515ed3c2b7b kubernetes.io/config.mirror: 7035982c3bafe5c31d1c2515ed3c2b7b kubernetes.io/config.seen: "2025-12-03T22:12:39.360492860Z" kubernetes.io/config.source: file target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' creationTimestamp: "2025-12-03T22:12:59Z" labels: apiserver: "true" app: openshift-kube-apiserver revision: "7" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:kubectl.kubernetes.io/default-container: {} f:kubernetes.io/config.hash: {} f:kubernetes.io/config.mirror: {} f:kubernetes.io/config.seen: {} f:kubernetes.io/config.source: {} f:target.workload.openshift.io/management: {} f:labels: .: {} f:apiserver: {} f:app: {} f:revision: {} f:ownerReferences: .: {} k:{"uid":"4b803739-f301-4b6d-80bd-2ffa2100c90b"}: {} f:spec: f:containers: k:{"name":"kube-apiserver"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"GOGC"}: .: {} f:name: {} f:value: {} k:{"name":"HOST_IP"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAMESPACE"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"STATIC_POD_VERSION"}: .: {} f:name: {} f:value: {} f:image: {} f:imagePullPolicy: {} f:livenessProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:name: {} f:ports: .: {} k:{"containerPort":6443,"protocol":"TCP"}: .: {} f:containerPort: {} f:hostPort: {} f:protocol: {} f:readinessProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:startupProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/static-pod-certs"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/etc/kubernetes/static-pod-resources"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/log/kube-apiserver"}: .: {} f:mountPath: {} f:name: {} k:{"name":"kube-apiserver-cert-regeneration-controller"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"POD_NAMESPACE"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/static-pod-resources"}: .: {} f:mountPath: {} f:name: {} k:{"name":"kube-apiserver-cert-syncer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAMESPACE"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/static-pod-certs"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/etc/kubernetes/static-pod-resources"}: .: {} f:mountPath: {} f:name: {} k:{"name":"kube-apiserver-check-endpoints"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAMESPACE"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:livenessProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:initialDelaySeconds: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:name: {} f:ports: .: {} k:{"containerPort":17697,"protocol":"TCP"}: .: {} f:containerPort: {} f:hostPort: {} f:name: {} f:protocol: {} f:readinessProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:initialDelaySeconds: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/static-pod-certs"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/etc/kubernetes/static-pod-resources"}: .: {} f:mountPath: {} f:name: {} k:{"name":"kube-apiserver-insecure-readyz"}: .: {} f:args: {} f:command: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:ports: .: {} k:{"containerPort":6080,"protocol":"TCP"}: .: {} f:containerPort: {} f:hostPort: {} f:protocol: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:hostNetwork: {} f:initContainers: .: {} k:{"name":"setup"}: .: {} f:args: {} f:command: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/var/log/kube-apiserver"}: .: {} f:mountPath: {} f:name: {} f:nodeName: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"audit-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"cert-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"resource-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: kubelet operation: Update time: "2025-12-03T22:12:59Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:initContainerStatuses: {} f:phase: {} f:podIP: {} f:podIPs: .: {} k:{"ip":"192.168.32.10"}: .: {} f:ip: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2025-12-03T22:13:14Z" name: kube-apiserver-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 controller: true kind: Node name: master-0 uid: 4b803739-f301-4b6d-80bd-2ffa2100c90b resourceVersion: "17813" uid: 38b28adc-5b89-41ff-a922-3f7bee5f8b6f spec: containers: - args: - | LOCK=/var/log/kube-apiserver/.lock # We should be able to acquire the lock immediatelly. If not, it means the init container has not released it yet and kubelet or CRI-O started container prematurely. exec {LOCK_FD}>${LOCK} && flock --verbose -w 30 "${LOCK_FD}" || { echo "Failed to acquire lock for kube-apiserver. Please check setup container for details. This is likely kubelet or CRI-O bug." exit 1 } if [ -f /etc/kubernetes/static-pod-certs/configmaps/trusted-ca-bundle/ca-bundle.crt ]; then echo "Copying system trust bundle ..." cp -f /etc/kubernetes/static-pod-certs/configmaps/trusted-ca-bundle/ca-bundle.crt /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem fi exec watch-termination --termination-touch-file=/var/log/kube-apiserver/.terminating --termination-log-file=/var/log/kube-apiserver/termination.log --graceful-termination-duration=15s --kubeconfig=/etc/kubernetes/static-pod-resources/configmaps/kube-apiserver-cert-syncer-kubeconfig/kubeconfig -- hyperkube kube-apiserver --openshift-config=/etc/kubernetes/static-pod-resources/configmaps/config/config.yaml --advertise-address=${HOST_IP} -v=2 --permit-address-sharing command: - /bin/bash - -ec env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: STATIC_POD_VERSION value: "7" - name: HOST_IP valueFrom: fieldRef: apiVersion: v1 fieldPath: status.hostIP - name: GOGC value: "100" image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d0bb91faa6e9f82b589a6535665e51517abe4a1b2eb5d0b3a36b36df6a5330a0 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 httpGet: path: livez?exclude=etcd port: 6443 scheme: HTTPS periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 name: kube-apiserver ports: - containerPort: 6443 hostPort: 6443 protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: readyz port: 6443 scheme: HTTPS periodSeconds: 5 successThreshold: 1 timeoutSeconds: 10 resources: requests: cpu: 265m memory: 1Gi securityContext: privileged: true startupProbe: failureThreshold: 30 httpGet: path: livez port: 6443 scheme: HTTPS periodSeconds: 5 successThreshold: 1 timeoutSeconds: 10 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/static-pod-resources name: resource-dir - mountPath: /etc/kubernetes/static-pod-certs name: cert-dir - mountPath: /var/log/kube-apiserver name: audit-dir - args: - --kubeconfig=/etc/kubernetes/static-pod-resources/configmaps/kube-apiserver-cert-syncer-kubeconfig/kubeconfig - --namespace=$(POD_NAMESPACE) - --destination-dir=/etc/kubernetes/static-pod-certs command: - cluster-kube-apiserver-operator - cert-syncer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imagePullPolicy: IfNotPresent name: kube-apiserver-cert-syncer resources: requests: cpu: 5m memory: 50Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/static-pod-resources name: resource-dir - mountPath: /etc/kubernetes/static-pod-certs name: cert-dir - args: - --kubeconfig=/etc/kubernetes/static-pod-resources/configmaps/kube-apiserver-cert-syncer-kubeconfig/kubeconfig - --namespace=$(POD_NAMESPACE) - -v=2 command: - cluster-kube-apiserver-operator - cert-regeneration-controller env: - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imagePullPolicy: IfNotPresent name: kube-apiserver-cert-regeneration-controller resources: requests: cpu: 5m memory: 50Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/static-pod-resources name: resource-dir - args: - --insecure-port=6080 - --delegate-url=https://localhost:6443/readyz command: - cluster-kube-apiserver-operator - insecure-readyz image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imagePullPolicy: IfNotPresent name: kube-apiserver-insecure-readyz ports: - containerPort: 6080 hostPort: 6080 protocol: TCP resources: requests: cpu: 5m memory: 50Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError - args: - --kubeconfig - /etc/kubernetes/static-pod-certs/configmaps/check-endpoints-kubeconfig/kubeconfig - --listen - 0.0.0.0:17697 - --namespace - $(POD_NAMESPACE) - --v - "2" command: - cluster-kube-apiserver-operator - check-endpoints env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 httpGet: path: healthz port: 17697 scheme: HTTPS initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 name: kube-apiserver-check-endpoints ports: - containerPort: 17697 hostPort: 17697 name: check-endpoints protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: healthz port: 17697 scheme: HTTPS initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 resources: requests: cpu: 10m memory: 50Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/static-pod-resources name: resource-dir - mountPath: /etc/kubernetes/static-pod-certs name: cert-dir dnsPolicy: ClusterFirst enableServiceLinks: true hostNetwork: true initContainers: - args: - | echo "Fixing audit permissions ..." chmod 0700 /var/log/kube-apiserver && touch /var/log/kube-apiserver/audit.log && chmod 0600 /var/log/kube-apiserver/* LOCK=/var/log/kube-apiserver/.lock echo "Acquiring exclusive lock ${LOCK} ..." # Waiting for 15s max for old kube-apiserver's watch-termination process to exit and remove the lock. # Two cases: # 1. if kubelet does not start the old and new in parallel (i.e. works as expected), the flock will always succeed without any time. # 2. if kubelet does overlap old and new pods for up to 130s, the flock will wait and immediate return when the old finishes. # # NOTE: We can increase 15s for a bigger expected overlap. But a higher value means less noise about the broken kubelet behaviour, i.e. we hide a bug. # NOTE: Do not tweak these timings without considering the livenessProbe initialDelaySeconds exec {LOCK_FD}>${LOCK} && flock --verbose -w 15 "${LOCK_FD}" || { echo "$(date -Iseconds -u) kubelet did not terminate old kube-apiserver before new one" >> /var/log/kube-apiserver/lock.log echo -n ": WARNING: kubelet did not terminate old kube-apiserver before new one." # We failed to acquire exclusive lock, which means there is old kube-apiserver running in system. # Since we utilize SO_REUSEPORT, we need to make sure the old kube-apiserver stopped listening. # # NOTE: This is a fallback for broken kubelet, if you observe this please report a bug. echo -n "Waiting for port 6443 to be released due to likely bug in kubelet or CRI-O " while [ -n "$(ss -Htan state listening '( sport = 6443 or sport = 6080 )')" ]; do echo -n "." sleep 1 (( tries += 1 )) if [[ "${tries}" -gt 10 ]]; then echo "Timed out waiting for port :6443 and :6080 to be released, this is likely a bug in kubelet or CRI-O" exit 1 fi done # This is to make sure the server has terminated independently from the lock. # After the port has been freed (requests can be pending and need 60s max). sleep 65 } # We cannot hold the lock from the init container to the main container. We release it here. There is no risk, at this point we know we are safe. flock -u "${LOCK_FD}" command: - /usr/bin/timeout - "100" - /bin/bash - -ec image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d0bb91faa6e9f82b589a6535665e51517abe4a1b2eb5d0b3a36b36df6a5330a0 imagePullPolicy: IfNotPresent name: setup resources: requests: cpu: 5m memory: 50Mi securityContext: privileged: true terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /var/log/kube-apiserver name: audit-dir nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Always schedulerName: default-scheduler securityContext: {} terminationGracePeriodSeconds: 15 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/static-pod-resources/kube-apiserver-pod-7 type: "" name: resource-dir - hostPath: path: /etc/kubernetes/static-pod-resources/kube-apiserver-certs type: "" name: cert-dir - hostPath: path: /var/log/kube-apiserver type: "" name: audit-dir status: conditions: - lastProbeTime: null lastTransitionTime: "2025-12-03T22:13:13Z" status: "True" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2025-12-03T22:13:13Z" status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2025-12-03T22:13:13Z" status: "True" type: Ready - lastProbeTime: null lastTransitionTime: "2025-12-03T22:13:13Z" status: "True" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2025-12-03T22:13:13Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://3272cc6886cf112de9d20011e390a8a35fac24de99f8ded9102adf6bdcb3c6c9 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d0bb91faa6e9f82b589a6535665e51517abe4a1b2eb5d0b3a36b36df6a5330a0 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d0bb91faa6e9f82b589a6535665e51517abe4a1b2eb5d0b3a36b36df6a5330a0 lastState: {} name: kube-apiserver ready: true restartCount: 0 started: true state: running: startedAt: "2025-12-03T22:12:52Z" - containerID: cri-o://65b5f79ca7b96ed93db29aba3ae8bc241455646d2e1529bdd643eae30fe1f668 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef lastState: {} name: kube-apiserver-cert-regeneration-controller ready: true restartCount: 0 started: true state: running: startedAt: "2025-12-03T22:12:53Z" - containerID: cri-o://8c7639960cf9c49e9622092221b6170e88fd2b93ed11b2c4ea659569ecee395e image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef lastState: {} name: kube-apiserver-cert-syncer ready: true restartCount: 0 started: true state: running: startedAt: "2025-12-03T22:12:52Z" - containerID: cri-o://955271c7013622323152b1ed3e8ef6837a03e28fba160cb3700fc501e00be066 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef lastState: {} name: kube-apiserver-check-endpoints ready: true restartCount: 0 started: true state: running: startedAt: "2025-12-03T22:12:53Z" - containerID: cri-o://42c10f8f264ec3e98ef146e8a892100d9ac508c4e49eb9423e0fb5e62132834c image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:91cbda9693e888881e7c45cd6e504b91ba8a203fe0596237a4a17b3ca4e18eef lastState: {} name: kube-apiserver-insecure-readyz ready: true restartCount: 0 started: true state: running: startedAt: "2025-12-03T22:12:53Z" hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 initContainerStatuses: - containerID: cri-o://d2f7ce9cba8cf9ed23de77d97df3b962c5afbd4108cacda986a7ddf6d2b22a14 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d0bb91faa6e9f82b589a6535665e51517abe4a1b2eb5d0b3a36b36df6a5330a0 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d0bb91faa6e9f82b589a6535665e51517abe4a1b2eb5d0b3a36b36df6a5330a0 lastState: {} name: setup ready: true restartCount: 0 started: false state: terminated: containerID: cri-o://d2f7ce9cba8cf9ed23de77d97df3b962c5afbd4108cacda986a7ddf6d2b22a14 exitCode: 0 finishedAt: "2025-12-03T22:12:51Z" reason: Completed startedAt: "2025-12-03T22:12:51Z" phase: Running podIP: 192.168.32.10 podIPs: - ip: 192.168.32.10 qosClass: Burstable startTime: "2025-12-03T22:13:13Z" kind: PodList metadata: resourceVersion: "69641"