--- apiVersion: v1 items: - apiVersion: v1 kind: Pod metadata: annotations: k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.128.0.45/23"],"mac_address":"0a:58:0a:80:00:2d","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/16","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.45/23","gateway_ip":"10.128.0.1","role":"primary"}}' k8s.v1.cni.cncf.io/network-status: |- [{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.45" ], "mac": "0a:58:0a:80:00:2d", "default": true, "dns": {} }] creationTimestamp: "2026-03-08T03:12:26Z" labels: app: installer managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:labels: .: {} f:app: {} f:ownerReferences: .: {} k:{"uid":"57b516c1-d32b-4453-a382-89b784c9bb69"}: {} f:spec: f:automountServiceAccountToken: {} f:containers: k:{"name":"installer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"NODE_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:limits: .: {} f:cpu: {} f:memory: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:runAsUser: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/lock"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}: .: {} f:mountPath: {} f:name: {} f:readOnly: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:nodeName: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: .: {} f:runAsUser: {} f:serviceAccount: {} f:serviceAccountName: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"kube-api-access"}: .: {} f:name: {} f:projected: .: {} f:defaultMode: {} f:sources: {} k:{"name":"kubelet-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"var-lock"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: cluster-kube-apiserver-operator operation: Update time: "2026-03-08T03:12:26Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:k8s.ovn.org/pod-networks: {} manager: master-0 operation: Update subresource: status time: "2026-03-08T03:12:26Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: f:k8s.v1.cni.cncf.io/network-status: {} manager: multus-daemon operation: Update subresource: status time: "2026-03-08T03:12:29Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:phase: {} f:podIP: {} f:podIPs: .: {} k:{"ip":"10.128.0.45"}: .: {} f:ip: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2026-03-08T03:18:05Z" name: installer-1-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 kind: ConfigMap name: revision-status-1 uid: 57b516c1-d32b-4453-a382-89b784c9bb69 resourceVersion: "9775" uid: 7ea81472-8a81-45ec-a07d-8710f47a927d spec: automountServiceAccountToken: false containers: - args: - -v=2 - --revision=1 - --namespace=openshift-kube-apiserver - --pod=kube-apiserver-pod - --resource-dir=/etc/kubernetes/static-pod-resources - --pod-manifest-dir=/etc/kubernetes/manifests - --configmaps=kube-apiserver-pod - --configmaps=config - --configmaps=kube-apiserver-cert-syncer-kubeconfig - --optional-configmaps=oauth-metadata - --optional-configmaps=cloud-config - --configmaps=bound-sa-token-signing-certs - --configmaps=etcd-serving-ca - --optional-configmaps=kube-apiserver-server-ca - --configmaps=kubelet-serving-ca - --configmaps=sa-token-signing-certs - --configmaps=kube-apiserver-audit-policies - --secrets=etcd-client - --optional-secrets=encryption-config - --secrets=localhost-recovery-serving-certkey - --secrets=localhost-recovery-client-token - --optional-secrets=webhook-authenticator - --cert-dir=/etc/kubernetes/static-pod-resources/kube-apiserver-certs - --cert-configmaps=aggregator-client-ca - --cert-configmaps=client-ca - --optional-cert-configmaps=trusted-ca-bundle - --cert-configmaps=control-plane-node-kubeconfig - --cert-configmaps=check-endpoints-kubeconfig - --cert-secrets=aggregator-client - --cert-secrets=localhost-serving-cert-certkey - --cert-secrets=service-network-serving-certkey - --cert-secrets=external-loadbalancer-serving-certkey - --cert-secrets=internal-loadbalancer-serving-certkey - --cert-secrets=bound-service-account-signing-key - --cert-secrets=control-plane-node-admin-client-cert-key - --cert-secrets=check-endpoints-client-cert-key - --cert-secrets=kubelet-client - --cert-secrets=node-kubeconfigs - --optional-cert-secrets=user-serving-cert - --optional-cert-secrets=user-serving-cert-000 - --optional-cert-secrets=user-serving-cert-001 - --optional-cert-secrets=user-serving-cert-002 - --optional-cert-secrets=user-serving-cert-003 - --optional-cert-secrets=user-serving-cert-004 - --optional-cert-secrets=user-serving-cert-005 - --optional-cert-secrets=user-serving-cert-006 - --optional-cert-secrets=user-serving-cert-007 - --optional-cert-secrets=user-serving-cert-008 - --optional-cert-secrets=user-serving-cert-009 command: - cluster-kube-apiserver-operator - installer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imagePullPolicy: IfNotPresent name: installer resources: limits: cpu: 150m memory: 200M requests: cpu: 150m memory: 200M securityContext: privileged: true runAsUser: 0 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true - mountPath: /var/lock name: var-lock dnsPolicy: ClusterFirst enableServiceLinks: true imagePullSecrets: - name: installer-sa-dockercfg-gcdsk nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Never schedulerName: default-scheduler securityContext: runAsUser: 0 serviceAccount: installer-sa serviceAccountName: installer-sa terminationGracePeriodSeconds: 30 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/ type: "" name: kubelet-dir - hostPath: path: /var/lock type: "" name: var-lock - name: kube-api-access projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3600 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace status: conditions: - lastProbeTime: null lastTransitionTime: "2026-03-08T03:18:01Z" status: "False" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2026-03-08T03:12:26Z" reason: PodCompleted status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2026-03-08T03:18:01Z" reason: PodCompleted status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2026-03-08T03:18:01Z" reason: PodCompleted status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2026-03-08T03:12:26Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://d57e1157c7569d934ea76665ae63811243fb6a6eb902e18c216d3947853ca6e4 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 lastState: {} name: installer ready: false restartCount: 0 started: false state: terminated: containerID: cri-o://d57e1157c7569d934ea76665ae63811243fb6a6eb902e18c216d3947853ca6e4 exitCode: 0 finishedAt: "2026-03-08T03:13:11Z" reason: Completed startedAt: "2026-03-08T03:12:29Z" volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true recursiveReadOnly: Disabled - mountPath: /var/lock name: var-lock hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 phase: Succeeded podIP: 10.128.0.45 podIPs: - ip: 10.128.0.45 qosClass: Guaranteed startTime: "2026-03-08T03:12:26Z" - apiVersion: v1 kind: Pod metadata: annotations: k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.128.0.75/23"],"mac_address":"0a:58:0a:80:00:4b","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/16","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.75/23","gateway_ip":"10.128.0.1","role":"primary"}}' k8s.v1.cni.cncf.io/network-status: |- [{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.75" ], "mac": "0a:58:0a:80:00:4b", "default": true, "dns": {} }] creationTimestamp: "2026-03-08T03:19:27Z" labels: app: installer managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:labels: .: {} f:app: {} f:ownerReferences: .: {} k:{"uid":"3df63cc6-dae4-4ae5-a602-08eacd8a6f5a"}: {} f:spec: f:automountServiceAccountToken: {} f:containers: k:{"name":"installer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"NODE_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:limits: .: {} f:cpu: {} f:memory: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:runAsUser: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/lock"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}: .: {} f:mountPath: {} f:name: {} f:readOnly: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:nodeName: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: .: {} f:runAsUser: {} f:serviceAccount: {} f:serviceAccountName: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"kube-api-access"}: .: {} f:name: {} f:projected: .: {} f:defaultMode: {} f:sources: {} k:{"name":"kubelet-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"var-lock"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: cluster-kube-apiserver-operator operation: Update time: "2026-03-08T03:19:27Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:k8s.ovn.org/pod-networks: {} manager: master-0 operation: Update subresource: status time: "2026-03-08T03:19:27Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: f:k8s.v1.cni.cncf.io/network-status: {} manager: multus-daemon operation: Update subresource: status time: "2026-03-08T03:19:28Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:phase: {} f:podIP: {} f:podIPs: .: {} k:{"ip":"10.128.0.75"}: .: {} f:ip: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2026-03-08T03:21:36Z" name: installer-2-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 kind: ConfigMap name: revision-status-2 uid: 3df63cc6-dae4-4ae5-a602-08eacd8a6f5a resourceVersion: "11095" uid: 1e34eb33-c8a4-4207-b78c-53e0cfc09e91 spec: automountServiceAccountToken: false containers: - args: - -v=2 - --revision=2 - --namespace=openshift-kube-apiserver - --pod=kube-apiserver-pod - --resource-dir=/etc/kubernetes/static-pod-resources - --pod-manifest-dir=/etc/kubernetes/manifests - --configmaps=kube-apiserver-pod - --configmaps=config - --configmaps=kube-apiserver-cert-syncer-kubeconfig - --optional-configmaps=oauth-metadata - --optional-configmaps=cloud-config - --configmaps=bound-sa-token-signing-certs - --configmaps=etcd-serving-ca - --optional-configmaps=kube-apiserver-server-ca - --configmaps=kubelet-serving-ca - --configmaps=sa-token-signing-certs - --configmaps=kube-apiserver-audit-policies - --secrets=etcd-client - --optional-secrets=encryption-config - --secrets=localhost-recovery-serving-certkey - --secrets=localhost-recovery-client-token - --optional-secrets=webhook-authenticator - --cert-dir=/etc/kubernetes/static-pod-resources/kube-apiserver-certs - --cert-configmaps=aggregator-client-ca - --cert-configmaps=client-ca - --optional-cert-configmaps=trusted-ca-bundle - --cert-configmaps=control-plane-node-kubeconfig - --cert-configmaps=check-endpoints-kubeconfig - --cert-secrets=aggregator-client - --cert-secrets=localhost-serving-cert-certkey - --cert-secrets=service-network-serving-certkey - --cert-secrets=external-loadbalancer-serving-certkey - --cert-secrets=internal-loadbalancer-serving-certkey - --cert-secrets=bound-service-account-signing-key - --cert-secrets=control-plane-node-admin-client-cert-key - --cert-secrets=check-endpoints-client-cert-key - --cert-secrets=kubelet-client - --cert-secrets=node-kubeconfigs - --optional-cert-secrets=user-serving-cert - --optional-cert-secrets=user-serving-cert-000 - --optional-cert-secrets=user-serving-cert-001 - --optional-cert-secrets=user-serving-cert-002 - --optional-cert-secrets=user-serving-cert-003 - --optional-cert-secrets=user-serving-cert-004 - --optional-cert-secrets=user-serving-cert-005 - --optional-cert-secrets=user-serving-cert-006 - --optional-cert-secrets=user-serving-cert-007 - --optional-cert-secrets=user-serving-cert-008 - --optional-cert-secrets=user-serving-cert-009 command: - cluster-kube-apiserver-operator - installer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imagePullPolicy: IfNotPresent name: installer resources: limits: cpu: 150m memory: 200M requests: cpu: 150m memory: 200M securityContext: privileged: true runAsUser: 0 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true - mountPath: /var/lock name: var-lock dnsPolicy: ClusterFirst enableServiceLinks: true imagePullSecrets: - name: installer-sa-dockercfg-gcdsk nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Never schedulerName: default-scheduler securityContext: runAsUser: 0 serviceAccount: installer-sa serviceAccountName: installer-sa terminationGracePeriodSeconds: 30 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/ type: "" name: kubelet-dir - hostPath: path: /var/lock type: "" name: var-lock - name: kube-api-access projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3600 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace status: conditions: - lastProbeTime: null lastTransitionTime: "2026-03-08T03:21:18Z" status: "False" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2026-03-08T03:19:27Z" status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2026-03-08T03:21:18Z" reason: PodFailed status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2026-03-08T03:21:18Z" reason: PodFailed status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2026-03-08T03:19:27Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://81690b2979176695e8530d71a981c729b1a1352c3403e63acd375ec96d2ffadd image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 lastState: {} name: installer ready: false restartCount: 0 started: false state: terminated: containerID: cri-o://81690b2979176695e8530d71a981c729b1a1352c3403e63acd375ec96d2ffadd exitCode: 1 finishedAt: "2026-03-08T03:20:13Z" message: | ving-cert", (string) (len=21) "user-serving-cert-000", (string) (len=21) "user-serving-cert-001", (string) (len=21) "user-serving-cert-002", (string) (len=21) "user-serving-cert-003", (string) (len=21) "user-serving-cert-004", (string) (len=21) "user-serving-cert-005", (string) (len=21) "user-serving-cert-006", (string) (len=21) "user-serving-cert-007", (string) (len=21) "user-serving-cert-008", (string) (len=21) "user-serving-cert-009" }, CertConfigMapNamePrefixes: ([]string) (len=4 cap=4) { (string) (len=20) "aggregator-client-ca", (string) (len=9) "client-ca", (string) (len=29) "control-plane-node-kubeconfig", (string) (len=26) "check-endpoints-kubeconfig" }, OptionalCertConfigMapNamePrefixes: ([]string) (len=1 cap=1) { (string) (len=17) "trusted-ca-bundle" }, CertDir: (string) (len=57) "/etc/kubernetes/static-pod-resources/kube-apiserver-certs", ResourceDir: (string) (len=36) "/etc/kubernetes/static-pod-resources", PodManifestDir: (string) (len=25) "/etc/kubernetes/manifests", Timeout: (time.Duration) 2m0s, StaticPodManifestsLockFile: (string) "", PodMutationFns: ([]installerpod.PodMutationFunc) , KubeletVersion: (string) "" }) I0308 03:19:29.841043 1 cmd.go:413] Getting controller reference for node master-0 I0308 03:19:29.855186 1 cmd.go:426] Waiting for installer revisions to settle for node master-0 I0308 03:19:29.855260 1 envvar.go:172] "Feature gate default state" feature="WatchListClient" enabled=false I0308 03:19:29.855279 1 envvar.go:172] "Feature gate default state" feature="InformerResourceVersion" enabled=false I0308 03:19:29.859112 1 cmd.go:518] Waiting additional period after revisions have settled for node master-0 I0308 03:19:59.859715 1 cmd.go:524] Getting installer pods for node master-0 F0308 03:20:13.862843 1 cmd.go:109] Get "https://172.30.0.1:443/api/v1/namespaces/openshift-kube-apiserver/pods?labelSelector=app%3Dinstaller": net/http: request canceled (Client.Timeout exceeded while awaiting headers) reason: Error startedAt: "2026-03-08T03:19:28Z" volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true recursiveReadOnly: Disabled - mountPath: /var/lock name: var-lock hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 phase: Failed podIP: 10.128.0.75 podIPs: - ip: 10.128.0.75 qosClass: Guaranteed startTime: "2026-03-08T03:19:27Z" - apiVersion: v1 kind: Pod metadata: annotations: k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.128.0.76/23"],"mac_address":"0a:58:0a:80:00:4c","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/16","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.76/23","gateway_ip":"10.128.0.1","role":"primary"}}' k8s.v1.cni.cncf.io/network-status: |- [{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.76" ], "mac": "0a:58:0a:80:00:4c", "default": true, "dns": {} }] creationTimestamp: "2026-03-08T03:21:42Z" labels: app: installer managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:labels: .: {} f:app: {} f:ownerReferences: .: {} k:{"uid":"f7e7c037-886f-4b4d-af89-cbb2d3e00844"}: {} f:spec: f:automountServiceAccountToken: {} f:containers: k:{"name":"installer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"NODE_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:limits: .: {} f:cpu: {} f:memory: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:runAsUser: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/lock"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}: .: {} f:mountPath: {} f:name: {} f:readOnly: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:nodeName: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: .: {} f:runAsUser: {} f:serviceAccount: {} f:serviceAccountName: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"kube-api-access"}: .: {} f:name: {} f:projected: .: {} f:defaultMode: {} f:sources: {} k:{"name":"kubelet-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"var-lock"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: cluster-kube-apiserver-operator operation: Update time: "2026-03-08T03:21:42Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:k8s.ovn.org/pod-networks: {} manager: master-0 operation: Update subresource: status time: "2026-03-08T03:21:42Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: f:k8s.v1.cni.cncf.io/network-status: {} manager: multus-daemon operation: Update subresource: status time: "2026-03-08T03:21:42Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:phase: {} f:podIP: {} f:podIPs: .: {} k:{"ip":"10.128.0.76"}: .: {} f:ip: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2026-03-08T03:22:43Z" name: installer-3-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 kind: ConfigMap name: revision-status-3 uid: f7e7c037-886f-4b4d-af89-cbb2d3e00844 resourceVersion: "11661" uid: d3035f9a-535f-4d1a-b3a0-02e2511894ff spec: automountServiceAccountToken: false containers: - args: - -v=2 - --revision=3 - --namespace=openshift-kube-apiserver - --pod=kube-apiserver-pod - --resource-dir=/etc/kubernetes/static-pod-resources - --pod-manifest-dir=/etc/kubernetes/manifests - --configmaps=kube-apiserver-pod - --configmaps=config - --configmaps=kube-apiserver-cert-syncer-kubeconfig - --optional-configmaps=oauth-metadata - --optional-configmaps=cloud-config - --configmaps=bound-sa-token-signing-certs - --configmaps=etcd-serving-ca - --optional-configmaps=kube-apiserver-server-ca - --configmaps=kubelet-serving-ca - --configmaps=sa-token-signing-certs - --configmaps=kube-apiserver-audit-policies - --secrets=etcd-client - --optional-secrets=encryption-config - --secrets=localhost-recovery-serving-certkey - --secrets=localhost-recovery-client-token - --optional-secrets=webhook-authenticator - --cert-dir=/etc/kubernetes/static-pod-resources/kube-apiserver-certs - --cert-configmaps=aggregator-client-ca - --cert-configmaps=client-ca - --optional-cert-configmaps=trusted-ca-bundle - --cert-configmaps=control-plane-node-kubeconfig - --cert-configmaps=check-endpoints-kubeconfig - --cert-secrets=aggregator-client - --cert-secrets=localhost-serving-cert-certkey - --cert-secrets=service-network-serving-certkey - --cert-secrets=external-loadbalancer-serving-certkey - --cert-secrets=internal-loadbalancer-serving-certkey - --cert-secrets=bound-service-account-signing-key - --cert-secrets=control-plane-node-admin-client-cert-key - --cert-secrets=check-endpoints-client-cert-key - --cert-secrets=kubelet-client - --cert-secrets=node-kubeconfigs - --optional-cert-secrets=user-serving-cert - --optional-cert-secrets=user-serving-cert-000 - --optional-cert-secrets=user-serving-cert-001 - --optional-cert-secrets=user-serving-cert-002 - --optional-cert-secrets=user-serving-cert-003 - --optional-cert-secrets=user-serving-cert-004 - --optional-cert-secrets=user-serving-cert-005 - --optional-cert-secrets=user-serving-cert-006 - --optional-cert-secrets=user-serving-cert-007 - --optional-cert-secrets=user-serving-cert-008 - --optional-cert-secrets=user-serving-cert-009 command: - cluster-kube-apiserver-operator - installer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imagePullPolicy: IfNotPresent name: installer resources: limits: cpu: 150m memory: 200M requests: cpu: 150m memory: 200M securityContext: privileged: true runAsUser: 0 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true - mountPath: /var/lock name: var-lock dnsPolicy: ClusterFirst enableServiceLinks: true imagePullSecrets: - name: installer-sa-dockercfg-gcdsk nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Never schedulerName: default-scheduler securityContext: runAsUser: 0 serviceAccount: installer-sa serviceAccountName: installer-sa terminationGracePeriodSeconds: 30 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/ type: "" name: kubelet-dir - hostPath: path: /var/lock type: "" name: var-lock - name: kube-api-access projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3600 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace status: conditions: - lastProbeTime: null lastTransitionTime: "2026-03-08T03:22:23Z" status: "False" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2026-03-08T03:21:42Z" reason: PodCompleted status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2026-03-08T03:22:21Z" reason: PodCompleted status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2026-03-08T03:22:21Z" reason: PodCompleted status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2026-03-08T03:21:42Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://d621d2f82abc5387792939e7e66660c43e46a86bde107e627d0911e7d0c6b86c image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 lastState: {} name: installer ready: false restartCount: 0 started: false state: terminated: containerID: cri-o://d621d2f82abc5387792939e7e66660c43e46a86bde107e627d0911e7d0c6b86c exitCode: 0 finishedAt: "2026-03-08T03:22:21Z" reason: Completed startedAt: "2026-03-08T03:21:43Z" volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true recursiveReadOnly: Disabled - mountPath: /var/lock name: var-lock hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 phase: Succeeded podIP: 10.128.0.76 podIPs: - ip: 10.128.0.76 qosClass: Guaranteed startTime: "2026-03-08T03:21:42Z" - apiVersion: v1 kind: Pod metadata: annotations: k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.128.0.106/23"],"mac_address":"0a:58:0a:80:00:6a","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/16","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.106/23","gateway_ip":"10.128.0.1","role":"primary"}}' k8s.v1.cni.cncf.io/network-status: |- [{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.106" ], "mac": "0a:58:0a:80:00:6a", "default": true, "dns": {} }] creationTimestamp: "2026-03-08T03:26:06Z" labels: app: installer managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:labels: .: {} f:app: {} f:ownerReferences: .: {} k:{"uid":"e25bc5c4-1bbf-4fb1-bb89-3e30382693ea"}: {} f:spec: f:automountServiceAccountToken: {} f:containers: k:{"name":"installer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"NODE_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:limits: .: {} f:cpu: {} f:memory: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:runAsUser: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/lock"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}: .: {} f:mountPath: {} f:name: {} f:readOnly: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:nodeName: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: .: {} f:runAsUser: {} f:serviceAccount: {} f:serviceAccountName: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"kube-api-access"}: .: {} f:name: {} f:projected: .: {} f:defaultMode: {} f:sources: {} k:{"name":"kubelet-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"var-lock"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: cluster-kube-apiserver-operator operation: Update time: "2026-03-08T03:26:06Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:k8s.ovn.org/pod-networks: {} manager: master-0 operation: Update subresource: status time: "2026-03-08T03:26:06Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: f:k8s.v1.cni.cncf.io/network-status: {} manager: multus-daemon operation: Update subresource: status time: "2026-03-08T03:26:15Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:phase: {} f:podIP: {} f:podIPs: .: {} k:{"ip":"10.128.0.106"}: .: {} f:ip: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2026-03-08T03:27:23Z" name: installer-5-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 kind: ConfigMap name: revision-status-5 uid: e25bc5c4-1bbf-4fb1-bb89-3e30382693ea resourceVersion: "16134" uid: 8517691b-937c-4cde-a7d2-fe18d6b7193d spec: automountServiceAccountToken: false containers: - args: - -v=2 - --revision=5 - --namespace=openshift-kube-apiserver - --pod=kube-apiserver-pod - --resource-dir=/etc/kubernetes/static-pod-resources - --pod-manifest-dir=/etc/kubernetes/manifests - --configmaps=kube-apiserver-pod - --configmaps=config - --configmaps=kube-apiserver-cert-syncer-kubeconfig - --optional-configmaps=oauth-metadata - --optional-configmaps=cloud-config - --configmaps=bound-sa-token-signing-certs - --configmaps=etcd-serving-ca - --optional-configmaps=kube-apiserver-server-ca - --configmaps=kubelet-serving-ca - --configmaps=sa-token-signing-certs - --configmaps=kube-apiserver-audit-policies - --secrets=etcd-client - --optional-secrets=encryption-config - --secrets=localhost-recovery-serving-certkey - --secrets=localhost-recovery-client-token - --optional-secrets=webhook-authenticator - --cert-dir=/etc/kubernetes/static-pod-resources/kube-apiserver-certs - --cert-configmaps=aggregator-client-ca - --cert-configmaps=client-ca - --optional-cert-configmaps=trusted-ca-bundle - --cert-configmaps=control-plane-node-kubeconfig - --cert-configmaps=check-endpoints-kubeconfig - --cert-secrets=aggregator-client - --cert-secrets=localhost-serving-cert-certkey - --cert-secrets=service-network-serving-certkey - --cert-secrets=external-loadbalancer-serving-certkey - --cert-secrets=internal-loadbalancer-serving-certkey - --cert-secrets=bound-service-account-signing-key - --cert-secrets=control-plane-node-admin-client-cert-key - --cert-secrets=check-endpoints-client-cert-key - --cert-secrets=kubelet-client - --cert-secrets=node-kubeconfigs - --optional-cert-secrets=user-serving-cert - --optional-cert-secrets=user-serving-cert-000 - --optional-cert-secrets=user-serving-cert-001 - --optional-cert-secrets=user-serving-cert-002 - --optional-cert-secrets=user-serving-cert-003 - --optional-cert-secrets=user-serving-cert-004 - --optional-cert-secrets=user-serving-cert-005 - --optional-cert-secrets=user-serving-cert-006 - --optional-cert-secrets=user-serving-cert-007 - --optional-cert-secrets=user-serving-cert-008 - --optional-cert-secrets=user-serving-cert-009 command: - cluster-kube-apiserver-operator - installer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imagePullPolicy: IfNotPresent name: installer resources: limits: cpu: 150m memory: 200M requests: cpu: 150m memory: 200M securityContext: privileged: true runAsUser: 0 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true - mountPath: /var/lock name: var-lock dnsPolicy: ClusterFirst enableServiceLinks: true imagePullSecrets: - name: installer-sa-dockercfg-gcdsk nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Never schedulerName: default-scheduler securityContext: runAsUser: 0 serviceAccount: installer-sa serviceAccountName: installer-sa terminationGracePeriodSeconds: 30 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/ type: "" name: kubelet-dir - hostPath: path: /var/lock type: "" name: var-lock - name: kube-api-access projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3600 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace status: conditions: - lastProbeTime: null lastTransitionTime: "2026-03-08T03:27:10Z" status: "False" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2026-03-08T03:26:06Z" reason: PodCompleted status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2026-03-08T03:27:09Z" reason: PodCompleted status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2026-03-08T03:27:09Z" reason: PodCompleted status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2026-03-08T03:26:06Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://3899d101e408c7a23497b90981b3128c53aac06bc31d8fc4ebd6216130e8cdcd image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 lastState: {} name: installer ready: false restartCount: 0 started: false state: terminated: containerID: cri-o://3899d101e408c7a23497b90981b3128c53aac06bc31d8fc4ebd6216130e8cdcd exitCode: 0 finishedAt: "2026-03-08T03:27:09Z" reason: Completed startedAt: "2026-03-08T03:26:16Z" volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true recursiveReadOnly: Disabled - mountPath: /var/lock name: var-lock hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 phase: Succeeded podIP: 10.128.0.106 podIPs: - ip: 10.128.0.106 qosClass: Guaranteed startTime: "2026-03-08T03:26:06Z" - apiVersion: v1 kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: kube-apiserver kubernetes.io/config.hash: 36d4251d3504cdc0ec85144c1379056c kubernetes.io/config.mirror: 36d4251d3504cdc0ec85144c1379056c kubernetes.io/config.seen: "2026-03-08T03:27:03.948325878Z" kubernetes.io/config.source: file target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' creationTimestamp: "2026-03-08T03:27:23Z" labels: apiserver: "true" app: openshift-kube-apiserver revision: "5" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:kubectl.kubernetes.io/default-container: {} f:kubernetes.io/config.hash: {} f:kubernetes.io/config.mirror: {} f:kubernetes.io/config.seen: {} f:kubernetes.io/config.source: {} f:target.workload.openshift.io/management: {} f:labels: .: {} f:apiserver: {} f:app: {} f:revision: {} f:ownerReferences: .: {} k:{"uid":"b4664360-7a98-4e67-95a6-3e39a3e2a113"}: {} f:spec: f:containers: k:{"name":"kube-apiserver"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"GOGC"}: .: {} f:name: {} f:value: {} k:{"name":"HOST_IP"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAMESPACE"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"STATIC_POD_VERSION"}: .: {} f:name: {} f:value: {} f:image: {} f:imagePullPolicy: {} f:livenessProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:name: {} f:ports: .: {} k:{"containerPort":6443,"protocol":"TCP"}: .: {} f:containerPort: {} f:hostPort: {} f:protocol: {} f:readinessProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:startupProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/static-pod-certs"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/etc/kubernetes/static-pod-resources"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/log/kube-apiserver"}: .: {} f:mountPath: {} f:name: {} k:{"name":"kube-apiserver-cert-regeneration-controller"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"POD_NAMESPACE"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/static-pod-resources"}: .: {} f:mountPath: {} f:name: {} k:{"name":"kube-apiserver-cert-syncer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAMESPACE"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/static-pod-certs"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/etc/kubernetes/static-pod-resources"}: .: {} f:mountPath: {} f:name: {} k:{"name":"kube-apiserver-check-endpoints"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAMESPACE"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:livenessProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:initialDelaySeconds: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:name: {} f:ports: .: {} k:{"containerPort":17697,"protocol":"TCP"}: .: {} f:containerPort: {} f:hostPort: {} f:name: {} f:protocol: {} f:readinessProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:initialDelaySeconds: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/static-pod-certs"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/etc/kubernetes/static-pod-resources"}: .: {} f:mountPath: {} f:name: {} k:{"name":"kube-apiserver-insecure-readyz"}: .: {} f:args: {} f:command: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:ports: .: {} k:{"containerPort":6080,"protocol":"TCP"}: .: {} f:containerPort: {} f:hostPort: {} f:protocol: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:hostNetwork: {} f:initContainers: .: {} k:{"name":"setup"}: .: {} f:args: {} f:command: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/var/log/kube-apiserver"}: .: {} f:mountPath: {} f:name: {} f:nodeName: {} f:priority: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"audit-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"cert-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"resource-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: kubelet operation: Update time: "2026-03-08T03:27:23Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:initContainerStatuses: {} f:phase: {} f:podIP: {} f:podIPs: .: {} k:{"ip":"192.168.32.10"}: .: {} f:ip: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2026-03-08T03:27:41Z" name: kube-apiserver-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 controller: true kind: Node name: master-0 uid: b4664360-7a98-4e67-95a6-3e39a3e2a113 resourceVersion: "16232" uid: 80282202-3305-4e26-ad5c-deb34cad07da spec: containers: - args: - | LOCK=/var/log/kube-apiserver/.lock # We should be able to acquire the lock immediatelly. If not, it means the init container has not released it yet and kubelet or CRI-O started container prematurely. exec {LOCK_FD}>${LOCK} && flock --verbose -w 30 "${LOCK_FD}" || { echo "Failed to acquire lock for kube-apiserver. Please check setup container for details. This is likely kubelet or CRI-O bug." exit 1 } if [ -f /etc/kubernetes/static-pod-certs/configmaps/trusted-ca-bundle/ca-bundle.crt ]; then echo "Copying system trust bundle ..." cp -f /etc/kubernetes/static-pod-certs/configmaps/trusted-ca-bundle/ca-bundle.crt /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem fi exec watch-termination --termination-touch-file=/var/log/kube-apiserver/.terminating --termination-log-file=/var/log/kube-apiserver/termination.log --graceful-termination-duration=15s --kubeconfig=/etc/kubernetes/static-pod-resources/configmaps/kube-apiserver-cert-syncer-kubeconfig/kubeconfig -- hyperkube kube-apiserver --openshift-config=/etc/kubernetes/static-pod-resources/configmaps/config/config.yaml --advertise-address=${HOST_IP} -v=2 --permit-address-sharing command: - /bin/bash - -ec env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: STATIC_POD_VERSION value: "5" - name: HOST_IP valueFrom: fieldRef: apiVersion: v1 fieldPath: status.hostIP - name: GOGC value: "100" image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8fca00eb71b1f03e5b5180a66f3871f5626d337b56196622f5842cfc165523b4 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 httpGet: path: livez?exclude=etcd port: 6443 scheme: HTTPS periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 name: kube-apiserver ports: - containerPort: 6443 hostPort: 6443 protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: readyz port: 6443 scheme: HTTPS periodSeconds: 5 successThreshold: 1 timeoutSeconds: 10 resources: requests: cpu: 265m memory: 1Gi securityContext: privileged: true startupProbe: failureThreshold: 30 httpGet: path: livez port: 6443 scheme: HTTPS periodSeconds: 5 successThreshold: 1 timeoutSeconds: 10 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/static-pod-resources name: resource-dir - mountPath: /etc/kubernetes/static-pod-certs name: cert-dir - mountPath: /var/log/kube-apiserver name: audit-dir - args: - --kubeconfig=/etc/kubernetes/static-pod-resources/configmaps/kube-apiserver-cert-syncer-kubeconfig/kubeconfig - --namespace=$(POD_NAMESPACE) - --destination-dir=/etc/kubernetes/static-pod-certs command: - cluster-kube-apiserver-operator - cert-syncer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imagePullPolicy: IfNotPresent name: kube-apiserver-cert-syncer resources: requests: cpu: 5m memory: 50Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/static-pod-resources name: resource-dir - mountPath: /etc/kubernetes/static-pod-certs name: cert-dir - args: - --kubeconfig=/etc/kubernetes/static-pod-resources/configmaps/kube-apiserver-cert-syncer-kubeconfig/kubeconfig - --namespace=$(POD_NAMESPACE) - -v=2 command: - cluster-kube-apiserver-operator - cert-regeneration-controller env: - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imagePullPolicy: IfNotPresent name: kube-apiserver-cert-regeneration-controller resources: requests: cpu: 5m memory: 50Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/static-pod-resources name: resource-dir - args: - --insecure-port=6080 - --delegate-url=https://localhost:6443/readyz command: - cluster-kube-apiserver-operator - insecure-readyz image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imagePullPolicy: IfNotPresent name: kube-apiserver-insecure-readyz ports: - containerPort: 6080 hostPort: 6080 protocol: TCP resources: requests: cpu: 5m memory: 50Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError - args: - --kubeconfig - /etc/kubernetes/static-pod-certs/configmaps/check-endpoints-kubeconfig/kubeconfig - --listen - 0.0.0.0:17697 - --namespace - $(POD_NAMESPACE) - --v - "2" command: - cluster-kube-apiserver-operator - check-endpoints env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 httpGet: path: healthz port: 17697 scheme: HTTPS initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 name: kube-apiserver-check-endpoints ports: - containerPort: 17697 hostPort: 17697 name: check-endpoints protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: healthz port: 17697 scheme: HTTPS initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 resources: requests: cpu: 10m memory: 50Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/static-pod-resources name: resource-dir - mountPath: /etc/kubernetes/static-pod-certs name: cert-dir dnsPolicy: ClusterFirst enableServiceLinks: true hostNetwork: true initContainers: - args: - | echo "Fixing audit permissions ..." chmod 0700 /var/log/kube-apiserver && touch /var/log/kube-apiserver/audit.log && chmod 0600 /var/log/kube-apiserver/* LOCK=/var/log/kube-apiserver/.lock echo "Acquiring exclusive lock ${LOCK} ..." # Waiting for 15s max for old kube-apiserver's watch-termination process to exit and remove the lock. # Two cases: # 1. if kubelet does not start the old and new in parallel (i.e. works as expected), the flock will always succeed without any time. # 2. if kubelet does overlap old and new pods for up to 130s, the flock will wait and immediate return when the old finishes. # # NOTE: We can increase 15s for a bigger expected overlap. But a higher value means less noise about the broken kubelet behaviour, i.e. we hide a bug. # NOTE: Do not tweak these timings without considering the livenessProbe initialDelaySeconds exec {LOCK_FD}>${LOCK} && flock --verbose -w 15 "${LOCK_FD}" || { echo "$(date -Iseconds -u) kubelet did not terminate old kube-apiserver before new one" >> /var/log/kube-apiserver/lock.log echo -n ": WARNING: kubelet did not terminate old kube-apiserver before new one." # We failed to acquire exclusive lock, which means there is old kube-apiserver running in system. # Since we utilize SO_REUSEPORT, we need to make sure the old kube-apiserver stopped listening. # # NOTE: This is a fallback for broken kubelet, if you observe this please report a bug. echo -n "Waiting for port 6443 to be released due to likely bug in kubelet or CRI-O " while [ -n "$(ss -Htan state listening '( sport = 6443 or sport = 6080 )')" ]; do echo -n "." sleep 1 (( tries += 1 )) if [[ "${tries}" -gt 10 ]]; then echo "Timed out waiting for port :6443 and :6080 to be released, this is likely a bug in kubelet or CRI-O" exit 1 fi done # This is to make sure the server has terminated independently from the lock. # After the port has been freed (requests can be pending and need 60s max). sleep 65 } # We cannot hold the lock from the init container to the main container. We release it here. There is no risk, at this point we know we are safe. flock -u "${LOCK_FD}" command: - /usr/bin/timeout - "100" - /bin/bash - -ec image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8fca00eb71b1f03e5b5180a66f3871f5626d337b56196622f5842cfc165523b4 imagePullPolicy: IfNotPresent name: setup resources: requests: cpu: 5m memory: 50Mi securityContext: privileged: true terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /var/log/kube-apiserver name: audit-dir nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Always schedulerName: default-scheduler securityContext: {} terminationGracePeriodSeconds: 15 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/static-pod-resources/kube-apiserver-pod-5 type: "" name: resource-dir - hostPath: path: /etc/kubernetes/static-pod-resources/kube-apiserver-certs type: "" name: cert-dir - hostPath: path: /var/log/kube-apiserver type: "" name: audit-dir status: conditions: - lastProbeTime: null lastTransitionTime: "2026-03-08T03:27:41Z" status: "True" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2026-03-08T03:27:41Z" status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2026-03-08T03:27:41Z" status: "True" type: Ready - lastProbeTime: null lastTransitionTime: "2026-03-08T03:27:41Z" status: "True" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2026-03-08T03:27:41Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://6a33281165537aeef328d69c6a24ccbd0c5ea48d1c2e24ffab9dd994662f052e image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8fca00eb71b1f03e5b5180a66f3871f5626d337b56196622f5842cfc165523b4 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8fca00eb71b1f03e5b5180a66f3871f5626d337b56196622f5842cfc165523b4 lastState: {} name: kube-apiserver ready: true restartCount: 0 started: true state: running: startedAt: "2026-03-08T03:27:16Z" - containerID: cri-o://ce8f427b8bf34ec53df435c60b07523ab5967605a19bfad31e9b12766825b4b6 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 lastState: {} name: kube-apiserver-cert-regeneration-controller ready: true restartCount: 0 started: true state: running: startedAt: "2026-03-08T03:27:17Z" - containerID: cri-o://9f0d21fb91fc4fd0ea0964241a4c87139cbec4f45ceccefbc4631309fa6444b4 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 lastState: {} name: kube-apiserver-cert-syncer ready: true restartCount: 0 started: true state: running: startedAt: "2026-03-08T03:27:17Z" - containerID: cri-o://2ef103dc7a8b8350771a738faee46c1b0b6a4c5e282113f0ad2646f21b84ba5d image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 lastState: {} name: kube-apiserver-check-endpoints ready: true restartCount: 0 started: true state: running: startedAt: "2026-03-08T03:27:17Z" - containerID: cri-o://87f6b455974537e41e8286c21c6338dab8837fafb38bede878aaff183d9c01de image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:5500329ab50804678fb8a90b96bf2a469bca16b620fb6dd2f5f5a17106e94898 lastState: {} name: kube-apiserver-insecure-readyz ready: true restartCount: 0 started: true state: running: startedAt: "2026-03-08T03:27:17Z" hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 initContainerStatuses: - containerID: cri-o://683836e41f92176912619eb6417e5ca89ed583121b54209e191375b1cb98561e image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8fca00eb71b1f03e5b5180a66f3871f5626d337b56196622f5842cfc165523b4 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:8fca00eb71b1f03e5b5180a66f3871f5626d337b56196622f5842cfc165523b4 lastState: {} name: setup ready: true restartCount: 0 started: false state: terminated: containerID: cri-o://683836e41f92176912619eb6417e5ca89ed583121b54209e191375b1cb98561e exitCode: 0 finishedAt: "2026-03-08T03:27:15Z" reason: Completed startedAt: "2026-03-08T03:27:15Z" phase: Running podIP: 192.168.32.10 podIPs: - ip: 192.168.32.10 qosClass: Burstable startTime: "2026-03-08T03:27:41Z" kind: PodList metadata: resourceVersion: "48176"