--- apiVersion: v1 items: - apiVersion: v1 kind: Pod metadata: annotations: k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.128.0.53/23"],"mac_address":"0a:58:0a:80:00:35","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/16","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.53/23","gateway_ip":"10.128.0.1","role":"primary"}}' k8s.v1.cni.cncf.io/network-status: |- [{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.53" ], "mac": "0a:58:0a:80:00:35", "default": true, "dns": {} }] creationTimestamp: "2026-03-19T09:21:21Z" labels: app: installer managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:labels: .: {} f:app: {} f:ownerReferences: .: {} k:{"uid":"226ceab8-a71c-4ab7-b11f-6de6658ec6f3"}: {} f:spec: f:automountServiceAccountToken: {} f:containers: k:{"name":"installer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"NODE_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:limits: .: {} f:cpu: {} f:memory: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:runAsUser: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/lock"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}: .: {} f:mountPath: {} f:name: {} f:readOnly: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:nodeName: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: .: {} f:runAsUser: {} f:serviceAccount: {} f:serviceAccountName: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"kube-api-access"}: .: {} f:name: {} f:projected: .: {} f:defaultMode: {} f:sources: {} k:{"name":"kubelet-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"var-lock"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: cluster-kube-apiserver-operator operation: Update time: "2026-03-19T09:21:21Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:k8s.ovn.org/pod-networks: {} manager: master-0 operation: Update subresource: status time: "2026-03-19T09:21:21Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: f:k8s.v1.cni.cncf.io/network-status: {} manager: multus-daemon operation: Update subresource: status time: "2026-03-19T09:21:22Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:phase: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2026-03-19T09:23:29Z" name: installer-1-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 kind: ConfigMap name: revision-status-1 uid: 226ceab8-a71c-4ab7-b11f-6de6658ec6f3 resourceVersion: "10649" uid: 0df23b55-3dea-4f5e-9d53-5c7755ea4e48 spec: automountServiceAccountToken: false containers: - args: - -v=2 - --revision=1 - --namespace=openshift-kube-apiserver - --pod=kube-apiserver-pod - --resource-dir=/etc/kubernetes/static-pod-resources - --pod-manifest-dir=/etc/kubernetes/manifests - --configmaps=kube-apiserver-pod - --configmaps=config - --configmaps=kube-apiserver-cert-syncer-kubeconfig - --optional-configmaps=oauth-metadata - --optional-configmaps=cloud-config - --configmaps=bound-sa-token-signing-certs - --configmaps=etcd-serving-ca - --optional-configmaps=kube-apiserver-server-ca - --configmaps=kubelet-serving-ca - --configmaps=sa-token-signing-certs - --configmaps=kube-apiserver-audit-policies - --secrets=etcd-client - --optional-secrets=encryption-config - --secrets=localhost-recovery-serving-certkey - --secrets=localhost-recovery-client-token - --optional-secrets=webhook-authenticator - --cert-dir=/etc/kubernetes/static-pod-resources/kube-apiserver-certs - --cert-configmaps=aggregator-client-ca - --cert-configmaps=client-ca - --optional-cert-configmaps=trusted-ca-bundle - --cert-configmaps=control-plane-node-kubeconfig - --cert-configmaps=check-endpoints-kubeconfig - --cert-secrets=aggregator-client - --cert-secrets=localhost-serving-cert-certkey - --cert-secrets=service-network-serving-certkey - --cert-secrets=external-loadbalancer-serving-certkey - --cert-secrets=internal-loadbalancer-serving-certkey - --cert-secrets=bound-service-account-signing-key - --cert-secrets=control-plane-node-admin-client-cert-key - --cert-secrets=check-endpoints-client-cert-key - --cert-secrets=kubelet-client - --cert-secrets=node-kubeconfigs - --optional-cert-secrets=user-serving-cert - --optional-cert-secrets=user-serving-cert-000 - --optional-cert-secrets=user-serving-cert-001 - --optional-cert-secrets=user-serving-cert-002 - --optional-cert-secrets=user-serving-cert-003 - --optional-cert-secrets=user-serving-cert-004 - --optional-cert-secrets=user-serving-cert-005 - --optional-cert-secrets=user-serving-cert-006 - --optional-cert-secrets=user-serving-cert-007 - --optional-cert-secrets=user-serving-cert-008 - --optional-cert-secrets=user-serving-cert-009 command: - cluster-kube-apiserver-operator - installer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imagePullPolicy: IfNotPresent name: installer resources: limits: cpu: 150m memory: 200M requests: cpu: 150m memory: 200M securityContext: privileged: true runAsUser: 0 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true - mountPath: /var/lock name: var-lock dnsPolicy: ClusterFirst enableServiceLinks: true nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Never schedulerName: default-scheduler securityContext: runAsUser: 0 serviceAccount: installer-sa serviceAccountName: installer-sa terminationGracePeriodSeconds: 30 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/ type: "" name: kubelet-dir - hostPath: path: /var/lock type: "" name: var-lock - name: kube-api-access projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3600 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace status: conditions: - lastProbeTime: null lastTransitionTime: "2026-03-19T09:22:41Z" status: "False" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2026-03-19T09:21:21Z" status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2026-03-19T09:22:40Z" reason: PodFailed status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2026-03-19T09:22:40Z" reason: PodFailed status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2026-03-19T09:21:21Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://cbac5fecef5ccbfed911c8dc762330e4e21b1d157632cde1feee52ece3850c21 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 lastState: {} name: installer ready: false restartCount: 0 started: false state: terminated: containerID: cri-o://cbac5fecef5ccbfed911c8dc762330e4e21b1d157632cde1feee52ece3850c21 exitCode: 1 finishedAt: "2026-03-19T09:22:07Z" message: | ving-cert", (string) (len=21) "user-serving-cert-000", (string) (len=21) "user-serving-cert-001", (string) (len=21) "user-serving-cert-002", (string) (len=21) "user-serving-cert-003", (string) (len=21) "user-serving-cert-004", (string) (len=21) "user-serving-cert-005", (string) (len=21) "user-serving-cert-006", (string) (len=21) "user-serving-cert-007", (string) (len=21) "user-serving-cert-008", (string) (len=21) "user-serving-cert-009" }, CertConfigMapNamePrefixes: ([]string) (len=4 cap=4) { (string) (len=20) "aggregator-client-ca", (string) (len=9) "client-ca", (string) (len=29) "control-plane-node-kubeconfig", (string) (len=26) "check-endpoints-kubeconfig" }, OptionalCertConfigMapNamePrefixes: ([]string) (len=1 cap=1) { (string) (len=17) "trusted-ca-bundle" }, CertDir: (string) (len=57) "/etc/kubernetes/static-pod-resources/kube-apiserver-certs", ResourceDir: (string) (len=36) "/etc/kubernetes/static-pod-resources", PodManifestDir: (string) (len=25) "/etc/kubernetes/manifests", Timeout: (time.Duration) 2m0s, StaticPodManifestsLockFile: (string) "", PodMutationFns: ([]installerpod.PodMutationFunc) , KubeletVersion: (string) "" }) I0319 09:21:23.206137 1 cmd.go:413] Getting controller reference for node master-0 I0319 09:21:23.219311 1 cmd.go:426] Waiting for installer revisions to settle for node master-0 I0319 09:21:23.219376 1 envvar.go:172] "Feature gate default state" feature="WatchListClient" enabled=false I0319 09:21:23.219389 1 envvar.go:172] "Feature gate default state" feature="InformerResourceVersion" enabled=false I0319 09:21:23.299806 1 cmd.go:518] Waiting additional period after revisions have settled for node master-0 I0319 09:21:53.300687 1 cmd.go:524] Getting installer pods for node master-0 F0319 09:22:07.301897 1 cmd.go:109] Get "https://172.30.0.1:443/api/v1/namespaces/openshift-kube-apiserver/pods?labelSelector=app%3Dinstaller": net/http: request canceled (Client.Timeout exceeded while awaiting headers) reason: Error startedAt: "2026-03-19T09:21:22Z" volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true recursiveReadOnly: Disabled - mountPath: /var/lock name: var-lock hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 phase: Failed qosClass: Guaranteed startTime: "2026-03-19T09:21:21Z" - apiVersion: v1 kind: Pod metadata: annotations: k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.128.0.58/23"],"mac_address":"0a:58:0a:80:00:3a","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/16","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.58/23","gateway_ip":"10.128.0.1","role":"primary"}}' k8s.v1.cni.cncf.io/network-status: |- [{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.58" ], "mac": "0a:58:0a:80:00:3a", "default": true, "dns": {} }] creationTimestamp: "2026-03-19T09:23:46Z" labels: app: installer managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:labels: .: {} f:app: {} f:ownerReferences: .: {} k:{"uid":"b4361809-28e9-46b5-bb66-397cb2ae6572"}: {} f:spec: f:automountServiceAccountToken: {} f:containers: k:{"name":"installer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"NODE_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:limits: .: {} f:cpu: {} f:memory: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:runAsUser: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/lock"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}: .: {} f:mountPath: {} f:name: {} f:readOnly: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:nodeName: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: .: {} f:runAsUser: {} f:serviceAccount: {} f:serviceAccountName: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"kube-api-access"}: .: {} f:name: {} f:projected: .: {} f:defaultMode: {} f:sources: {} k:{"name":"kubelet-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"var-lock"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: cluster-kube-apiserver-operator operation: Update time: "2026-03-19T09:23:45Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:k8s.ovn.org/pod-networks: {} manager: master-0 operation: Update subresource: status time: "2026-03-19T09:23:46Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: f:k8s.v1.cni.cncf.io/network-status: {} manager: multus-daemon operation: Update subresource: status time: "2026-03-19T09:23:47Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:phase: {} f:podIP: {} f:podIPs: .: {} k:{"ip":"10.128.0.58"}: .: {} f:ip: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2026-03-19T09:25:10Z" name: installer-2-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 kind: ConfigMap name: revision-status-2 uid: b4361809-28e9-46b5-bb66-397cb2ae6572 resourceVersion: "10634" uid: 89be0036-a2c8-48b4-9eaf-17fab972c4f4 spec: automountServiceAccountToken: false containers: - args: - -v=2 - --revision=2 - --namespace=openshift-kube-apiserver - --pod=kube-apiserver-pod - --resource-dir=/etc/kubernetes/static-pod-resources - --pod-manifest-dir=/etc/kubernetes/manifests - --configmaps=kube-apiserver-pod - --configmaps=config - --configmaps=kube-apiserver-cert-syncer-kubeconfig - --optional-configmaps=oauth-metadata - --optional-configmaps=cloud-config - --configmaps=bound-sa-token-signing-certs - --configmaps=etcd-serving-ca - --optional-configmaps=kube-apiserver-server-ca - --configmaps=kubelet-serving-ca - --configmaps=sa-token-signing-certs - --configmaps=kube-apiserver-audit-policies - --secrets=etcd-client - --optional-secrets=encryption-config - --secrets=localhost-recovery-serving-certkey - --secrets=localhost-recovery-client-token - --optional-secrets=webhook-authenticator - --cert-dir=/etc/kubernetes/static-pod-resources/kube-apiserver-certs - --cert-configmaps=aggregator-client-ca - --cert-configmaps=client-ca - --optional-cert-configmaps=trusted-ca-bundle - --cert-configmaps=control-plane-node-kubeconfig - --cert-configmaps=check-endpoints-kubeconfig - --cert-secrets=aggregator-client - --cert-secrets=localhost-serving-cert-certkey - --cert-secrets=service-network-serving-certkey - --cert-secrets=external-loadbalancer-serving-certkey - --cert-secrets=internal-loadbalancer-serving-certkey - --cert-secrets=bound-service-account-signing-key - --cert-secrets=control-plane-node-admin-client-cert-key - --cert-secrets=check-endpoints-client-cert-key - --cert-secrets=kubelet-client - --cert-secrets=node-kubeconfigs - --optional-cert-secrets=user-serving-cert - --optional-cert-secrets=user-serving-cert-000 - --optional-cert-secrets=user-serving-cert-001 - --optional-cert-secrets=user-serving-cert-002 - --optional-cert-secrets=user-serving-cert-003 - --optional-cert-secrets=user-serving-cert-004 - --optional-cert-secrets=user-serving-cert-005 - --optional-cert-secrets=user-serving-cert-006 - --optional-cert-secrets=user-serving-cert-007 - --optional-cert-secrets=user-serving-cert-008 - --optional-cert-secrets=user-serving-cert-009 command: - cluster-kube-apiserver-operator - installer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imagePullPolicy: IfNotPresent name: installer resources: limits: cpu: 150m memory: 200M requests: cpu: 150m memory: 200M securityContext: privileged: true runAsUser: 0 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true - mountPath: /var/lock name: var-lock dnsPolicy: ClusterFirst enableServiceLinks: true imagePullSecrets: - name: installer-sa-dockercfg-w5d24 nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Never schedulerName: default-scheduler securityContext: runAsUser: 0 serviceAccount: installer-sa serviceAccountName: installer-sa terminationGracePeriodSeconds: 30 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/ type: "" name: kubelet-dir - hostPath: path: /var/lock type: "" name: var-lock - name: kube-api-access projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3600 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace status: conditions: - lastProbeTime: null lastTransitionTime: "2026-03-19T09:25:03Z" status: "False" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2026-03-19T09:23:46Z" reason: PodCompleted status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2026-03-19T09:25:02Z" reason: PodCompleted status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2026-03-19T09:25:02Z" reason: PodCompleted status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2026-03-19T09:23:46Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://5d59e82ae91c2ed1c8a992bffe58e7eea15792d208f2b71cb72f5ee7bff4f994 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 lastState: {} name: installer ready: false restartCount: 0 started: false state: terminated: containerID: cri-o://5d59e82ae91c2ed1c8a992bffe58e7eea15792d208f2b71cb72f5ee7bff4f994 exitCode: 0 finishedAt: "2026-03-19T09:24:33Z" reason: Completed startedAt: "2026-03-19T09:23:47Z" volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true recursiveReadOnly: Disabled - mountPath: /var/lock name: var-lock hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 phase: Succeeded podIP: 10.128.0.58 podIPs: - ip: 10.128.0.58 qosClass: Guaranteed startTime: "2026-03-19T09:23:46Z" - apiVersion: v1 kind: Pod metadata: annotations: k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.128.0.96/23"],"mac_address":"0a:58:0a:80:00:60","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/16","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.96/23","gateway_ip":"10.128.0.1","role":"primary"}}' k8s.v1.cni.cncf.io/network-status: |- [{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.96" ], "mac": "0a:58:0a:80:00:60", "default": true, "dns": {} }] creationTimestamp: "2026-03-19T09:27:49Z" labels: app: installer managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:labels: .: {} f:app: {} f:ownerReferences: .: {} k:{"uid":"9992fde9-6b48-4c65-b6dc-2db7b8b0ef26"}: {} f:spec: f:automountServiceAccountToken: {} f:containers: k:{"name":"installer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"NODE_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:limits: .: {} f:cpu: {} f:memory: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:runAsUser: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/lock"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}: .: {} f:mountPath: {} f:name: {} f:readOnly: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:nodeName: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: .: {} f:runAsUser: {} f:serviceAccount: {} f:serviceAccountName: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"kube-api-access"}: .: {} f:name: {} f:projected: .: {} f:defaultMode: {} f:sources: {} k:{"name":"kubelet-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"var-lock"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: cluster-kube-apiserver-operator operation: Update time: "2026-03-19T09:27:49Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:k8s.ovn.org/pod-networks: {} manager: master-0 operation: Update subresource: status time: "2026-03-19T09:27:49Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: f:k8s.v1.cni.cncf.io/network-status: {} manager: multus-daemon operation: Update subresource: status time: "2026-03-19T09:28:06Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:phase: {} f:podIP: {} f:podIPs: .: {} k:{"ip":"10.128.0.96"}: .: {} f:ip: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2026-03-19T09:29:10Z" name: installer-4-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 kind: ConfigMap name: revision-status-4 uid: 9992fde9-6b48-4c65-b6dc-2db7b8b0ef26 resourceVersion: "14395" uid: 86b617be-3cae-4fea-bba7-199de3e4ecf6 spec: automountServiceAccountToken: false containers: - args: - -v=2 - --revision=4 - --namespace=openshift-kube-apiserver - --pod=kube-apiserver-pod - --resource-dir=/etc/kubernetes/static-pod-resources - --pod-manifest-dir=/etc/kubernetes/manifests - --configmaps=kube-apiserver-pod - --configmaps=config - --configmaps=kube-apiserver-cert-syncer-kubeconfig - --optional-configmaps=oauth-metadata - --optional-configmaps=cloud-config - --configmaps=bound-sa-token-signing-certs - --configmaps=etcd-serving-ca - --optional-configmaps=kube-apiserver-server-ca - --configmaps=kubelet-serving-ca - --configmaps=sa-token-signing-certs - --configmaps=kube-apiserver-audit-policies - --secrets=etcd-client - --optional-secrets=encryption-config - --secrets=localhost-recovery-serving-certkey - --secrets=localhost-recovery-client-token - --optional-secrets=webhook-authenticator - --cert-dir=/etc/kubernetes/static-pod-resources/kube-apiserver-certs - --cert-configmaps=aggregator-client-ca - --cert-configmaps=client-ca - --optional-cert-configmaps=trusted-ca-bundle - --cert-configmaps=control-plane-node-kubeconfig - --cert-configmaps=check-endpoints-kubeconfig - --cert-secrets=aggregator-client - --cert-secrets=localhost-serving-cert-certkey - --cert-secrets=service-network-serving-certkey - --cert-secrets=external-loadbalancer-serving-certkey - --cert-secrets=internal-loadbalancer-serving-certkey - --cert-secrets=bound-service-account-signing-key - --cert-secrets=control-plane-node-admin-client-cert-key - --cert-secrets=check-endpoints-client-cert-key - --cert-secrets=kubelet-client - --cert-secrets=node-kubeconfigs - --optional-cert-secrets=user-serving-cert - --optional-cert-secrets=user-serving-cert-000 - --optional-cert-secrets=user-serving-cert-001 - --optional-cert-secrets=user-serving-cert-002 - --optional-cert-secrets=user-serving-cert-003 - --optional-cert-secrets=user-serving-cert-004 - --optional-cert-secrets=user-serving-cert-005 - --optional-cert-secrets=user-serving-cert-006 - --optional-cert-secrets=user-serving-cert-007 - --optional-cert-secrets=user-serving-cert-008 - --optional-cert-secrets=user-serving-cert-009 command: - cluster-kube-apiserver-operator - installer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imagePullPolicy: IfNotPresent name: installer resources: limits: cpu: 150m memory: 200M requests: cpu: 150m memory: 200M securityContext: privileged: true runAsUser: 0 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true - mountPath: /var/lock name: var-lock dnsPolicy: ClusterFirst enableServiceLinks: true imagePullSecrets: - name: installer-sa-dockercfg-w5d24 nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Never schedulerName: default-scheduler securityContext: runAsUser: 0 serviceAccount: installer-sa serviceAccountName: installer-sa terminationGracePeriodSeconds: 30 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/ type: "" name: kubelet-dir - hostPath: path: /var/lock type: "" name: var-lock - name: kube-api-access projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3600 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace status: conditions: - lastProbeTime: null lastTransitionTime: "2026-03-19T09:28:52Z" status: "False" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2026-03-19T09:27:49Z" reason: PodCompleted status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2026-03-19T09:28:51Z" reason: PodCompleted status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2026-03-19T09:28:51Z" reason: PodCompleted status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2026-03-19T09:27:49Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://355cd89078453b5d440100fc82d5481b6a468e22fd22088f1a704fff3b57820e image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 lastState: {} name: installer ready: false restartCount: 0 started: false state: terminated: containerID: cri-o://355cd89078453b5d440100fc82d5481b6a468e22fd22088f1a704fff3b57820e exitCode: 0 finishedAt: "2026-03-19T09:28:50Z" reason: Completed startedAt: "2026-03-19T09:28:07Z" volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true recursiveReadOnly: Disabled - mountPath: /var/lock name: var-lock hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 phase: Succeeded podIP: 10.128.0.96 podIPs: - ip: 10.128.0.96 qosClass: Guaranteed startTime: "2026-03-19T09:27:49Z" - apiVersion: v1 kind: Pod metadata: annotations: k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.128.0.110/23"],"mac_address":"0a:58:0a:80:00:6e","gateway_ips":["10.128.0.1"],"routes":[{"dest":"10.128.0.0/16","nextHop":"10.128.0.1"},{"dest":"172.30.0.0/16","nextHop":"10.128.0.1"},{"dest":"169.254.0.5/32","nextHop":"10.128.0.1"},{"dest":"100.64.0.0/16","nextHop":"10.128.0.1"}],"ip_address":"10.128.0.110/23","gateway_ip":"10.128.0.1","role":"primary"}}' k8s.v1.cni.cncf.io/network-status: |- [{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.0.110" ], "mac": "0a:58:0a:80:00:6e", "default": true, "dns": {} }] creationTimestamp: "2026-03-19T09:35:45Z" labels: app: installer managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:labels: .: {} f:app: {} f:ownerReferences: .: {} k:{"uid":"b8d5bc46-b3e2-4917-9e30-8e07c63baf9b"}: {} f:spec: f:automountServiceAccountToken: {} f:containers: k:{"name":"installer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"NODE_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:limits: .: {} f:cpu: {} f:memory: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:runAsUser: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/lock"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/run/secrets/kubernetes.io/serviceaccount"}: .: {} f:mountPath: {} f:name: {} f:readOnly: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:nodeName: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: .: {} f:runAsUser: {} f:serviceAccount: {} f:serviceAccountName: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"kube-api-access"}: .: {} f:name: {} f:projected: .: {} f:defaultMode: {} f:sources: {} k:{"name":"kubelet-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"var-lock"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: cluster-kube-apiserver-operator operation: Update time: "2026-03-19T09:35:45Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:k8s.ovn.org/pod-networks: {} manager: master-0 operation: Update subresource: status time: "2026-03-19T09:35:45Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: f:k8s.v1.cni.cncf.io/network-status: {} manager: multus-daemon operation: Update subresource: status time: "2026-03-19T09:35:46Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:reason: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:phase: {} f:podIP: {} f:podIPs: .: {} k:{"ip":"10.128.0.110"}: .: {} f:ip: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2026-03-19T09:37:07Z" name: installer-6-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 kind: ConfigMap name: revision-status-6 uid: b8d5bc46-b3e2-4917-9e30-8e07c63baf9b resourceVersion: "16355" uid: 8f348ddf-ee67-4f81-9c16-b35d1a918669 spec: automountServiceAccountToken: false containers: - args: - -v=2 - --revision=6 - --namespace=openshift-kube-apiserver - --pod=kube-apiserver-pod - --resource-dir=/etc/kubernetes/static-pod-resources - --pod-manifest-dir=/etc/kubernetes/manifests - --configmaps=kube-apiserver-pod - --configmaps=config - --configmaps=kube-apiserver-cert-syncer-kubeconfig - --optional-configmaps=oauth-metadata - --optional-configmaps=cloud-config - --configmaps=bound-sa-token-signing-certs - --configmaps=etcd-serving-ca - --optional-configmaps=kube-apiserver-server-ca - --configmaps=kubelet-serving-ca - --configmaps=sa-token-signing-certs - --configmaps=kube-apiserver-audit-policies - --secrets=etcd-client - --optional-secrets=encryption-config - --secrets=localhost-recovery-serving-certkey - --secrets=localhost-recovery-client-token - --optional-secrets=webhook-authenticator - --cert-dir=/etc/kubernetes/static-pod-resources/kube-apiserver-certs - --cert-configmaps=aggregator-client-ca - --cert-configmaps=client-ca - --optional-cert-configmaps=trusted-ca-bundle - --cert-configmaps=control-plane-node-kubeconfig - --cert-configmaps=check-endpoints-kubeconfig - --cert-secrets=aggregator-client - --cert-secrets=localhost-serving-cert-certkey - --cert-secrets=service-network-serving-certkey - --cert-secrets=external-loadbalancer-serving-certkey - --cert-secrets=internal-loadbalancer-serving-certkey - --cert-secrets=bound-service-account-signing-key - --cert-secrets=control-plane-node-admin-client-cert-key - --cert-secrets=check-endpoints-client-cert-key - --cert-secrets=kubelet-client - --cert-secrets=node-kubeconfigs - --optional-cert-secrets=user-serving-cert - --optional-cert-secrets=user-serving-cert-000 - --optional-cert-secrets=user-serving-cert-001 - --optional-cert-secrets=user-serving-cert-002 - --optional-cert-secrets=user-serving-cert-003 - --optional-cert-secrets=user-serving-cert-004 - --optional-cert-secrets=user-serving-cert-005 - --optional-cert-secrets=user-serving-cert-006 - --optional-cert-secrets=user-serving-cert-007 - --optional-cert-secrets=user-serving-cert-008 - --optional-cert-secrets=user-serving-cert-009 command: - cluster-kube-apiserver-operator - installer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imagePullPolicy: IfNotPresent name: installer resources: limits: cpu: 150m memory: 200M requests: cpu: 150m memory: 200M securityContext: privileged: true runAsUser: 0 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true - mountPath: /var/lock name: var-lock dnsPolicy: ClusterFirst enableServiceLinks: true imagePullSecrets: - name: installer-sa-dockercfg-w5d24 nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Never schedulerName: default-scheduler securityContext: runAsUser: 0 serviceAccount: installer-sa serviceAccountName: installer-sa terminationGracePeriodSeconds: 30 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/ type: "" name: kubelet-dir - hostPath: path: /var/lock type: "" name: var-lock - name: kube-api-access projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3600 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace status: conditions: - lastProbeTime: null lastTransitionTime: "2026-03-19T09:36:46Z" status: "False" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2026-03-19T09:35:45Z" reason: PodCompleted status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2026-03-19T09:36:44Z" reason: PodCompleted status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2026-03-19T09:36:44Z" reason: PodCompleted status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2026-03-19T09:35:45Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://e2243249baeedf2fe21ce14cc48c1196cadeee34e720ae9084b1811ac9f8a731 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 lastState: {} name: installer ready: false restartCount: 0 started: false state: terminated: containerID: cri-o://e2243249baeedf2fe21ce14cc48c1196cadeee34e720ae9084b1811ac9f8a731 exitCode: 0 finishedAt: "2026-03-19T09:36:44Z" reason: Completed startedAt: "2026-03-19T09:35:46Z" volumeMounts: - mountPath: /etc/kubernetes/ name: kubelet-dir - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access readOnly: true recursiveReadOnly: Disabled - mountPath: /var/lock name: var-lock hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 phase: Succeeded podIP: 10.128.0.110 podIPs: - ip: 10.128.0.110 qosClass: Guaranteed startTime: "2026-03-19T09:35:45Z" - apiVersion: v1 kind: Pod metadata: annotations: kubectl.kubernetes.io/default-container: kube-apiserver kubernetes.io/config.hash: 274c4bebf95a655851b2cf276fe43ef7 kubernetes.io/config.mirror: 274c4bebf95a655851b2cf276fe43ef7 kubernetes.io/config.seen: "2026-03-19T09:36:44.735048934Z" kubernetes.io/config.source: file target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' creationTimestamp: "2026-03-19T09:37:07Z" labels: apiserver: "true" app: openshift-kube-apiserver revision: "6" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:metadata: f:annotations: .: {} f:kubectl.kubernetes.io/default-container: {} f:kubernetes.io/config.hash: {} f:kubernetes.io/config.mirror: {} f:kubernetes.io/config.seen: {} f:kubernetes.io/config.source: {} f:target.workload.openshift.io/management: {} f:labels: .: {} f:apiserver: {} f:app: {} f:revision: {} f:ownerReferences: .: {} k:{"uid":"e4052aff-9961-4131-8036-ded1e1d0911b"}: {} f:spec: f:containers: k:{"name":"kube-apiserver"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"GOGC"}: .: {} f:name: {} f:value: {} k:{"name":"HOST_IP"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAMESPACE"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"STATIC_POD_VERSION"}: .: {} f:name: {} f:value: {} f:image: {} f:imagePullPolicy: {} f:livenessProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:name: {} f:ports: .: {} k:{"containerPort":6443,"protocol":"TCP"}: .: {} f:containerPort: {} f:hostPort: {} f:protocol: {} f:readinessProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:startupProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/static-pod-certs"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/etc/kubernetes/static-pod-resources"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/var/log/kube-apiserver"}: .: {} f:mountPath: {} f:name: {} k:{"name":"kube-apiserver-cert-regeneration-controller"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"POD_NAMESPACE"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/static-pod-resources"}: .: {} f:mountPath: {} f:name: {} k:{"name":"kube-apiserver-cert-syncer"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAMESPACE"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/static-pod-certs"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/etc/kubernetes/static-pod-resources"}: .: {} f:mountPath: {} f:name: {} k:{"name":"kube-apiserver-check-endpoints"}: .: {} f:args: {} f:command: {} f:env: .: {} k:{"name":"POD_NAME"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} k:{"name":"POD_NAMESPACE"}: .: {} f:name: {} f:valueFrom: .: {} f:fieldRef: {} f:image: {} f:imagePullPolicy: {} f:livenessProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:initialDelaySeconds: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:name: {} f:ports: .: {} k:{"containerPort":17697,"protocol":"TCP"}: .: {} f:containerPort: {} f:hostPort: {} f:name: {} f:protocol: {} f:readinessProbe: .: {} f:failureThreshold: {} f:httpGet: .: {} f:path: {} f:port: {} f:scheme: {} f:initialDelaySeconds: {} f:periodSeconds: {} f:successThreshold: {} f:timeoutSeconds: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/etc/kubernetes/static-pod-certs"}: .: {} f:mountPath: {} f:name: {} k:{"mountPath":"/etc/kubernetes/static-pod-resources"}: .: {} f:mountPath: {} f:name: {} k:{"name":"kube-apiserver-insecure-readyz"}: .: {} f:args: {} f:command: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:ports: .: {} k:{"containerPort":6080,"protocol":"TCP"}: .: {} f:containerPort: {} f:hostPort: {} f:protocol: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:dnsPolicy: {} f:enableServiceLinks: {} f:hostNetwork: {} f:initContainers: .: {} k:{"name":"setup"}: .: {} f:args: {} f:command: {} f:image: {} f:imagePullPolicy: {} f:name: {} f:resources: .: {} f:requests: .: {} f:cpu: {} f:memory: {} f:securityContext: .: {} f:privileged: {} f:terminationMessagePath: {} f:terminationMessagePolicy: {} f:volumeMounts: .: {} k:{"mountPath":"/var/log/kube-apiserver"}: .: {} f:mountPath: {} f:name: {} f:nodeName: {} f:priority: {} f:priorityClassName: {} f:restartPolicy: {} f:schedulerName: {} f:securityContext: {} f:terminationGracePeriodSeconds: {} f:tolerations: {} f:volumes: .: {} k:{"name":"audit-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"cert-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} k:{"name":"resource-dir"}: .: {} f:hostPath: .: {} f:path: {} f:type: {} f:name: {} manager: kubelet operation: Update time: "2026-03-19T09:37:07Z" - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:status: f:conditions: .: {} k:{"type":"ContainersReady"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Initialized"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodReadyToStartContainers"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"PodScheduled"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} k:{"type":"Ready"}: .: {} f:lastProbeTime: {} f:lastTransitionTime: {} f:status: {} f:type: {} f:containerStatuses: {} f:hostIP: {} f:hostIPs: {} f:initContainerStatuses: {} f:phase: {} f:podIP: {} f:podIPs: .: {} k:{"ip":"192.168.32.10"}: .: {} f:ip: {} f:startTime: {} manager: kubelet operation: Update subresource: status time: "2026-03-19T09:37:20Z" name: kube-apiserver-master-0 namespace: openshift-kube-apiserver ownerReferences: - apiVersion: v1 controller: true kind: Node name: master-0 uid: e4052aff-9961-4131-8036-ded1e1d0911b resourceVersion: "16398" uid: 4478155f-e1c8-42e1-8cac-2b6d6c16c81c spec: containers: - args: - | LOCK=/var/log/kube-apiserver/.lock # We should be able to acquire the lock immediatelly. If not, it means the init container has not released it yet and kubelet or CRI-O started container prematurely. exec {LOCK_FD}>${LOCK} && flock --verbose -w 30 "${LOCK_FD}" || { echo "Failed to acquire lock for kube-apiserver. Please check setup container for details. This is likely kubelet or CRI-O bug." exit 1 } if [ -f /etc/kubernetes/static-pod-certs/configmaps/trusted-ca-bundle/ca-bundle.crt ]; then echo "Copying system trust bundle ..." cp -f /etc/kubernetes/static-pod-certs/configmaps/trusted-ca-bundle/ca-bundle.crt /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem fi exec watch-termination --termination-touch-file=/var/log/kube-apiserver/.terminating --termination-log-file=/var/log/kube-apiserver/termination.log --graceful-termination-duration=15s --kubeconfig=/etc/kubernetes/static-pod-resources/configmaps/kube-apiserver-cert-syncer-kubeconfig/kubeconfig -- hyperkube kube-apiserver --openshift-config=/etc/kubernetes/static-pod-resources/configmaps/config/config.yaml --advertise-address=${HOST_IP} -v=2 --permit-address-sharing command: - /bin/bash - -ec env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace - name: STATIC_POD_VERSION value: "6" - name: HOST_IP valueFrom: fieldRef: apiVersion: v1 fieldPath: status.hostIP - name: GOGC value: "100" image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b23c544d3894e5b31f66a18c554f03b0d29f92c2000c46b57b1c96da7ec25db9 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 httpGet: path: livez?exclude=etcd port: 6443 scheme: HTTPS periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 name: kube-apiserver ports: - containerPort: 6443 hostPort: 6443 protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: readyz port: 6443 scheme: HTTPS periodSeconds: 5 successThreshold: 1 timeoutSeconds: 10 resources: requests: cpu: 265m memory: 1Gi securityContext: privileged: true startupProbe: failureThreshold: 30 httpGet: path: livez port: 6443 scheme: HTTPS periodSeconds: 5 successThreshold: 1 timeoutSeconds: 10 terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/static-pod-resources name: resource-dir - mountPath: /etc/kubernetes/static-pod-certs name: cert-dir - mountPath: /var/log/kube-apiserver name: audit-dir - args: - --kubeconfig=/etc/kubernetes/static-pod-resources/configmaps/kube-apiserver-cert-syncer-kubeconfig/kubeconfig - --namespace=$(POD_NAMESPACE) - --destination-dir=/etc/kubernetes/static-pod-certs command: - cluster-kube-apiserver-operator - cert-syncer env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imagePullPolicy: IfNotPresent name: kube-apiserver-cert-syncer resources: requests: cpu: 5m memory: 50Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/static-pod-resources name: resource-dir - mountPath: /etc/kubernetes/static-pod-certs name: cert-dir - args: - --kubeconfig=/etc/kubernetes/static-pod-resources/configmaps/kube-apiserver-cert-syncer-kubeconfig/kubeconfig - --namespace=$(POD_NAMESPACE) - -v=2 command: - cluster-kube-apiserver-operator - cert-regeneration-controller env: - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imagePullPolicy: IfNotPresent name: kube-apiserver-cert-regeneration-controller resources: requests: cpu: 5m memory: 50Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/static-pod-resources name: resource-dir - args: - --insecure-port=6080 - --delegate-url=https://localhost:6443/readyz command: - cluster-kube-apiserver-operator - insecure-readyz image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imagePullPolicy: IfNotPresent name: kube-apiserver-insecure-readyz ports: - containerPort: 6080 hostPort: 6080 protocol: TCP resources: requests: cpu: 5m memory: 50Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError - args: - --kubeconfig - /etc/kubernetes/static-pod-certs/configmaps/check-endpoints-kubeconfig/kubeconfig - --listen - 0.0.0.0:17697 - --namespace - $(POD_NAMESPACE) - --v - "2" command: - cluster-kube-apiserver-operator - check-endpoints env: - name: POD_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: apiVersion: v1 fieldPath: metadata.namespace image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 3 httpGet: path: healthz port: 17697 scheme: HTTPS initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 name: kube-apiserver-check-endpoints ports: - containerPort: 17697 hostPort: 17697 name: check-endpoints protocol: TCP readinessProbe: failureThreshold: 3 httpGet: path: healthz port: 17697 scheme: HTTPS initialDelaySeconds: 10 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 resources: requests: cpu: 10m memory: 50Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /etc/kubernetes/static-pod-resources name: resource-dir - mountPath: /etc/kubernetes/static-pod-certs name: cert-dir dnsPolicy: ClusterFirst enableServiceLinks: true hostNetwork: true initContainers: - args: - | echo "Fixing audit permissions ..." chmod 0700 /var/log/kube-apiserver && touch /var/log/kube-apiserver/audit.log && chmod 0600 /var/log/kube-apiserver/* LOCK=/var/log/kube-apiserver/.lock echo "Acquiring exclusive lock ${LOCK} ..." # Waiting for 15s max for old kube-apiserver's watch-termination process to exit and remove the lock. # Two cases: # 1. if kubelet does not start the old and new in parallel (i.e. works as expected), the flock will always succeed without any time. # 2. if kubelet does overlap old and new pods for up to 130s, the flock will wait and immediate return when the old finishes. # # NOTE: We can increase 15s for a bigger expected overlap. But a higher value means less noise about the broken kubelet behaviour, i.e. we hide a bug. # NOTE: Do not tweak these timings without considering the livenessProbe initialDelaySeconds exec {LOCK_FD}>${LOCK} && flock --verbose -w 15 "${LOCK_FD}" || { echo "$(date -Iseconds -u) kubelet did not terminate old kube-apiserver before new one" >> /var/log/kube-apiserver/lock.log echo -n ": WARNING: kubelet did not terminate old kube-apiserver before new one." # We failed to acquire exclusive lock, which means there is old kube-apiserver running in system. # Since we utilize SO_REUSEPORT, we need to make sure the old kube-apiserver stopped listening. # # NOTE: This is a fallback for broken kubelet, if you observe this please report a bug. echo -n "Waiting for port 6443 to be released due to likely bug in kubelet or CRI-O " while [ -n "$(ss -Htan state listening '( sport = 6443 or sport = 6080 )')" ]; do echo -n "." sleep 1 (( tries += 1 )) if [[ "${tries}" -gt 10 ]]; then echo "Timed out waiting for port :6443 and :6080 to be released, this is likely a bug in kubelet or CRI-O" exit 1 fi done # This is to make sure the server has terminated independently from the lock. # After the port has been freed (requests can be pending and need 60s max). sleep 65 } # We cannot hold the lock from the init container to the main container. We release it here. There is no risk, at this point we know we are safe. flock -u "${LOCK_FD}" command: - /usr/bin/timeout - "100" - /bin/bash - -ec image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b23c544d3894e5b31f66a18c554f03b0d29f92c2000c46b57b1c96da7ec25db9 imagePullPolicy: IfNotPresent name: setup resources: requests: cpu: 5m memory: 50Mi securityContext: privileged: true terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - mountPath: /var/log/kube-apiserver name: audit-dir nodeName: master-0 preemptionPolicy: PreemptLowerPriority priority: 2000001000 priorityClassName: system-node-critical restartPolicy: Always schedulerName: default-scheduler securityContext: {} terminationGracePeriodSeconds: 15 tolerations: - operator: Exists volumes: - hostPath: path: /etc/kubernetes/static-pod-resources/kube-apiserver-pod-6 type: "" name: resource-dir - hostPath: path: /etc/kubernetes/static-pod-resources/kube-apiserver-certs type: "" name: cert-dir - hostPath: path: /var/log/kube-apiserver type: "" name: audit-dir status: conditions: - lastProbeTime: null lastTransitionTime: "2026-03-19T09:37:20Z" status: "True" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2026-03-19T09:37:20Z" status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2026-03-19T09:37:20Z" status: "True" type: Ready - lastProbeTime: null lastTransitionTime: "2026-03-19T09:37:20Z" status: "True" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2026-03-19T09:37:20Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://0b525ca042a3a8494f293de5fcc5012055226efbfa9bbffaf16a4958cfe9d336 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b23c544d3894e5b31f66a18c554f03b0d29f92c2000c46b57b1c96da7ec25db9 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b23c544d3894e5b31f66a18c554f03b0d29f92c2000c46b57b1c96da7ec25db9 lastState: {} name: kube-apiserver ready: true restartCount: 0 started: true state: running: startedAt: "2026-03-19T09:37:00Z" - containerID: cri-o://4260432a4b18e068a083e08336f7185c72de3274580ea2779a467a2176566f70 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 lastState: {} name: kube-apiserver-cert-regeneration-controller ready: true restartCount: 0 started: true state: running: startedAt: "2026-03-19T09:37:01Z" - containerID: cri-o://cac05d0f0455c10474ba08f296fcb3a7e92ae437cff75a5a560cff0d46c43342 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 lastState: {} name: kube-apiserver-cert-syncer ready: true restartCount: 0 started: true state: running: startedAt: "2026-03-19T09:37:00Z" - containerID: cri-o://66923e4748d2dad6ee0b3afe85f1841e8354d5d5d382a1641b21abd97a174046 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 lastState: {} name: kube-apiserver-check-endpoints ready: true restartCount: 0 started: true state: running: startedAt: "2026-03-19T09:37:01Z" - containerID: cri-o://9ed6f4ea6f7a2a454d5812317c20c3922c80995b5caebb5b8ceaa512b6d67873 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:c5ce3d1134d6500e2b8528516c1889d7bbc6259aba4981c6983395b0e9eeff65 lastState: {} name: kube-apiserver-insecure-readyz ready: true restartCount: 0 started: true state: running: startedAt: "2026-03-19T09:37:01Z" hostIP: 192.168.32.10 hostIPs: - ip: 192.168.32.10 initContainerStatuses: - containerID: cri-o://a531dbba5ef958f61968e5a7d57e1d6bb83d80f6985cf21326d2998412c22393 image: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b23c544d3894e5b31f66a18c554f03b0d29f92c2000c46b57b1c96da7ec25db9 imageID: quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:b23c544d3894e5b31f66a18c554f03b0d29f92c2000c46b57b1c96da7ec25db9 lastState: {} name: setup ready: true restartCount: 0 started: false state: terminated: containerID: cri-o://a531dbba5ef958f61968e5a7d57e1d6bb83d80f6985cf21326d2998412c22393 exitCode: 0 finishedAt: "2026-03-19T09:36:59Z" reason: Completed startedAt: "2026-03-19T09:36:59Z" phase: Running podIP: 192.168.32.10 podIPs: - ip: 192.168.32.10 qosClass: Burstable startTime: "2026-03-19T09:37:20Z" kind: PodList metadata: resourceVersion: "75715"