all: children: zuul_unreachable: hosts: {} hosts: container: ansible_connection: kubectl ansible_host: pod-centos-9-stream-0000069145 ansible_kubectl_context: main:zuul-worker/38-102-83-186:9023 ansible_port: null ansible_python_interpreter: /bin/python3 ansible_ssh_common_args: -o StrictHostKeyChecking=false ansible_user: zuul ara_report_path: '{{ zuul.executor.log_root }}/ara-report' ara_report_type: database nodepool: az: null cloud: /k1s05/ external_id: main-pod-centos-9-stream-0000069145 host_id: null interface_ip: pod-centos-9-stream-0000069145 label: pod-centos-9-stream node_properties: {} private_ipv4: null private_ipv6: null provider: k1s05 public_ipv4: null public_ipv6: null region: null slot: null run_test: pre_commit_nodeps zuul_node: az: null cloud: /k1s05/ external_id: main-pod-centos-9-stream-0000069145 host_id: null interface_ip: pod-centos-9-stream-0000069145 label: pod-centos-9-stream node_properties: {} private_ipv4: null private_ipv6: null provider: k1s05 public_ipv4: null public_ipv6: null region: null slot: null uuid: null vars: ara_report_path: '{{ zuul.executor.log_root }}/ara-report' ara_report_type: database run_test: pre_commit_nodeps zuul: _inheritance_path: - '' - '' - '' - '' - '' ansible_version: '9' attempts: 1 branch: main build: 4c1ecac9d18642909c7d43e8bcc2e5ce build_refs: - branch: main change: '3989' change_message: "[DNM] [cifmw_helpers] Fix CRC certificate renewal wait using kubeconfig\n\n## Problem\r\nSince ~2026-06-03, all CRC-based molecule jobs (`ci_local_storage`, `env_op_images`, etc.) fail consistently during the prepare phase, stuck on `TASK [cifmw_helpers : Login to the OpenShift when certificate is expired]` for 15 minutes before timing out with 401 Unauthorized.\r\n\r\n## Root cause\r\nThe `oc login -u kubeadmin` command in `crc_start.yml` never included the p flag. In non-interactive CI the empty password always returns 401, regardless of cluster state.\r\n\r\n## Fix\r\n- Replace `oc login` with `oc get nodes --kubeconfig=...` which uses the CRC kubeconfig file directly, avoiding the OAuth dependency and the need for credentials during certificate renewal.\r\n- Increase retries from 90 to 150 (~25 min) to cover the observed cluster recovery time.\r\n \r\nRefs:\r\n - https://gateway-cloud-softwarefactory.apps.ocp.cloud.ci.centos.org/zuul/t/rdoproject.org/build/4d4440cfa7f44ab59da90c14c51d8bd4 \r\n - https://gateway-cloud-softwarefactory.apps.ocp.cloud.ci.centos.org/logs//4d4/rdoproject.org/4d4440cfa7f44ab59da90c14c51d8bd4/ci-framework-data/logs/molecule-execution.log\r\n" change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3989 commit_id: 6b5d3ae5832a8daf8e75b53e155d530d9e9ce1db patchset: 6b5d3ae5832a8daf8e75b53e155d530d9e9ce1db project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework topic: null buildset: 187a9c32bf50489e89ab0260bbfbb5d2 buildset_refs: - branch: main change: '3989' change_message: "[DNM] [cifmw_helpers] Fix CRC certificate renewal wait using kubeconfig\n\n## Problem\r\nSince ~2026-06-03, all CRC-based molecule jobs (`ci_local_storage`, `env_op_images`, etc.) fail consistently during the prepare phase, stuck on `TASK [cifmw_helpers : Login to the OpenShift when certificate is expired]` for 15 minutes before timing out with 401 Unauthorized.\r\n\r\n## Root cause\r\nThe `oc login -u kubeadmin` command in `crc_start.yml` never included the p flag. In non-interactive CI the empty password always returns 401, regardless of cluster state.\r\n\r\n## Fix\r\n- Replace `oc login` with `oc get nodes --kubeconfig=...` which uses the CRC kubeconfig file directly, avoiding the OAuth dependency and the need for credentials during certificate renewal.\r\n- Increase retries from 90 to 150 (~25 min) to cover the observed cluster recovery time.\r\n \r\nRefs:\r\n - https://gateway-cloud-softwarefactory.apps.ocp.cloud.ci.centos.org/zuul/t/rdoproject.org/build/4d4440cfa7f44ab59da90c14c51d8bd4 \r\n - https://gateway-cloud-softwarefactory.apps.ocp.cloud.ci.centos.org/logs//4d4/rdoproject.org/4d4440cfa7f44ab59da90c14c51d8bd4/ci-framework-data/logs/molecule-execution.log\r\n" change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3989 commit_id: 6b5d3ae5832a8daf8e75b53e155d530d9e9ce1db patchset: 6b5d3ae5832a8daf8e75b53e155d530d9e9ce1db project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework topic: null change: '3989' change_message: "[DNM] [cifmw_helpers] Fix CRC certificate renewal wait using kubeconfig\n\n## Problem\r\nSince ~2026-06-03, all CRC-based molecule jobs (`ci_local_storage`, `env_op_images`, etc.) fail consistently during the prepare phase, stuck on `TASK [cifmw_helpers : Login to the OpenShift when certificate is expired]` for 15 minutes before timing out with 401 Unauthorized.\r\n\r\n## Root cause\r\nThe `oc login -u kubeadmin` command in `crc_start.yml` never included the p flag. In non-interactive CI the empty password always returns 401, regardless of cluster state.\r\n\r\n## Fix\r\n- Replace `oc login` with `oc get nodes --kubeconfig=...` which uses the CRC kubeconfig file directly, avoiding the OAuth dependency and the need for credentials during certificate renewal.\r\n- Increase retries from 90 to 150 (~25 min) to cover the observed cluster recovery time.\r\n \r\nRefs:\r\n - https://gateway-cloud-softwarefactory.apps.ocp.cloud.ci.centos.org/zuul/t/rdoproject.org/build/4d4440cfa7f44ab59da90c14c51d8bd4 \r\n - https://gateway-cloud-softwarefactory.apps.ocp.cloud.ci.centos.org/logs//4d4/rdoproject.org/4d4440cfa7f44ab59da90c14c51d8bd4/ci-framework-data/logs/molecule-execution.log\r\n" change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3989 child_jobs: [] commit_id: 6b5d3ae5832a8daf8e75b53e155d530d9e9ce1db event_id: 2bc12bf0-644f-11f1-82a3-305ccb44cad7 executor: hostname: 38.102.83.32 inventory_file: /var/lib/zuul/builds/4c1ecac9d18642909c7d43e8bcc2e5ce/ansible/inventory.yaml log_root: /var/lib/zuul/builds/4c1ecac9d18642909c7d43e8bcc2e5ce/work/logs result_data_file: /var/lib/zuul/builds/4c1ecac9d18642909c7d43e8bcc2e5ce/work/results.json src_root: /var/lib/zuul/builds/4c1ecac9d18642909c7d43e8bcc2e5ce/work/src work_root: /var/lib/zuul/builds/4c1ecac9d18642909c7d43e8bcc2e5ce/work include_vars: [] items: - branch: main change: '3989' change_message: "[DNM] [cifmw_helpers] Fix CRC certificate renewal wait using kubeconfig\n\n## Problem\r\nSince ~2026-06-03, all CRC-based molecule jobs (`ci_local_storage`, `env_op_images`, etc.) fail consistently during the prepare phase, stuck on `TASK [cifmw_helpers : Login to the OpenShift when certificate is expired]` for 15 minutes before timing out with 401 Unauthorized.\r\n\r\n## Root cause\r\nThe `oc login -u kubeadmin` command in `crc_start.yml` never included the p flag. In non-interactive CI the empty password always returns 401, regardless of cluster state.\r\n\r\n## Fix\r\n- Replace `oc login` with `oc get nodes --kubeconfig=...` which uses the CRC kubeconfig file directly, avoiding the OAuth dependency and the need for credentials during certificate renewal.\r\n- Increase retries from 90 to 150 (~25 min) to cover the observed cluster recovery time.\r\n \r\nRefs:\r\n - https://gateway-cloud-softwarefactory.apps.ocp.cloud.ci.centos.org/zuul/t/rdoproject.org/build/4d4440cfa7f44ab59da90c14c51d8bd4 \r\n - https://gateway-cloud-softwarefactory.apps.ocp.cloud.ci.centos.org/logs//4d4/rdoproject.org/4d4440cfa7f44ab59da90c14c51d8bd4/ci-framework-data/logs/molecule-execution.log\r\n" change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3989 commit_id: 6b5d3ae5832a8daf8e75b53e155d530d9e9ce1db patchset: 6b5d3ae5832a8daf8e75b53e155d530d9e9ce1db project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework topic: null job: cifmw-pod-pre-commit jobtags: [] max_attempts: 3 message: W0ROTV0gW2NpZm13X2hlbHBlcnNdIEZpeCBDUkMgY2VydGlmaWNhdGUgcmVuZXdhbCB3YWl0IHVzaW5nIGt1YmVjb25maWcKCiMjIFByb2JsZW0NClNpbmNlIH4yMDI2LTA2LTAzLCBhbGwgQ1JDLWJhc2VkIG1vbGVjdWxlIGpvYnMgKGBjaV9sb2NhbF9zdG9yYWdlYCwgYGVudl9vcF9pbWFnZXNgLCBldGMuKSBmYWlsIGNvbnNpc3RlbnRseSBkdXJpbmcgdGhlIHByZXBhcmUgcGhhc2UsIHN0dWNrIG9uIGBUQVNLIFtjaWZtd19oZWxwZXJzIDogTG9naW4gdG8gdGhlIE9wZW5TaGlmdCB3aGVuIGNlcnRpZmljYXRlIGlzIGV4cGlyZWRdYCBmb3IgMTUgbWludXRlcyBiZWZvcmUgdGltaW5nIG91dCB3aXRoIDQwMSBVbmF1dGhvcml6ZWQuDQoNCiMjIFJvb3QgY2F1c2UNClRoZSBgb2MgbG9naW4gLXUga3ViZWFkbWluYCBjb21tYW5kIGluIGBjcmNfc3RhcnQueW1sYCBuZXZlciBpbmNsdWRlZCB0aGUgcCBmbGFnLiBJbiBub24taW50ZXJhY3RpdmUgQ0kgdGhlIGVtcHR5IHBhc3N3b3JkIGFsd2F5cyByZXR1cm5zIDQwMSwgcmVnYXJkbGVzcyBvZiBjbHVzdGVyIHN0YXRlLg0KDQojIyBGaXgNCi0gUmVwbGFjZSBgb2MgbG9naW5gIHdpdGggYG9jIGdldCBub2RlcyAtLWt1YmVjb25maWc9Li4uYCB3aGljaCB1c2VzICB0aGUgQ1JDIGt1YmVjb25maWcgZmlsZSBkaXJlY3RseSwgYXZvaWRpbmcgdGhlIE9BdXRoIGRlcGVuZGVuY3kgYW5kIHRoZSBuZWVkIGZvciBjcmVkZW50aWFscyBkdXJpbmcgY2VydGlmaWNhdGUgcmVuZXdhbC4NCi0gSW5jcmVhc2UgcmV0cmllcyBmcm9tIDkwIHRvIDE1MCAofjI1IG1pbikgdG8gY292ZXIgdGhlIG9ic2VydmVkIGNsdXN0ZXIgcmVjb3ZlcnkgdGltZS4NCiAgDQpSZWZzOg0KIC0gaHR0cHM6Ly9nYXRld2F5LWNsb3VkLXNvZnR3YXJlZmFjdG9yeS5hcHBzLm9jcC5jbG91ZC5jaS5jZW50b3Mub3JnL3p1dWwvdC9yZG9wcm9qZWN0Lm9yZy9idWlsZC80ZDQ0NDBjZmE3ZjQ0YWI1OWRhOTBjMTRjNTFkOGJkNCANCiAtICBodHRwczovL2dhdGV3YXktY2xvdWQtc29mdHdhcmVmYWN0b3J5LmFwcHMub2NwLmNsb3VkLmNpLmNlbnRvcy5vcmcvbG9ncy8vNGQ0L3Jkb3Byb2plY3Qub3JnLzRkNDQ0MGNmYTdmNDRhYjU5ZGE5MGMxNGM1MWQ4YmQ0L2NpLWZyYW1ld29yay1kYXRhL2xvZ3MvbW9sZWN1bGUtZXhlY3V0aW9uLmxvZw0K patchset: 6b5d3ae5832a8daf8e75b53e155d530d9e9ce1db pipeline: github-check playbook_context: playbook_projects: trusted/project_0/review.rdoproject.org/config: canonical_name: review.rdoproject.org/config checkout: master commit: f4086da87f3046c98f2a4b907deaeed353c7ab49 trusted/project_1/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 82e3e8b1292a25e38235bb92b157992d326531d1 trusted/project_2/review.rdoproject.org/rdo-jobs: canonical_name: review.rdoproject.org/rdo-jobs checkout: master commit: ae3f884c6c8c32b779dc61f758c0f6a43ade6877 untrusted/project_0/review.rdoproject.org/rdo-jobs: canonical_name: review.rdoproject.org/rdo-jobs checkout: master commit: ae3f884c6c8c32b779dc61f758c0f6a43ade6877 untrusted/project_1/review.rdoproject.org/config: canonical_name: review.rdoproject.org/config checkout: master commit: f4086da87f3046c98f2a4b907deaeed353c7ab49 untrusted/project_2/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 82e3e8b1292a25e38235bb92b157992d326531d1 untrusted/project_3/github.com/openstack-k8s-operators/ci-framework: canonical_name: github.com/openstack-k8s-operators/ci-framework checkout: main commit: 6b5d3ae5832a8daf8e75b53e155d530d9e9ce1db playbooks: - path: untrusted/project_3/github.com/openstack-k8s-operators/ci-framework/ci/playbooks/pod-jobs.yml roles: - checkout: main checkout_description: playbook branch link_name: ansible/playbook_0/role_0/ci-framework link_target: untrusted/project_3/github.com/openstack-k8s-operators/ci-framework role_path: ansible/playbook_0/role_0/ci-framework/roles - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_1/config link_target: untrusted/project_1/review.rdoproject.org/config role_path: ansible/playbook_0/role_1/config/roles - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_2/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/playbook_0/role_2/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_3/rdo-jobs link_target: untrusted/project_0/review.rdoproject.org/rdo-jobs role_path: ansible/playbook_0/role_3/rdo-jobs/roles post_playbooks: - path: trusted/project_0/review.rdoproject.org/config/playbooks/base-minimal/post-ssh.yaml roles: - checkout: master checkout_description: playbook branch link_name: ansible/post_playbook_0/role_0/config link_target: trusted/project_0/review.rdoproject.org/config role_path: ansible/post_playbook_0/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_0/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_0/role_1/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_0/role_2/rdo-jobs link_target: trusted/project_2/review.rdoproject.org/rdo-jobs role_path: ansible/post_playbook_0/role_2/rdo-jobs/roles - path: trusted/project_0/review.rdoproject.org/config/playbooks/base-minimal/post-logs.yaml roles: - checkout: master checkout_description: playbook branch link_name: ansible/post_playbook_1/role_0/config link_target: trusted/project_0/review.rdoproject.org/config role_path: ansible/post_playbook_1/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_1/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_1/role_1/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_1/role_2/rdo-jobs link_target: trusted/project_2/review.rdoproject.org/rdo-jobs role_path: ansible/post_playbook_1/role_2/rdo-jobs/roles pre_playbooks: - path: trusted/project_0/review.rdoproject.org/config/playbooks/base-minimal/pre.yaml roles: - checkout: master checkout_description: playbook branch link_name: ansible/pre_playbook_0/role_0/config link_target: trusted/project_0/review.rdoproject.org/config role_path: ansible/pre_playbook_0/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_0/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_0/role_1/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_0/role_2/rdo-jobs link_target: trusted/project_2/review.rdoproject.org/rdo-jobs role_path: ansible/pre_playbook_0/role_2/rdo-jobs/roles - path: untrusted/project_0/review.rdoproject.org/rdo-jobs/playbooks/base/pre.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_1/role_0/config link_target: untrusted/project_1/review.rdoproject.org/config role_path: ansible/pre_playbook_1/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_1/role_1/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_1/role_1/zuul-jobs/roles - checkout: master checkout_description: playbook branch link_name: ansible/pre_playbook_1/role_2/rdo-jobs link_target: untrusted/project_0/review.rdoproject.org/rdo-jobs role_path: ansible/pre_playbook_1/role_2/rdo-jobs/roles post_review: false post_timeout: 5400 pre_timeout: null project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework projects: github.com/openstack-k8s-operators/ci-framework: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework checkout: main checkout_description: zuul branch commit: 6b5d3ae5832a8daf8e75b53e155d530d9e9ce1db name: openstack-k8s-operators/ci-framework required: true short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework ref: refs/pull/3989/head resources: container: context: main:zuul-worker/38-102-83-186:9023 namespace: main pod: pod-centos-9-stream-0000069145 resources: cores: 0 instances: 1 ram: 0 stream_port1: 37633 stream_port2: 46091 tenant: rdoproject.org timeout: 1800 topic: null voting: true