all: children: zuul_unreachable: hosts: {} hosts: container: ansible_connection: kubectl ansible_host: pod-centos-9-stream-0000069573 ansible_kubectl_context: main:zuul-worker/38-102-83-49:9023 ansible_port: null ansible_python_interpreter: /bin/python3 ansible_ssh_common_args: -o StrictHostKeyChecking=false ansible_user: zuul ara_report_path: '{{ zuul.executor.log_root }}/ara-report' ara_report_type: database nodepool: az: null cloud: /k1s-k1s06/ external_id: main-pod-centos-9-stream-0000069573 host_id: null interface_ip: pod-centos-9-stream-0000069573 label: pod-centos-9-stream node_properties: {} private_ipv4: null private_ipv6: null provider: k1s06 public_ipv4: null public_ipv6: null region: null slot: null run_test: ansible_test_nodeps zuul_node: az: null cloud: /k1s-k1s06/ external_id: main-pod-centos-9-stream-0000069573 host_id: null interface_ip: pod-centos-9-stream-0000069573 label: pod-centos-9-stream node_properties: {} private_ipv4: null private_ipv6: null provider: k1s06 public_ipv4: null public_ipv6: null region: null slot: null uuid: null vars: ara_report_path: '{{ zuul.executor.log_root }}/ara-report' ara_report_type: database run_test: ansible_test_nodeps zuul: _inheritance_path: - '' - '' - '' - '' - '' ansible_version: '9' attempts: 1 branch: main build: 8ecce4ec27ec4b23a5da097914a2aa45 build_refs: - branch: main change: '3868' change_message: "[ci_local_storage] Fall back to oc debug for PV dir creation\n\nWhen no Ansible inventory host matches a k8s node hostname (e.g. bare metal SNO where the node is not SSH-accessible), the role silently skips directory creation while still creating PVs that reference non-existent paths. Add an oc debug fallback that creates directories on each node via a debug pod.\r\n\r\nDepends-On: https://github.com/openstack-k8s-operators/ci-framework/pull/3989\r\n\r\nRequired-by: \ https://github.com/openstack-k8s-operators/ci-framework/pull/3739\r\nJira: [OSPRH-26767](https://issues.redhat.com/browse/OSPRH-26767)\r\nGenerated-by: claude-4.6-opus-high" change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3868 commit_id: 19e61090ad44008046333639aa9a672fbf50b86c patchset: 19e61090ad44008046333639aa9a672fbf50b86c project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework topic: null buildset: 0bc67efffa2c47bf8f6a17c5b09d8812 buildset_refs: - branch: main change: '3868' change_message: "[ci_local_storage] Fall back to oc debug for PV dir creation\n\nWhen no Ansible inventory host matches a k8s node hostname (e.g. bare metal SNO where the node is not SSH-accessible), the role silently skips directory creation while still creating PVs that reference non-existent paths. Add an oc debug fallback that creates directories on each node via a debug pod.\r\n\r\nDepends-On: https://github.com/openstack-k8s-operators/ci-framework/pull/3989\r\n\r\nRequired-by: \ https://github.com/openstack-k8s-operators/ci-framework/pull/3739\r\nJira: [OSPRH-26767](https://issues.redhat.com/browse/OSPRH-26767)\r\nGenerated-by: claude-4.6-opus-high" change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3868 commit_id: 19e61090ad44008046333639aa9a672fbf50b86c patchset: 19e61090ad44008046333639aa9a672fbf50b86c project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework topic: null change: '3868' change_message: "[ci_local_storage] Fall back to oc debug for PV dir creation\n\nWhen no Ansible inventory host matches a k8s node hostname (e.g. bare metal SNO where the node is not SSH-accessible), the role silently skips directory creation while still creating PVs that reference non-existent paths. Add an oc debug fallback that creates directories on each node via a debug pod.\r\n\r\nDepends-On: https://github.com/openstack-k8s-operators/ci-framework/pull/3989\r\n\r\nRequired-by: \ https://github.com/openstack-k8s-operators/ci-framework/pull/3739\r\nJira: [OSPRH-26767](https://issues.redhat.com/browse/OSPRH-26767)\r\nGenerated-by: claude-4.6-opus-high" change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3868 child_jobs: [] commit_id: 19e61090ad44008046333639aa9a672fbf50b86c event_id: 3d1700b0-64b2-11f1-9d7e-089b464e492d executor: hostname: 38.129.56.84 inventory_file: /var/lib/zuul/builds/8ecce4ec27ec4b23a5da097914a2aa45/ansible/inventory.yaml log_root: /var/lib/zuul/builds/8ecce4ec27ec4b23a5da097914a2aa45/work/logs result_data_file: /var/lib/zuul/builds/8ecce4ec27ec4b23a5da097914a2aa45/work/results.json src_root: /var/lib/zuul/builds/8ecce4ec27ec4b23a5da097914a2aa45/work/src work_root: /var/lib/zuul/builds/8ecce4ec27ec4b23a5da097914a2aa45/work include_vars: [] items: - branch: main change: '3989' change_message: "[cifmw_helpers] Fix CRC certificate renewal wait using kubeconfig\n\n## Problem\r\nSince ~2026-06-03, all CRC-based molecule jobs (`ci_local_storage`, `env_op_images`, etc.) fail consistently during the prepare phase, stuck on `TASK [cifmw_helpers : Login to the OpenShift when certificate is expired]` for 15 minutes before timing out with 401 Unauthorized.\r\n\r\n## Root cause\r\nThe `oc login -u kubeadmin` command in `crc_start.yml` never included the p flag. In non-interactive CI the empty password always returns 401, regardless of cluster state.\r\n\r\n## Fix\r\n- Replace `oc login` with `oc get nodes --kubeconfig=...` which uses the CRC kubeconfig file directly, avoiding the OAuth dependency and the need for credentials during certificate renewal.\r\n- Increase retries from 90 to 150 (~25 min) to cover the observed cluster recovery time.\r\n \r\nRefs:\r\n - https://gateway-cloud-softwarefactory.apps.ocp.cloud.ci.centos.org/zuul/t/rdoproject.org/build/4d4440cfa7f44ab59da90c14c51d8bd4 \r\n - https://gateway-cloud-softwarefactory.apps.ocp.cloud.ci.centos.org/logs//4d4/rdoproject.org/4d4440cfa7f44ab59da90c14c51d8bd4/ci-framework-data/logs/molecule-execution.log\r\n" change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3989 commit_id: 40bee6119c5e40bf54a372b80a12a0ebdff9661f patchset: 40bee6119c5e40bf54a372b80a12a0ebdff9661f project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework topic: null - branch: main change: '3868' change_message: "[ci_local_storage] Fall back to oc debug for PV dir creation\n\nWhen no Ansible inventory host matches a k8s node hostname (e.g. bare metal SNO where the node is not SSH-accessible), the role silently skips directory creation while still creating PVs that reference non-existent paths. Add an oc debug fallback that creates directories on each node via a debug pod.\r\n\r\nDepends-On: https://github.com/openstack-k8s-operators/ci-framework/pull/3989\r\n\r\nRequired-by: \ https://github.com/openstack-k8s-operators/ci-framework/pull/3739\r\nJira: [OSPRH-26767](https://issues.redhat.com/browse/OSPRH-26767)\r\nGenerated-by: claude-4.6-opus-high" change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3868 commit_id: 19e61090ad44008046333639aa9a672fbf50b86c patchset: 19e61090ad44008046333639aa9a672fbf50b86c project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework topic: null job: cifmw-pod-ansible-test jobtags: [] max_attempts: 3 message: W2NpX2xvY2FsX3N0b3JhZ2VdIEZhbGwgYmFjayB0byBvYyBkZWJ1ZyBmb3IgUFYgZGlyIGNyZWF0aW9uCgpXaGVuIG5vIEFuc2libGUgaW52ZW50b3J5IGhvc3QgbWF0Y2hlcyBhIGs4cyBub2RlIGhvc3RuYW1lIChlLmcuIGJhcmUgbWV0YWwgU05PIHdoZXJlIHRoZSBub2RlIGlzIG5vdCBTU0gtYWNjZXNzaWJsZSksIHRoZSByb2xlIHNpbGVudGx5IHNraXBzIGRpcmVjdG9yeSBjcmVhdGlvbiB3aGlsZSBzdGlsbCBjcmVhdGluZyBQVnMgdGhhdCByZWZlcmVuY2Ugbm9uLWV4aXN0ZW50IHBhdGhzLiBBZGQgYW4gb2MgZGVidWcgZmFsbGJhY2sgdGhhdCBjcmVhdGVzIGRpcmVjdG9yaWVzIG9uIGVhY2ggbm9kZSB2aWEgYSBkZWJ1ZyBwb2QuDQoNCkRlcGVuZHMtT246IGh0dHBzOi8vZ2l0aHViLmNvbS9vcGVuc3RhY2stazhzLW9wZXJhdG9ycy9jaS1mcmFtZXdvcmsvcHVsbC8zOTg5DQoNClJlcXVpcmVkLWJ5OiAgaHR0cHM6Ly9naXRodWIuY29tL29wZW5zdGFjay1rOHMtb3BlcmF0b3JzL2NpLWZyYW1ld29yay9wdWxsLzM3MzkNCkppcmE6IFtPU1BSSC0yNjc2N10oaHR0cHM6Ly9pc3N1ZXMucmVkaGF0LmNvbS9icm93c2UvT1NQUkgtMjY3NjcpDQpHZW5lcmF0ZWQtYnk6IGNsYXVkZS00LjYtb3B1cy1oaWdo patchset: 19e61090ad44008046333639aa9a672fbf50b86c pipeline: github-check playbook_context: playbook_projects: trusted/project_0/review.rdoproject.org/config: canonical_name: review.rdoproject.org/config checkout: master commit: 4605eaf5fa5cd3f351940ab9df8f9f0ea6b97ad5 trusted/project_1/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 82e3e8b1292a25e38235bb92b157992d326531d1 trusted/project_2/review.rdoproject.org/rdo-jobs: canonical_name: review.rdoproject.org/rdo-jobs checkout: master commit: ae3f884c6c8c32b779dc61f758c0f6a43ade6877 untrusted/project_0/review.rdoproject.org/rdo-jobs: canonical_name: review.rdoproject.org/rdo-jobs checkout: master commit: ae3f884c6c8c32b779dc61f758c0f6a43ade6877 untrusted/project_1/review.rdoproject.org/config: canonical_name: review.rdoproject.org/config checkout: master commit: 4605eaf5fa5cd3f351940ab9df8f9f0ea6b97ad5 untrusted/project_2/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 82e3e8b1292a25e38235bb92b157992d326531d1 untrusted/project_3/github.com/openstack-k8s-operators/ci-framework: canonical_name: github.com/openstack-k8s-operators/ci-framework checkout: main commit: 62a58af188e5f3d8d9015ab8c036d74a2b58145f playbooks: - path: untrusted/project_3/github.com/openstack-k8s-operators/ci-framework/ci/playbooks/pod-jobs.yml roles: - checkout: main checkout_description: playbook branch link_name: ansible/playbook_0/role_0/ci-framework link_target: untrusted/project_3/github.com/openstack-k8s-operators/ci-framework role_path: ansible/playbook_0/role_0/ci-framework/roles - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_1/config link_target: untrusted/project_1/review.rdoproject.org/config role_path: ansible/playbook_0/role_1/config/roles - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_2/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/playbook_0/role_2/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_3/rdo-jobs link_target: untrusted/project_0/review.rdoproject.org/rdo-jobs role_path: ansible/playbook_0/role_3/rdo-jobs/roles post_playbooks: - path: trusted/project_0/review.rdoproject.org/config/playbooks/base-minimal/post-ssh.yaml roles: - checkout: master checkout_description: playbook branch link_name: ansible/post_playbook_0/role_0/config link_target: trusted/project_0/review.rdoproject.org/config role_path: ansible/post_playbook_0/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_0/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_0/role_1/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_0/role_2/rdo-jobs link_target: trusted/project_2/review.rdoproject.org/rdo-jobs role_path: ansible/post_playbook_0/role_2/rdo-jobs/roles - path: trusted/project_0/review.rdoproject.org/config/playbooks/base-minimal/post-logs.yaml roles: - checkout: master checkout_description: playbook branch link_name: ansible/post_playbook_1/role_0/config link_target: trusted/project_0/review.rdoproject.org/config role_path: ansible/post_playbook_1/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_1/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_1/role_1/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_1/role_2/rdo-jobs link_target: trusted/project_2/review.rdoproject.org/rdo-jobs role_path: ansible/post_playbook_1/role_2/rdo-jobs/roles pre_playbooks: - path: trusted/project_0/review.rdoproject.org/config/playbooks/base-minimal/pre.yaml roles: - checkout: master checkout_description: playbook branch link_name: ansible/pre_playbook_0/role_0/config link_target: trusted/project_0/review.rdoproject.org/config role_path: ansible/pre_playbook_0/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_0/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_0/role_1/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_0/role_2/rdo-jobs link_target: trusted/project_2/review.rdoproject.org/rdo-jobs role_path: ansible/pre_playbook_0/role_2/rdo-jobs/roles - path: untrusted/project_0/review.rdoproject.org/rdo-jobs/playbooks/base/pre.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_1/role_0/config link_target: untrusted/project_1/review.rdoproject.org/config role_path: ansible/pre_playbook_1/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_1/role_1/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_1/role_1/zuul-jobs/roles - checkout: master checkout_description: playbook branch link_name: ansible/pre_playbook_1/role_2/rdo-jobs link_target: untrusted/project_0/review.rdoproject.org/rdo-jobs role_path: ansible/pre_playbook_1/role_2/rdo-jobs/roles post_review: false post_timeout: 5400 pre_timeout: null project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework projects: github.com/openstack-k8s-operators/ci-framework: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework checkout: main checkout_description: zuul branch commit: 62a58af188e5f3d8d9015ab8c036d74a2b58145f name: openstack-k8s-operators/ci-framework required: true short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework ref: refs/pull/3868/head resources: container: context: main:zuul-worker/38-102-83-49:9023 namespace: main pod: pod-centos-9-stream-0000069573 resources: cores: 0 instances: 1 ram: 0 stream_port1: 44623 stream_port2: 44051 tenant: rdoproject.org timeout: 1800 topic: null voting: true