all: children: zuul_unreachable: hosts: {} hosts: container: ansible_connection: kubectl ansible_host: pod-centos-9-stream-0000069533 ansible_kubectl_context: main:zuul-worker/38-102-83-139:9023 ansible_port: null ansible_python_interpreter: /bin/python3 ansible_ssh_common_args: -o StrictHostKeyChecking=false ansible_user: zuul ara_report_path: '{{ zuul.executor.log_root }}/ara-report' ara_report_type: database nodepool: az: null cloud: /k1s03/ external_id: main-pod-centos-9-stream-0000069533 host_id: null interface_ip: pod-centos-9-stream-0000069533 label: pod-centos-9-stream node_properties: {} private_ipv4: null private_ipv6: null provider: k1s03 public_ipv4: null public_ipv6: null region: null slot: null run_test: ansible_test_nodeps zuul_node: az: null cloud: /k1s03/ external_id: main-pod-centos-9-stream-0000069533 host_id: null interface_ip: pod-centos-9-stream-0000069533 label: pod-centos-9-stream node_properties: {} private_ipv4: null private_ipv6: null provider: k1s03 public_ipv4: null public_ipv6: null region: null slot: null uuid: null vars: ara_report_path: '{{ zuul.executor.log_root }}/ara-report' ara_report_type: database run_test: ansible_test_nodeps zuul: _inheritance_path: - '' - '' - '' - '' - '' ansible_version: '9' attempts: 1 branch: main build: bfced832372b4db896e30b8120c82867 build_refs: - branch: main change: '3980' change_message: "[DNM] [env_op_images] Improve CRI-O image origin verification accuracy\n\nDepends-on: https://github.com/openstack-k8s-operators/ci-framework/pull/3989\r\n\r\nCRI-O always reports the canonical source registry (e.g. registry.redhat.io) in \"Pulled image\" log lines, even when the image was actually fetched from a mirror. This made node_verified_image_origin unreliable -- it nearly always reported \"source\" for mirrored images.\r\n\r\nThis MR switches to using \"Trying to access\" log lines as the authoritative evidence for determining image origin, since those show the actual registry CRI-O contacted.\r\n\r\nAdditionally, the pulled-images report now aligns with PR #3865 which replaced deprecated ImageContentSourcePolicy (ICSP) with ImageDigestMirrorSet (IDMS) and ImageTagMirrorSet (ITMS) in the openshift_setup role:\r\n\r\nRemoved ICSP lookup (oc get imagecontentsourcepolicy)\r\nAdded ITMS lookup (oc get imagetagmirrorset)\r\nMirror mapping builder now flattens IDMS imageDigestMirrors and ITMS imageTagMirrors (no more ICSP repositoryDigestMirrors)\r\nUnit tests were added covering mirror-override, fallback, and pull_failed scenarios." change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3980 commit_id: db045fc008bdc3cc4495198de690b1680575f233 patchset: db045fc008bdc3cc4495198de690b1680575f233 project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework topic: null buildset: df080d569397435e853e9e27e28dd0b2 buildset_refs: - branch: main change: '3980' change_message: "[DNM] [env_op_images] Improve CRI-O image origin verification accuracy\n\nDepends-on: https://github.com/openstack-k8s-operators/ci-framework/pull/3989\r\n\r\nCRI-O always reports the canonical source registry (e.g. registry.redhat.io) in \"Pulled image\" log lines, even when the image was actually fetched from a mirror. This made node_verified_image_origin unreliable -- it nearly always reported \"source\" for mirrored images.\r\n\r\nThis MR switches to using \"Trying to access\" log lines as the authoritative evidence for determining image origin, since those show the actual registry CRI-O contacted.\r\n\r\nAdditionally, the pulled-images report now aligns with PR #3865 which replaced deprecated ImageContentSourcePolicy (ICSP) with ImageDigestMirrorSet (IDMS) and ImageTagMirrorSet (ITMS) in the openshift_setup role:\r\n\r\nRemoved ICSP lookup (oc get imagecontentsourcepolicy)\r\nAdded ITMS lookup (oc get imagetagmirrorset)\r\nMirror mapping builder now flattens IDMS imageDigestMirrors and ITMS imageTagMirrors (no more ICSP repositoryDigestMirrors)\r\nUnit tests were added covering mirror-override, fallback, and pull_failed scenarios." change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3980 commit_id: db045fc008bdc3cc4495198de690b1680575f233 patchset: db045fc008bdc3cc4495198de690b1680575f233 project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework topic: null change: '3980' change_message: "[DNM] [env_op_images] Improve CRI-O image origin verification accuracy\n\nDepends-on: https://github.com/openstack-k8s-operators/ci-framework/pull/3989\r\n\r\nCRI-O always reports the canonical source registry (e.g. registry.redhat.io) in \"Pulled image\" log lines, even when the image was actually fetched from a mirror. This made node_verified_image_origin unreliable -- it nearly always reported \"source\" for mirrored images.\r\n\r\nThis MR switches to using \"Trying to access\" log lines as the authoritative evidence for determining image origin, since those show the actual registry CRI-O contacted.\r\n\r\nAdditionally, the pulled-images report now aligns with PR #3865 which replaced deprecated ImageContentSourcePolicy (ICSP) with ImageDigestMirrorSet (IDMS) and ImageTagMirrorSet (ITMS) in the openshift_setup role:\r\n\r\nRemoved ICSP lookup (oc get imagecontentsourcepolicy)\r\nAdded ITMS lookup (oc get imagetagmirrorset)\r\nMirror mapping builder now flattens IDMS imageDigestMirrors and ITMS imageTagMirrors (no more ICSP repositoryDigestMirrors)\r\nUnit tests were added covering mirror-override, fallback, and pull_failed scenarios." change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3980 child_jobs: [] commit_id: db045fc008bdc3cc4495198de690b1680575f233 event_id: 03fc07f0-64ab-11f1-814c-75d3fc3055eb executor: hostname: 38.102.83.51 inventory_file: /var/lib/zuul/builds/bfced832372b4db896e30b8120c82867/ansible/inventory.yaml log_root: /var/lib/zuul/builds/bfced832372b4db896e30b8120c82867/work/logs result_data_file: /var/lib/zuul/builds/bfced832372b4db896e30b8120c82867/work/results.json src_root: /var/lib/zuul/builds/bfced832372b4db896e30b8120c82867/work/src work_root: /var/lib/zuul/builds/bfced832372b4db896e30b8120c82867/work include_vars: [] items: - branch: main change: '3989' change_message: "[cifmw_helpers] Fix CRC certificate renewal wait using kubeconfig\n\n## Problem\r\nSince ~2026-06-03, all CRC-based molecule jobs (`ci_local_storage`, `env_op_images`, etc.) fail consistently during the prepare phase, stuck on `TASK [cifmw_helpers : Login to the OpenShift when certificate is expired]` for 15 minutes before timing out with 401 Unauthorized.\r\n\r\n## Root cause\r\nThe `oc login -u kubeadmin` command in `crc_start.yml` never included the p flag. In non-interactive CI the empty password always returns 401, regardless of cluster state.\r\n\r\n## Fix\r\n- Replace `oc login` with `oc get nodes --kubeconfig=...` which uses the CRC kubeconfig file directly, avoiding the OAuth dependency and the need for credentials during certificate renewal.\r\n- Increase retries from 90 to 150 (~25 min) to cover the observed cluster recovery time.\r\n \r\nRefs:\r\n - https://gateway-cloud-softwarefactory.apps.ocp.cloud.ci.centos.org/zuul/t/rdoproject.org/build/4d4440cfa7f44ab59da90c14c51d8bd4 \r\n - https://gateway-cloud-softwarefactory.apps.ocp.cloud.ci.centos.org/logs//4d4/rdoproject.org/4d4440cfa7f44ab59da90c14c51d8bd4/ci-framework-data/logs/molecule-execution.log\r\n" change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3989 commit_id: 40bee6119c5e40bf54a372b80a12a0ebdff9661f patchset: 40bee6119c5e40bf54a372b80a12a0ebdff9661f project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework topic: null - branch: main change: '3980' change_message: "[DNM] [env_op_images] Improve CRI-O image origin verification accuracy\n\nDepends-on: https://github.com/openstack-k8s-operators/ci-framework/pull/3989\r\n\r\nCRI-O always reports the canonical source registry (e.g. registry.redhat.io) in \"Pulled image\" log lines, even when the image was actually fetched from a mirror. This made node_verified_image_origin unreliable -- it nearly always reported \"source\" for mirrored images.\r\n\r\nThis MR switches to using \"Trying to access\" log lines as the authoritative evidence for determining image origin, since those show the actual registry CRI-O contacted.\r\n\r\nAdditionally, the pulled-images report now aligns with PR #3865 which replaced deprecated ImageContentSourcePolicy (ICSP) with ImageDigestMirrorSet (IDMS) and ImageTagMirrorSet (ITMS) in the openshift_setup role:\r\n\r\nRemoved ICSP lookup (oc get imagecontentsourcepolicy)\r\nAdded ITMS lookup (oc get imagetagmirrorset)\r\nMirror mapping builder now flattens IDMS imageDigestMirrors and ITMS imageTagMirrors (no more ICSP repositoryDigestMirrors)\r\nUnit tests were added covering mirror-override, fallback, and pull_failed scenarios." change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3980 commit_id: db045fc008bdc3cc4495198de690b1680575f233 patchset: db045fc008bdc3cc4495198de690b1680575f233 project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework topic: null job: cifmw-pod-ansible-test jobtags: [] max_attempts: 3 message: W0ROTV0gW2Vudl9vcF9pbWFnZXNdIEltcHJvdmUgQ1JJLU8gaW1hZ2Ugb3JpZ2luIHZlcmlmaWNhdGlvbiBhY2N1cmFjeQoKRGVwZW5kcy1vbjogaHR0cHM6Ly9naXRodWIuY29tL29wZW5zdGFjay1rOHMtb3BlcmF0b3JzL2NpLWZyYW1ld29yay9wdWxsLzM5ODkNCg0KQ1JJLU8gYWx3YXlzIHJlcG9ydHMgdGhlIGNhbm9uaWNhbCBzb3VyY2UgcmVnaXN0cnkgKGUuZy4gcmVnaXN0cnkucmVkaGF0LmlvKSBpbiAiUHVsbGVkIGltYWdlIiBsb2cgbGluZXMsIGV2ZW4gd2hlbiB0aGUgaW1hZ2Ugd2FzIGFjdHVhbGx5IGZldGNoZWQgZnJvbSBhIG1pcnJvci4gVGhpcyBtYWRlIG5vZGVfdmVyaWZpZWRfaW1hZ2Vfb3JpZ2luIHVucmVsaWFibGUgLS0gaXQgbmVhcmx5IGFsd2F5cyByZXBvcnRlZCAic291cmNlIiBmb3IgbWlycm9yZWQgaW1hZ2VzLg0KDQpUaGlzIE1SIHN3aXRjaGVzIHRvIHVzaW5nICJUcnlpbmcgdG8gYWNjZXNzIiBsb2cgbGluZXMgYXMgdGhlIGF1dGhvcml0YXRpdmUgZXZpZGVuY2UgZm9yIGRldGVybWluaW5nIGltYWdlIG9yaWdpbiwgc2luY2UgdGhvc2Ugc2hvdyB0aGUgYWN0dWFsIHJlZ2lzdHJ5IENSSS1PIGNvbnRhY3RlZC4NCg0KQWRkaXRpb25hbGx5LCB0aGUgcHVsbGVkLWltYWdlcyByZXBvcnQgbm93IGFsaWducyB3aXRoIFBSICMzODY1IHdoaWNoIHJlcGxhY2VkIGRlcHJlY2F0ZWQgSW1hZ2VDb250ZW50U291cmNlUG9saWN5IChJQ1NQKSB3aXRoIEltYWdlRGlnZXN0TWlycm9yU2V0IChJRE1TKSBhbmQgSW1hZ2VUYWdNaXJyb3JTZXQgKElUTVMpIGluIHRoZSBvcGVuc2hpZnRfc2V0dXAgcm9sZToNCg0KUmVtb3ZlZCBJQ1NQIGxvb2t1cCAob2MgZ2V0IGltYWdlY29udGVudHNvdXJjZXBvbGljeSkNCkFkZGVkIElUTVMgbG9va3VwIChvYyBnZXQgaW1hZ2V0YWdtaXJyb3JzZXQpDQpNaXJyb3IgbWFwcGluZyBidWlsZGVyIG5vdyBmbGF0dGVucyBJRE1TIGltYWdlRGlnZXN0TWlycm9ycyBhbmQgSVRNUyBpbWFnZVRhZ01pcnJvcnMgKG5vIG1vcmUgSUNTUCByZXBvc2l0b3J5RGlnZXN0TWlycm9ycykNClVuaXQgdGVzdHMgd2VyZSBhZGRlZCBjb3ZlcmluZyBtaXJyb3Itb3ZlcnJpZGUsIGZhbGxiYWNrLCBhbmQgcHVsbF9mYWlsZWQgc2NlbmFyaW9zLg== patchset: db045fc008bdc3cc4495198de690b1680575f233 pipeline: github-check playbook_context: playbook_projects: trusted/project_0/review.rdoproject.org/config: canonical_name: review.rdoproject.org/config checkout: master commit: 4605eaf5fa5cd3f351940ab9df8f9f0ea6b97ad5 trusted/project_1/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 82e3e8b1292a25e38235bb92b157992d326531d1 trusted/project_2/review.rdoproject.org/rdo-jobs: canonical_name: review.rdoproject.org/rdo-jobs checkout: master commit: ae3f884c6c8c32b779dc61f758c0f6a43ade6877 untrusted/project_0/review.rdoproject.org/rdo-jobs: canonical_name: review.rdoproject.org/rdo-jobs checkout: master commit: ae3f884c6c8c32b779dc61f758c0f6a43ade6877 untrusted/project_1/review.rdoproject.org/config: canonical_name: review.rdoproject.org/config checkout: master commit: 4605eaf5fa5cd3f351940ab9df8f9f0ea6b97ad5 untrusted/project_2/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 82e3e8b1292a25e38235bb92b157992d326531d1 untrusted/project_3/github.com/openstack-k8s-operators/ci-framework: canonical_name: github.com/openstack-k8s-operators/ci-framework checkout: main commit: 33dcb7c2be8e3a573ec0becf63934dfec38f95ca playbooks: - path: untrusted/project_3/github.com/openstack-k8s-operators/ci-framework/ci/playbooks/pod-jobs.yml roles: - checkout: main checkout_description: playbook branch link_name: ansible/playbook_0/role_0/ci-framework link_target: untrusted/project_3/github.com/openstack-k8s-operators/ci-framework role_path: ansible/playbook_0/role_0/ci-framework/roles - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_1/config link_target: untrusted/project_1/review.rdoproject.org/config role_path: ansible/playbook_0/role_1/config/roles - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_2/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/playbook_0/role_2/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_3/rdo-jobs link_target: untrusted/project_0/review.rdoproject.org/rdo-jobs role_path: ansible/playbook_0/role_3/rdo-jobs/roles post_playbooks: - path: trusted/project_0/review.rdoproject.org/config/playbooks/base-minimal/post-ssh.yaml roles: - checkout: master checkout_description: playbook branch link_name: ansible/post_playbook_0/role_0/config link_target: trusted/project_0/review.rdoproject.org/config role_path: ansible/post_playbook_0/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_0/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_0/role_1/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_0/role_2/rdo-jobs link_target: trusted/project_2/review.rdoproject.org/rdo-jobs role_path: ansible/post_playbook_0/role_2/rdo-jobs/roles - path: trusted/project_0/review.rdoproject.org/config/playbooks/base-minimal/post-logs.yaml roles: - checkout: master checkout_description: playbook branch link_name: ansible/post_playbook_1/role_0/config link_target: trusted/project_0/review.rdoproject.org/config role_path: ansible/post_playbook_1/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_1/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_1/role_1/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_1/role_2/rdo-jobs link_target: trusted/project_2/review.rdoproject.org/rdo-jobs role_path: ansible/post_playbook_1/role_2/rdo-jobs/roles pre_playbooks: - path: trusted/project_0/review.rdoproject.org/config/playbooks/base-minimal/pre.yaml roles: - checkout: master checkout_description: playbook branch link_name: ansible/pre_playbook_0/role_0/config link_target: trusted/project_0/review.rdoproject.org/config role_path: ansible/pre_playbook_0/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_0/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_0/role_1/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_0/role_2/rdo-jobs link_target: trusted/project_2/review.rdoproject.org/rdo-jobs role_path: ansible/pre_playbook_0/role_2/rdo-jobs/roles - path: untrusted/project_0/review.rdoproject.org/rdo-jobs/playbooks/base/pre.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_1/role_0/config link_target: untrusted/project_1/review.rdoproject.org/config role_path: ansible/pre_playbook_1/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_1/role_1/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_1/role_1/zuul-jobs/roles - checkout: master checkout_description: playbook branch link_name: ansible/pre_playbook_1/role_2/rdo-jobs link_target: untrusted/project_0/review.rdoproject.org/rdo-jobs role_path: ansible/pre_playbook_1/role_2/rdo-jobs/roles post_review: false post_timeout: 5400 pre_timeout: null project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework projects: github.com/openstack-k8s-operators/ci-framework: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework checkout: main checkout_description: zuul branch commit: 33dcb7c2be8e3a573ec0becf63934dfec38f95ca name: openstack-k8s-operators/ci-framework required: true short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework ref: refs/pull/3980/head resources: container: context: main:zuul-worker/38-102-83-139:9023 namespace: main pod: pod-centos-9-stream-0000069533 resources: cores: 0 instances: 1 ram: 0 stream_port1: 34605 stream_port2: 43811 tenant: rdoproject.org timeout: 1800 topic: null voting: true