all: children: zuul_unreachable: hosts: {} hosts: container: ansible_connection: kubectl ansible_host: pod-centos-9-stream-0000074729 ansible_kubectl_context: main:zuul-worker/38-102-83-54:9023 ansible_port: null ansible_python_interpreter: /bin/python3 ansible_ssh_common_args: -o StrictHostKeyChecking=false ansible_user: zuul ara_report_path: '{{ zuul.executor.log_root }}/ara-report' ara_report_type: database nodepool: az: null cloud: /k1s04/ external_id: main-pod-centos-9-stream-0000074729 host_id: null interface_ip: pod-centos-9-stream-0000074729 label: pod-centos-9-stream node_properties: {} private_ipv4: null private_ipv6: null provider: k1s04 public_ipv4: null public_ipv6: null region: null slot: null run_test: ansible_test_nodeps zuul_node: az: null cloud: /k1s04/ external_id: main-pod-centos-9-stream-0000074729 host_id: null interface_ip: pod-centos-9-stream-0000074729 label: pod-centos-9-stream node_properties: {} private_ipv4: null private_ipv6: null provider: k1s04 public_ipv4: null public_ipv6: null region: null slot: null uuid: null vars: ara_report_path: '{{ zuul.executor.log_root }}/ara-report' ara_report_type: database run_test: ansible_test_nodeps zuul: _inheritance_path: - '' - '' - '' - '' - '' ansible_version: '9' attempts: 1 branch: main build: da1ea9578c624df18f37274f5118315a build_refs: - branch: main change: '3978' change_message: "[DNM] [update] fix: stop control plane check during split variant post-services tests\n\n## Context\r\nThis is a targeted fix to unblock the job which is currently failing due to a race condition between the continuous control plane check and tobiko tests.\r\n\r\nA more comprehensive solution, such as adding a pause/resume mechanism to the control plane check scripts or reworking how background tests interact with the split update variant, could be evaluated as a follow-up.\r\n\r\n## Problem\r\nSome jobs fails because tobiko tests encounter VMs in `BUILD` state during the post-services test phase.\r\n\r\nThe root cause is a race condition: the continuous control plane check (`cifmw_update_control_plane_check`) runs `workload_launch.sh` in an infinite loop, creating and tearing down VMs every few seconds. \r\n\r\nIn the split variant, tests run between the services update and system update phases while this loop is still active.\r\nTobiko lists all servers, finds a VM mid-creation, and fails:\r\n`novaclient.exceptions.Conflict: Cannot 'reboot' instance ... while it is in vm_state building (HTTP 409)`\r\n\r\n## Proposed solution\r\nStop the control plane check before running the post-services tests in `update_variant_split.yml`, then restart it so it monitors the system update phase as well.\r\n\r\nThe ping test is not stopped because it only runs `ping` against an existing VM and does not create any new OpenStack resources.\r\n\r\nEach control plane check run produces PID-scoped log files (`control-plane-test-.log`), so the stop/restart cycle creates two separate runs with independent validation, no log collision.\r\n\r\nCloses: [OSPCIX-1381](https://redhat.atlassian.net/browse/OSPCIX-1381)\r\nAssisted-By: Cursor" change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3978 commit_id: fa56392621484da940a992294783b036e64b37ea patchset: fa56392621484da940a992294783b036e64b37ea project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework topic: null buildset: bce93d9b85fa43638d686a8176477539 buildset_refs: - branch: main change: '3978' change_message: "[DNM] [update] fix: stop control plane check during split variant post-services tests\n\n## Context\r\nThis is a targeted fix to unblock the job which is currently failing due to a race condition between the continuous control plane check and tobiko tests.\r\n\r\nA more comprehensive solution, such as adding a pause/resume mechanism to the control plane check scripts or reworking how background tests interact with the split update variant, could be evaluated as a follow-up.\r\n\r\n## Problem\r\nSome jobs fails because tobiko tests encounter VMs in `BUILD` state during the post-services test phase.\r\n\r\nThe root cause is a race condition: the continuous control plane check (`cifmw_update_control_plane_check`) runs `workload_launch.sh` in an infinite loop, creating and tearing down VMs every few seconds. \r\n\r\nIn the split variant, tests run between the services update and system update phases while this loop is still active.\r\nTobiko lists all servers, finds a VM mid-creation, and fails:\r\n`novaclient.exceptions.Conflict: Cannot 'reboot' instance ... while it is in vm_state building (HTTP 409)`\r\n\r\n## Proposed solution\r\nStop the control plane check before running the post-services tests in `update_variant_split.yml`, then restart it so it monitors the system update phase as well.\r\n\r\nThe ping test is not stopped because it only runs `ping` against an existing VM and does not create any new OpenStack resources.\r\n\r\nEach control plane check run produces PID-scoped log files (`control-plane-test-.log`), so the stop/restart cycle creates two separate runs with independent validation, no log collision.\r\n\r\nCloses: [OSPCIX-1381](https://redhat.atlassian.net/browse/OSPCIX-1381)\r\nAssisted-By: Cursor" change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3978 commit_id: fa56392621484da940a992294783b036e64b37ea patchset: fa56392621484da940a992294783b036e64b37ea project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework topic: null change: '3978' change_message: "[DNM] [update] fix: stop control plane check during split variant post-services tests\n\n## Context\r\nThis is a targeted fix to unblock the job which is currently failing due to a race condition between the continuous control plane check and tobiko tests.\r\n\r\nA more comprehensive solution, such as adding a pause/resume mechanism to the control plane check scripts or reworking how background tests interact with the split update variant, could be evaluated as a follow-up.\r\n\r\n## Problem\r\nSome jobs fails because tobiko tests encounter VMs in `BUILD` state during the post-services test phase.\r\n\r\nThe root cause is a race condition: the continuous control plane check (`cifmw_update_control_plane_check`) runs `workload_launch.sh` in an infinite loop, creating and tearing down VMs every few seconds. \r\n\r\nIn the split variant, tests run between the services update and system update phases while this loop is still active.\r\nTobiko lists all servers, finds a VM mid-creation, and fails:\r\n`novaclient.exceptions.Conflict: Cannot 'reboot' instance ... while it is in vm_state building (HTTP 409)`\r\n\r\n## Proposed solution\r\nStop the control plane check before running the post-services tests in `update_variant_split.yml`, then restart it so it monitors the system update phase as well.\r\n\r\nThe ping test is not stopped because it only runs `ping` against an existing VM and does not create any new OpenStack resources.\r\n\r\nEach control plane check run produces PID-scoped log files (`control-plane-test-.log`), so the stop/restart cycle creates two separate runs with independent validation, no log collision.\r\n\r\nCloses: [OSPCIX-1381](https://redhat.atlassian.net/browse/OSPCIX-1381)\r\nAssisted-By: Cursor" change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3978 child_jobs: [] commit_id: fa56392621484da940a992294783b036e64b37ea event_id: 22e09c00-690e-11f1-9feb-b05d70484f1e executor: hostname: 38.129.56.84 inventory_file: /var/lib/zuul/builds/da1ea9578c624df18f37274f5118315a/ansible/inventory.yaml log_root: /var/lib/zuul/builds/da1ea9578c624df18f37274f5118315a/work/logs result_data_file: /var/lib/zuul/builds/da1ea9578c624df18f37274f5118315a/work/results.json src_root: /var/lib/zuul/builds/da1ea9578c624df18f37274f5118315a/work/src work_root: /var/lib/zuul/builds/da1ea9578c624df18f37274f5118315a/work include_vars: [] items: - branch: main change: '3978' change_message: "[DNM] [update] fix: stop control plane check during split variant post-services tests\n\n## Context\r\nThis is a targeted fix to unblock the job which is currently failing due to a race condition between the continuous control plane check and tobiko tests.\r\n\r\nA more comprehensive solution, such as adding a pause/resume mechanism to the control plane check scripts or reworking how background tests interact with the split update variant, could be evaluated as a follow-up.\r\n\r\n## Problem\r\nSome jobs fails because tobiko tests encounter VMs in `BUILD` state during the post-services test phase.\r\n\r\nThe root cause is a race condition: the continuous control plane check (`cifmw_update_control_plane_check`) runs `workload_launch.sh` in an infinite loop, creating and tearing down VMs every few seconds. \r\n\r\nIn the split variant, tests run between the services update and system update phases while this loop is still active.\r\nTobiko lists all servers, finds a VM mid-creation, and fails:\r\n`novaclient.exceptions.Conflict: Cannot 'reboot' instance ... while it is in vm_state building (HTTP 409)`\r\n\r\n## Proposed solution\r\nStop the control plane check before running the post-services tests in `update_variant_split.yml`, then restart it so it monitors the system update phase as well.\r\n\r\nThe ping test is not stopped because it only runs `ping` against an existing VM and does not create any new OpenStack resources.\r\n\r\nEach control plane check run produces PID-scoped log files (`control-plane-test-.log`), so the stop/restart cycle creates two separate runs with independent validation, no log collision.\r\n\r\nCloses: [OSPCIX-1381](https://redhat.atlassian.net/browse/OSPCIX-1381)\r\nAssisted-By: Cursor" change_url: https://github.com/openstack-k8s-operators/ci-framework/pull/3978 commit_id: fa56392621484da940a992294783b036e64b37ea patchset: fa56392621484da940a992294783b036e64b37ea project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework topic: null job: cifmw-pod-ansible-test jobtags: [] max_attempts: 3 message: W0ROTV0gW3VwZGF0ZV0gZml4OiBzdG9wIGNvbnRyb2wgcGxhbmUgY2hlY2sgZHVyaW5nIHNwbGl0IHZhcmlhbnQgcG9zdC1zZXJ2aWNlcyB0ZXN0cwoKIyMgQ29udGV4dA0KVGhpcyBpcyBhIHRhcmdldGVkIGZpeCB0byB1bmJsb2NrIHRoZSBqb2Igd2hpY2ggaXMgY3VycmVudGx5IGZhaWxpbmcgZHVlIHRvIGEgcmFjZSBjb25kaXRpb24gYmV0d2VlbiB0aGUgY29udGludW91cyBjb250cm9sIHBsYW5lIGNoZWNrIGFuZCB0b2Jpa28gdGVzdHMuDQoNCkEgbW9yZSBjb21wcmVoZW5zaXZlIHNvbHV0aW9uLCBzdWNoIGFzIGFkZGluZyBhIHBhdXNlL3Jlc3VtZSBtZWNoYW5pc20gdG8gdGhlIGNvbnRyb2wgcGxhbmUgY2hlY2sgc2NyaXB0cyBvciByZXdvcmtpbmcgaG93IGJhY2tncm91bmQgdGVzdHMgaW50ZXJhY3Qgd2l0aCB0aGUgc3BsaXQgdXBkYXRlIHZhcmlhbnQsIGNvdWxkIGJlIGV2YWx1YXRlZCBhcyBhIGZvbGxvdy11cC4NCg0KIyMgUHJvYmxlbQ0KU29tZSBqb2JzIGZhaWxzIGJlY2F1c2UgdG9iaWtvIHRlc3RzIGVuY291bnRlciBWTXMgaW4gYEJVSUxEYCBzdGF0ZSBkdXJpbmcgdGhlIHBvc3Qtc2VydmljZXMgdGVzdCBwaGFzZS4NCg0KVGhlIHJvb3QgY2F1c2UgaXMgYSByYWNlIGNvbmRpdGlvbjogdGhlIGNvbnRpbnVvdXMgY29udHJvbCBwbGFuZSBjaGVjayAoYGNpZm13X3VwZGF0ZV9jb250cm9sX3BsYW5lX2NoZWNrYCkgcnVucyBgd29ya2xvYWRfbGF1bmNoLnNoYCBpbiBhbiBpbmZpbml0ZSBsb29wLCBjcmVhdGluZyBhbmQgdGVhcmluZyBkb3duIFZNcyBldmVyeSBmZXcgc2Vjb25kcy4gDQoNCkluIHRoZSBzcGxpdCB2YXJpYW50LCB0ZXN0cyBydW4gYmV0d2VlbiB0aGUgc2VydmljZXMgdXBkYXRlIGFuZCBzeXN0ZW0gdXBkYXRlIHBoYXNlcyB3aGlsZSB0aGlzIGxvb3AgaXMgc3RpbGwgYWN0aXZlLg0KVG9iaWtvIGxpc3RzIGFsbCBzZXJ2ZXJzLCBmaW5kcyBhIFZNIG1pZC1jcmVhdGlvbiwgYW5kIGZhaWxzOg0KYG5vdmFjbGllbnQuZXhjZXB0aW9ucy5Db25mbGljdDogQ2Fubm90ICdyZWJvb3QnIGluc3RhbmNlIC4uLiB3aGlsZSBpdCBpcyBpbiB2bV9zdGF0ZSBidWlsZGluZyAoSFRUUCA0MDkpYA0KDQojIyBQcm9wb3NlZCBzb2x1dGlvbg0KU3RvcCB0aGUgY29udHJvbCBwbGFuZSBjaGVjayBiZWZvcmUgcnVubmluZyB0aGUgcG9zdC1zZXJ2aWNlcyB0ZXN0cyBpbiBgdXBkYXRlX3ZhcmlhbnRfc3BsaXQueW1sYCwgdGhlbiByZXN0YXJ0IGl0IHNvIGl0IG1vbml0b3JzIHRoZSBzeXN0ZW0gdXBkYXRlIHBoYXNlIGFzIHdlbGwuDQoNClRoZSBwaW5nIHRlc3QgaXMgbm90IHN0b3BwZWQgYmVjYXVzZSBpdCBvbmx5IHJ1bnMgYHBpbmdgIGFnYWluc3QgYW4gZXhpc3RpbmcgVk0gYW5kIGRvZXMgbm90IGNyZWF0ZSBhbnkgbmV3IE9wZW5TdGFjayByZXNvdXJjZXMuDQoNCkVhY2ggY29udHJvbCBwbGFuZSBjaGVjayBydW4gcHJvZHVjZXMgUElELXNjb3BlZCBsb2cgZmlsZXMgKGBjb250cm9sLXBsYW5lLXRlc3QtPFBJRD4ubG9nYCksIHNvIHRoZSBzdG9wL3Jlc3RhcnQgY3ljbGUgY3JlYXRlcyB0d28gc2VwYXJhdGUgcnVucyB3aXRoIGluZGVwZW5kZW50IHZhbGlkYXRpb24sIG5vIGxvZyBjb2xsaXNpb24uDQoNCkNsb3NlczogW09TUENJWC0xMzgxXShodHRwczovL3JlZGhhdC5hdGxhc3NpYW4ubmV0L2Jyb3dzZS9PU1BDSVgtMTM4MSkNCkFzc2lzdGVkLUJ5OiBDdXJzb3I= patchset: fa56392621484da940a992294783b036e64b37ea pipeline: github-check playbook_context: playbook_projects: trusted/project_0/review.rdoproject.org/config: canonical_name: review.rdoproject.org/config checkout: master commit: 6906bd265a5624e6f7c7ca8448bbeb7f7d7d975d trusted/project_1/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 82e3e8b1292a25e38235bb92b157992d326531d1 trusted/project_2/review.rdoproject.org/rdo-jobs: canonical_name: review.rdoproject.org/rdo-jobs checkout: master commit: ae3f884c6c8c32b779dc61f758c0f6a43ade6877 untrusted/project_0/review.rdoproject.org/rdo-jobs: canonical_name: review.rdoproject.org/rdo-jobs checkout: master commit: ae3f884c6c8c32b779dc61f758c0f6a43ade6877 untrusted/project_1/review.rdoproject.org/config: canonical_name: review.rdoproject.org/config checkout: master commit: 6906bd265a5624e6f7c7ca8448bbeb7f7d7d975d untrusted/project_2/opendev.org/zuul/zuul-jobs: canonical_name: opendev.org/zuul/zuul-jobs checkout: master commit: 82e3e8b1292a25e38235bb92b157992d326531d1 untrusted/project_3/github.com/openstack-k8s-operators/ci-framework: canonical_name: github.com/openstack-k8s-operators/ci-framework checkout: main commit: fa56392621484da940a992294783b036e64b37ea playbooks: - path: untrusted/project_3/github.com/openstack-k8s-operators/ci-framework/ci/playbooks/pod-jobs.yml roles: - checkout: main checkout_description: playbook branch link_name: ansible/playbook_0/role_0/ci-framework link_target: untrusted/project_3/github.com/openstack-k8s-operators/ci-framework role_path: ansible/playbook_0/role_0/ci-framework/roles - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_1/config link_target: untrusted/project_1/review.rdoproject.org/config role_path: ansible/playbook_0/role_1/config/roles - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_2/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/playbook_0/role_2/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/playbook_0/role_3/rdo-jobs link_target: untrusted/project_0/review.rdoproject.org/rdo-jobs role_path: ansible/playbook_0/role_3/rdo-jobs/roles post_playbooks: - path: trusted/project_0/review.rdoproject.org/config/playbooks/base-minimal/post-ssh.yaml roles: - checkout: master checkout_description: playbook branch link_name: ansible/post_playbook_0/role_0/config link_target: trusted/project_0/review.rdoproject.org/config role_path: ansible/post_playbook_0/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_0/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_0/role_1/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_0/role_2/rdo-jobs link_target: trusted/project_2/review.rdoproject.org/rdo-jobs role_path: ansible/post_playbook_0/role_2/rdo-jobs/roles - path: trusted/project_0/review.rdoproject.org/config/playbooks/base-minimal/post-logs.yaml roles: - checkout: master checkout_description: playbook branch link_name: ansible/post_playbook_1/role_0/config link_target: trusted/project_0/review.rdoproject.org/config role_path: ansible/post_playbook_1/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_1/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/post_playbook_1/role_1/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/post_playbook_1/role_2/rdo-jobs link_target: trusted/project_2/review.rdoproject.org/rdo-jobs role_path: ansible/post_playbook_1/role_2/rdo-jobs/roles pre_playbooks: - path: trusted/project_0/review.rdoproject.org/config/playbooks/base-minimal/pre.yaml roles: - checkout: master checkout_description: playbook branch link_name: ansible/pre_playbook_0/role_0/config link_target: trusted/project_0/review.rdoproject.org/config role_path: ansible/pre_playbook_0/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_0/role_1/zuul-jobs link_target: trusted/project_1/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_0/role_1/zuul-jobs/roles - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_0/role_2/rdo-jobs link_target: trusted/project_2/review.rdoproject.org/rdo-jobs role_path: ansible/pre_playbook_0/role_2/rdo-jobs/roles - path: untrusted/project_0/review.rdoproject.org/rdo-jobs/playbooks/base/pre.yaml roles: - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_1/role_0/config link_target: untrusted/project_1/review.rdoproject.org/config role_path: ansible/pre_playbook_1/role_0/config/roles - checkout: master checkout_description: project default branch link_name: ansible/pre_playbook_1/role_1/zuul-jobs link_target: untrusted/project_2/opendev.org/zuul/zuul-jobs role_path: ansible/pre_playbook_1/role_1/zuul-jobs/roles - checkout: master checkout_description: playbook branch link_name: ansible/pre_playbook_1/role_2/rdo-jobs link_target: untrusted/project_0/review.rdoproject.org/rdo-jobs role_path: ansible/pre_playbook_1/role_2/rdo-jobs/roles post_review: false post_timeout: 5400 pre_timeout: null project: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework name: openstack-k8s-operators/ci-framework short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework projects: github.com/openstack-k8s-operators/ci-framework: canonical_hostname: github.com canonical_name: github.com/openstack-k8s-operators/ci-framework checkout: main checkout_description: zuul branch commit: fa56392621484da940a992294783b036e64b37ea name: openstack-k8s-operators/ci-framework required: true short_name: ci-framework src_dir: src/github.com/openstack-k8s-operators/ci-framework ref: refs/pull/3978/head resources: container: context: main:zuul-worker/38-102-83-54:9023 namespace: main pod: pod-centos-9-stream-0000074729 resources: cores: 0 instances: 1 ram: 0 stream_port1: 36373 stream_port2: 44301 tenant: rdoproject.org timeout: 1800 topic: null voting: true