Name:         zuul-scheduler-tooling-config-map
Namespace:    sf
Labels:       <none>
Annotations:  <none>

Data
====
fetch-config-repo.sh:
----
#!/bin/sh

set -ex

# config-update usage context required a specific git ref
REF=$1

if [[ "$CONFIG_REPO_BASE_URL" =~ https://gerrit.sfop.me.* ]]; then
    # This url is not easily reachable from within pods thus into that
    # specific context we can use the Service address.
    CONFIG_REPO_BASE_URL="http://gerrit-httpd:8080"
fi

# Clone or fetch config repository
if [ -d ~/config/.git ]; then
  pushd ~/config
  git remote | grep origin && git remote remove origin
  git remote add origin ${CONFIG_REPO_BASE_URL}/${CONFIG_REPO_NAME}
  if [ -z "$REF" ]; then
    # Discover default remote branch ref
    REF="origin/$(git remote show origin | sed -n '/HEAD branch/s/.*: //p')"
  fi
  if [ "$INIT_CONTAINER" == "1" ]; then
    git fetch origin || true
    git reset --hard $REF || true
  else
    git fetch origin
    git reset --hard $REF
  fi
  popd
else
  pushd ~/
  if [ "$INIT_CONTAINER" == "1" ]; then
    git clone ${CONFIG_REPO_BASE_URL}/${CONFIG_REPO_NAME} config || true
  else
    git clone ${CONFIG_REPO_BASE_URL}/${CONFIG_REPO_NAME} config
  fi
  popd
fi


generate-zuul-tenant-yaml.sh:
----
#!/bin/sh

set -ex

export HOME=/var/lib/zuul

# Generate the default tenants configuration file
cat << EOF > ~/main.yaml
- tenant:
    max-job-timeout: 10800
    name: internal
    report-build-page: true
    admin-rules:
      - admin-internal
    exclude-unprotected-branches: true
    source:
      git-server:
        config-projects:
        - system-config
      opendev.org:
        untrusted-projects:
        - zuul/zuul-jobs
EOF

if [ "$CONFIG_REPO_SET" == "TRUE" ]; then
  # A config repository has been set

  # config-update usage context required a specific git ref
  REF=$1

  /usr/local/bin/fetch-config-repo.sh $REF

  # Ensure the config repo enabled into the tenants config
  cat << EOF >> ~/main.yaml
      ${CONFIG_REPO_CONNECTION_NAME}:
        config-projects:
        - ${CONFIG_REPO_NAME}:
            load-branch: ${CONFIG_REPO_BRANCH}
EOF

  # Append the config repo provided tenant file to the default one
  if [ -f ~/config/zuul/main.yaml ]; then
    cat ~/config/zuul/main.yaml >> ~/main.yaml
  fi

fi

echo "Generated tenants config:"
echo
cat ~/main.yaml


hound-search-config.sh:
----
#!/bin/sh
# Copyright (C) 2025 Red Hat
# SPDX-License-Identifier: Apache-2.0

set -ex

export HOME=/var/lib/hound
bash /sf-tooling/fetch-config-repo.sh $1
exec python3 /sf-tooling/hound-search-render.py


hound-search-init.sh:
----
#!/bin/sh
# Copyright (C) 2025 Red Hat
# SPDX-License-Identifier: Apache-2.0

set -ex

if [ ! -d /etc/pki/ca-trust/extracted/openssl ]; then
  cd /etc/pki/ca-trust/extracted
  mkdir -p openssl pem java edk2
  cd -
  update-ca-trust extract -o /etc/pki/ca-trust/extracted
fi

export HOME=/var/lib/hound
mkdir -p ${HOME}/data
cd $HOME
if [ ! -z "${CONFIG_REPO_BASE_URL}" ]; then
    bash /sf-tooling/hound-search-config.sh
fi
if [ ! -f "/var/lib/hound/config.json" ]; then
    cat <<EOF> /var/lib/hound/config.json
{
 "dbpath": "/var/lib/hound/data",
 "max-concurrent-indexers": 2,
 "repos": {}
}
EOF
fi
exec /go/bin/houndd -conf /var/lib/hound/config.json


hound-search-render.py:
----
#!/bin/env pythons.
# Copyright (C) 2025 Red Hat
# SPDX-License-Identifier: Apache-2.0

import configparser
import json
import sys
import yaml


def read_connections(zuul_conf):
    """Read connections from zuul.conf"""
    parser = configparser.ConfigParser()
    parser.read_string(zuul_conf)
    connections = {}
    for section in parser.sections():
        kv = section.split()
        if kv[0] == "connection":
            if parser.has_option(section, "baseurl"):
                url = parser.get(section, "baseurl").rstrip("/")
            elif section == "connection softwarefactory-project.io":
                url = "https://softwarefactory-project.io/r"
            else:
                url = ""
            # Get the connection name, driver and baseurl
            connections[kv[1]] = dict(
                driver=parser.get(section, "driver"), baseurl=url)
    return connections


test_zuul_conf = """
[merger]
[connection opendev.org]
driver              = gerrit
baseurl             = https://review.opendev.org
[connection gerrit]
driver              = gerrit
baseurl             = https://gerrit.sfop.me
[connection gitlab.com]
driver              = gitlab
baseurl             = https://gitlab.com
[connection github.com]
driver              = github
"""


def read_repos(zuul_yaml):
    """Read repositories from zuul main.yaml"""
    tenants = yaml.safe_load(zuul_yaml)
    projs = []
    for tenant in tenants:
        if not tenant.get("tenant"):
            continue
        for conn, conf in tenant["tenant"].get("source", {}).items():
            for proj in conf.get("config-projects", []) + conf.get(
                "untrusted-projects", []
            ):
                # TODO: add support for project group
                if isinstance(proj, str):
                    # This is a literal project, assume default branch name
                    projs.append((conn, proj))
                else:
                    # This is a project object, it's name is the first key
                    name = list(proj.keys())[0]
                    projs.append((conn, name))

    return projs


test_zuul_yaml = """
- tenant:
    name: demo-tenant
    source:
        gitlab.com:
            config-projects:
                - demo-tenant-config
            untrusted-projects:
                - demo-project
        opendev.org:
            config-projects:
                - zuul/sandbox-config:
                    load-branch: main
            untrusted-projects:
                - zuul/zuul-jobs
        gerrit:
            untrusted-projects:
                - demo-project-local
"""


def get_git_urls(conn, repo):
    """Create the hound URLs from the zuul connection and repo config."""
    base_url = conn["baseurl"]
    if conn["driver"] == "gerrit":
        uri = f"{base_url}/{repo}"
        if base_url.rstrip('/') == "https://gerrit.sfop.me":
            uri = f"http://gerrit-httpd:8080/{repo}"
        gitweb = (
            base_url
            + f"/plugins/gitiles/{repo}/+/{{rev}}/"
            + "{path}{anchor}"
        )
        anchor = "#{line}"
        if "https://review.gerrithub.io" in base_url:
            gitweb = f"http://github.com/{repo}/blob/{{rev}}/" + \
                "{path}{anchor}"
            anchor = "#L{line}"
    elif conn["driver"] == "github":
        uri = f"http://github.com/{repo}"
        gitweb = f"http://github.com/{repo}/blob/{{rev}}/" + "{path}{anchor}"
        anchor = "#L{line}"
    elif conn["driver"] == "pagure":
        uri = base_url + f"/{repo}"
        gitweb = base_url + f"/{repo}/blob/{{rev}}/f/" + "{path}{anchor}"
        anchor = "#_{line}"
    elif conn["driver"] == "gitlab":
        uri = base_url + f"/{repo}"
        gitweb = base_url + f"/{repo}/-/blob/{{rev}}/" + "{path}{anchor}"
        anchor = "#L{line}"
    elif conn["driver"] == "git" and \
            base_url.startswith("https://opendev.org"):
        uri = base_url + f"/{repo}"
        gitweb = base_url + f"/{repo}/src/commit/{{rev}}/" + "{path}{anchor}"
        anchor = "#L{line}"
    else:
        return None, None, None
    return uri, gitweb, anchor


def render_hound(connections, projs):
    """Create the hound-search config"""
    repos = {}
    for conn, repo in projs:
        url, base_url, anchor = get_git_urls(connections[conn], repo)
        if not url:
            continue
        repos[repo] = {
            "url": url,
            "ms-between-poll": int(12 * 60 * 60 * 1000),
            "url-pattern": {
                "base-url": base_url,
                "anchor": anchor,
            },
        }
    return {
        "max-concurrent-indexers": 4,
        "dbpath": "/var/lib/hound/data",
        "vcs-config": {
            "git": {
                "detect-ref": True
            }
        },
        "repos": repos,
    }


def do_main():
    try:
        zuul_yaml = open("/var/lib/hound/config/zuul/main.yaml").read()
    except Exception:
        zuul_yaml = "[]"
    conf = json.dumps(
        render_hound(
            read_connections(open("/etc/zuul/zuul.conf").read()),
            read_repos(zuul_yaml),
        )
    )
    open("/var/lib/hound/config.json", "w").write(conf)


def do_test():
    conf = render_hound(read_connections(test_zuul_conf),
                        read_repos(test_zuul_yaml))
    expected = {
        "max-concurrent-indexers": 4,
        "dbpath": "/var/lib/hound/data",
        "vcs-config": {
            "git": {
                "detect-ref": True
            }
        },
        "repos": {
            "demo-tenant-config": {
                "url": "https://gitlab.com/demo-tenant-config",
                "ms-between-poll": 43200000,
                "url-pattern": {
                    "base-url": "https://gitlab.com/demo-tenant-config/-/" +
                                "blob/{rev}/{path}{anchor}",
                    "anchor": "#L{line}",
                },
            },
            "demo-project": {
                "url": "https://gitlab.com/demo-project",
                "ms-between-poll": 43200000,
                "url-pattern": {
                    "base-url": "https://gitlab.com/demo-project/-/" +
                                "blob/{rev}/{path}{anchor}",
                    "anchor": "#L{line}",
                },
            },
            "zuul/sandbox-config": {
                "url": "https://review.opendev.org/zuul/sandbox-config",
                "ms-between-poll": 43200000,
                "url-pattern": {
                    "base-url": "https://review.opendev.org/plugins/gitiles/" +
                                "zuul/sandbox-config/+/{rev}/" +
                                "{path}{anchor}",
                    "anchor": "#{line}",
                },
            },
            "zuul/zuul-jobs": {
                "url": "https://review.opendev.org/zuul/zuul-jobs",
                "ms-between-poll": 43200000,
                "url-pattern": {
                    "base-url": "https://review.opendev.org/plugins/gitiles/" +
                                "zuul/zuul-jobs/+/{rev}/" +
                                "{path}{anchor}",
                    "anchor": "#{line}",
                },
            },
            "demo-project-local": {
                "url": "http://gerrit-httpd:8080/demo-project-local",
                "ms-between-poll": 43200000,
                "url-pattern": {
                    "base-url": "https://gerrit.sfop.me/plugins/gitiles/" +
                                "demo-project-local/+/{rev}/" +
                                "{path}{anchor}",
                    "anchor": "#{line}",
                },
            },
        },
    }
    if conf != expected:
        print("Bad config:")
        print(conf)


if __name__ == "__main__":
    if "test" in sys.argv:
        do_test()
    else:
        do_main()


init-container.sh:
----
#!/bin/sh

set -ex

# Update the CA Trust chain
update-ca-trust extract -o /etc/pki/ca-trust/extracted

# This is needed when we mount the local zuul source from the host
# to bypass the git ownership verification
# https://git-scm.com/docs/git-config#Documentation/git-config.txt-safedirectory
git config --global --add safe.directory $HOME/config

# Generate the Zuul tenant configuration
/usr/local/bin/generate-zuul-tenant-yaml.sh

reconnect-zk.py:
----
#!/bin/env python3
# Copyright (C) 2025 Red Hat
# SPDX-License-Identifier: Apache-2.0

# A small script to force zookeeper reconnection after a service restart.

import socket
import os


def main():
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    def send_cmd(code):
        if not code:
            s.connect(("127.0.0.1", 3000))
        else:
            s.send(b"print(self.server.scheduler." + code + b")\r\n")
        return s.recv(1024)

    send_cmd(None)
    send_cmd(b"log.info('Restarting zookeeper client')")
    send_cmd(b"zk_client.client.stop()")
    send_cmd(b"zk_client.client.start()")
    return b"imok" in send_cmd(b"zk_client.client.command(b'ruok')")


os.system("zuul-scheduler repl")
try:
    result = main()
finally:
    os.system("zuul-scheduler norepl")
exit(result)


rotate-keystore.py:
----
#!/usr/bin/env python3
# Copyright (C) 2026 Red Hat
# SPDX-License-Identifier: Apache-2.0

import configparser
import sys
import json

from zuul.lib.keystorage import KeyStorage
from zuul.lib import encryption
from zuul.zk import ZooKeeperClient

try:
    old_password = sys.argv[1].encode("utf-8")
    new_password = sys.argv[2].encode("utf-8")
except IndexError:
    print("usage: rotate-keystore old-password new-password")
    sys.exit(1)


def encrypt_keys(keys):
    new_keys = dict()
    for path, obj in keys.items():
        new_keys[path] = dict(keys=[])
        for key in obj["keys"]:
            pem_private_key = key.get("private_key").encode("utf-8")
            private_key, public_key = encryption.deserialize_rsa_keypair(
                pem_private_key, old_password
            )
            encrypted_private_key = encryption.serialize_rsa_private_key(
                private_key, new_password
            )
            new_key = key.copy()
            new_key["private_key"] = encrypted_private_key.decode("utf-8")
            new_keys[path]["keys"].append(new_key)
    return new_keys


# Connect to ZooKeeper
config = configparser.ConfigParser()
config.read("/etc/zuul/zuul.conf")
zk_client = ZooKeeperClient.fromConfig(config)
zk_client.connect()

# Read the keys
ks = KeyStorage(zk_client, "unused")
old_keys = ks.exportKeys()
with open("/var/lib/zuul/keys-backup.json", "w") as f:
    json.dump(old_keys, f)

# Write the new keys
new_keys = dict(keys=encrypt_keys(old_keys["keys"]))
ks.importKeys(new_keys, True)


zuul-change-dump.py:
----
#!/bin/env python3
# Copyright 2013 OpenStack Foundation
# Copyright 2015 Hewlett-Packard Development Company, L.P.
# Copyright 2016 Red Hat
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import requests
import argparse
import os
import time


dump_file = "/var/lib/zuul/zuul-queues-dump.sh"


def get_tenants(args):
    req = requests.get("%s/tenants" % args.url)
    tenants = req.json()
    status = {}
    for tenant in tenants:
        req = requests.get("%s/tenant/%s/status" % (args.url, tenant["name"]))
        status[tenant["name"]] = req.json()
    if os.path.isfile(args.dump_file):
        os.rename(args.dump_file, "%s.orig" % args.dump_file)
    return (tenants, status)


def retry_get_tenants(args):
    count, max_count = 0, 5
    while True:
        try:
            return get_tenants(args)
        except Exception as e:
            if count > max_count:
                raise e
            count += 1
            print("Request fails (%s), retrying %d/%d" %
                  (str(e), count, max_count))
            time.sleep(count * 2)


def dump(args):
    (tenants, status) = retry_get_tenants(args)
    of = open(args.dump_file, "w")
    of.write("#/bin/sh\nset -ex\n")
    for tenant in tenants:
        for pipeline in status[tenant["name"]].get('pipelines', []):
            for queue in pipeline['change_queues']:
                for head in queue['heads']:
                    for change in head:
                        if (not change['live'] or
                                not change.get('id') or
                                ',' not in change['id']):
                            continue
                        cid, cps = change['id'].split(',')
                        cmd = (
                            "zuul-client enqueue --tenant %s "
                            "--pipeline %s --project %s --change %s,%s" % (
                                tenant["name"],
                                pipeline['name'],
                                change['project_canonical'],
                                cid, cps)
                        )
                        if ";" in cmd or "|" in cmd:
                            raise RuntimeError("Forbidden char in [%s]" % cmd)
                        print(cmd)
                        of.write("%s\n" % cmd)
    of.write(
        "curl %s/info 2>&1 | grep 'capabilities' > /dev/null\n" % args.url)
    of.write("echo SUCCESS: zuul queues restored\n")
    of.close()
    os.chmod(args.dump_file, 0o755)


def load(args):
    if not os.path.isfile(args.dump_file):
        print("%s: no such file, please dump first" % args.dump_file)
    if os.stat(args.dump_file).st_mtime + 172800 < time.time():
        if not args.force:
            raise RuntimeError("%s is too old, use --force to use it" %
                               args.dump_file)
    os.system(dump_file)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--force', action="store_const", const=True)
    parser.add_argument('--url', default="https://sfop.me/zuul/api")
    parser.add_argument('--dump_file', default=dump_file)
    parser.add_argument('action', choices=("dump", "load"))
    args = parser.parse_args()
    if args.action == "dump":
        dump(args)
    else:
        load(args)



BinaryData
====

Events:  <none>