tasks: Add local mock PR run to run-local.sh

Makefile

@@ -22,7 +22,7 @@ tasks-shell:
 	$(DOCKER) run -ti --rm \
 		--shm-size=1024m \
 		--volume=$(CURDIR)/tasks:/usr/local/bin \
-		--volume=$(TASK_SECRETS):/secrets:ro \
+		--volume=$(TASK_SECRETS):/run/secrets/tasks/:ro \
 		--volume=$(WEBHOOK_SECRETS):/run/secrets/webhook/:ro \
 		--volume=$(TASK_CACHE):/cache:rw \
 		--entrypoint=/bin/bash \

tasks/Containerfile

@@ -68,13 +68,11 @@ COPY setup-tasks cockpit-tasks install-service webhook github_handler.py /usr/lo
 
 RUN groupadd -g 1111 -r user && useradd -r -g user -u 1111 user --home-dir /work && \
     groupadd -g 1001 -r github && useradd -r --no-create-home -g github -u 1001 github && \
-    mkdir -p /usr/local/bin /secrets /cache/images /cache/github && \
+    mkdir -p /usr/local/bin /cache/images /cache/github && \
     mkdir -p /work/.config /work/.config/cockpit-dev /work/.ssh /work/.cache /work/.rhel && \
     printf '[user]\n\t\nemail = cockpituous@cockpit-project.org\n\tname = Cockpituous\n[cockpit "bots"]\n\timages-data-dir = /cache/images\n' >/work/.gitconfig && \
-    ln -snf /secrets/s3-keys /work/.config/cockpit-dev/s3-keys && \
-    ln -snf /run/secrets/webhook/.config--github-token /work/.config/github-token && \
     chmod g=u /etc/passwd && \
-    chmod -R ugo+w /cache /secrets /cache /work && \
+    chmod -R ugo+w /cache /work && \
     chown -R user:user /cache /work && \
     printf '[libdefaults]\ndefault_ccache_name = FILE:/tmp/krb5.ccache\n' > /etc/krb5.conf.d/0_file_ccache && \
     echo 'user ALL=NOPASSWD: /usr/bin/chmod 666 /dev/kvm' > /etc/sudoers.d/user-fix-kvm
@@ -86,4 +84,3 @@ VOLUME /cache/images
 
 USER user
 WORKDIR /work
-CMD ["/usr/local/bin/cockpit-tasks", "--verbose"]

tasks/cockpit-tasks

@@ -4,9 +4,6 @@ set -eux
 
 setup-tasks
 
-COCKPIT_BOTS_REPO=${COCKPIT_BOTS_REPO:-https://github.com/cockpit-project/bots}
-COCKPIT_BOTS_BRANCH=${COCKPIT_BOTS_BRANCH:-main}
-
 # let's just do our work in the current directory
 WORKDIR="$PWD"
 BOTS_DIR="$WORKDIR"/bots
@@ -19,11 +16,7 @@ fi
 echo "Starting testing"
 
 function update_bots() {
-    if [ -d "$BOTS_DIR" ]; then
-        git -C "$BOTS_DIR" pull --rebase
-    else
-        git clone --quiet -b "$COCKPIT_BOTS_BRANCH" "$COCKPIT_BOTS_REPO" "$BOTS_DIR"
-    fi
+    git -C "$BOTS_DIR" pull --rebase
 }
 
 # wait between 1 and 10 minutes, but not in an interactive terminal (annoying for debugging)

tasks/cockpit-tasks-centosci.yaml

@@ -22,7 +22,7 @@ spec:
           value: '1'
         volumeMounts:
         - name: secrets
-          mountPath: "/secrets"
+          mountPath: /run/secrets/tasks
           readOnly: true
         - name: webhook-secrets
           mountPath: /run/secrets/webhook

tasks/install-service

@@ -23,7 +23,7 @@ chown -R 1111:1111 $SECRETS $CACHE
 chcon -R -t container_file_t $SECRETS $CACHE
 
 if [ -e "${SECRETS}/tasks/npm-registry.crt" ]; then
-    NODE_EXTRA_CA_CERTS=/secrets/npm-registry.crt
+    NODE_EXTRA_CA_CERTS=/run/secrets/tasks/npm-registry.crt
 fi
 
 if [ $INSTANCES -eq 1 ]; then
@@ -62,15 +62,15 @@ ExecStart=/usr/bin/podman run --name=cockpit-tasks-%i --hostname=${CONTAINER_HOS
     --device=/dev/kvm --network=cockpit-tasks-%i \
     --memory=24g --pids-limit=16384 --shm-size=1024m ${TMPVOL:-} \
     --volume=\${TEST_CACHE}/images:/cache/images:rw \
-    --volume=\${TEST_SECRETS}/tasks:/secrets:ro \
+    --volume=\${TEST_SECRETS}/tasks:/run/secrets/tasks:ro \
     --volume=\${TEST_SECRETS}/webhook:/run/secrets/webhook:ro \
     --volume=${IMAGE_STORES}:/work/.config/cockpit-dev/image-stores:ro \
     --env=NPM_REGISTRY=\${NPM_REGISTRY} \
     --env=NODE_EXTRA_CA_CERTS=${NODE_EXTRA_CA_CERTS:-} \
     --env=TEST_JOBS=\${TEST_JOBS} \
     --env=TEST_NOTIFICATION_MX=\${TEST_NOTIFICATION_MX} \
     --env=TEST_NOTIFICATION_TO=\${TEST_NOTIFICATION_TO} \
-    quay.io/cockpit/tasks
+    quay.io/cockpit/tasks cockpit-tasks --verbose
 ExecStop=/usr/bin/podman rm -f cockpit-tasks-%i
 ExecStop=/usr/bin/podman network rm cockpit-tasks-%i
 

tasks/mock-github-pr

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+# Mock GitHub API server for testing an opened PR
+# You can run this manually in `tasks/run-local.sh -i` with `podman cp` and running
+# cd bots
+# PYTHONPATH=. ./mock-github-pr cockpit-project/bots $(git rev-parse HEAD) &
+# export GITHUB_API=http://127.0.0.7:8443
+# PYTHONPATH=. ./mock-github-pr --print-event cockpit-project/bots $(git rev-parse HEAD) | \
+#     ./publish-queue --amqp localhost:5671 --queue webhook
+#
+# and then two `./run-queue --amqp localhost:5671`
+# first to process webhook → tests-scan → public, second to actually run it
+
+import argparse
+import json
+import os
+import tempfile
+
+from task.test_mock_server import MockHandler, MockServer
+
+repo = None
+sha = None
+
+
+class Handler(MockHandler):
+    def do_GET(self):
+        if self.path in self.server.data:
+            self.replyJson(self.server.data[self.path])
+        elif self.path.startswith(f'/repos/{repo}/pulls?'):
+            self.replyJson([self.server.data[f'/repos/{repo}/pulls/1']])
+        elif self.path == f'/{repo}/{sha}/.cockpit-ci/container':
+            self.replyData('quay.io/cockpit/tasks')
+        else:
+            self.send_error(404, 'Mock Not Found: ' + self.path)
+
+    def do_POST(self):
+        if self.path.startswith(f'/repos/{repo}/statuses/{sha}'):
+            self.replyJson({})
+        else:
+            self.send_error(405, 'Method not allowed: ' + self.path)
+
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument('--port', type=int, default=8443, help="Port to listen on (default: %(default)s)")
+argparser.add_argument('--print-event', action='store_true', help="Print GitHub webhook pull_request event and exit")
+argparser.add_argument('repo', metavar='USER/PROJECT', help="GitHub user/org and project name")
+argparser.add_argument('sha', help="SHA to test in repo for the mock PR")
+args = argparser.parse_args()
+repo = args.repo
+sha = args.sha
+
+ADDRESS = ('127.0.0.7', args.port)
+
+GITHUB_DATA = {
+    f'/repos/{repo}/pulls/1': {
+        'title': 'mock PR',
+        'number': 1,
+        'state': 'open',
+        'body': "This is the body",
+        'base': {'repo': {'full_name': repo}, 'ref': 'main'},
+        'head': {'sha': args.sha, 'user': {'login': repo.split('/')[0]}},
+        'labels': [],
+        'updated_at': 0,
+    },
+    f'/repos/{repo}/commits/{args.sha}/status?page=1&per_page=100': {
+        'state': 'pending',
+        'statuses': [],
+        'sha': sha,
+    },
+}
+
+if args.print_event:
+    print(json.dumps({
+        'event': 'pull_request',
+        'request': {
+            'action': 'opened',
+            'pull_request': GITHUB_DATA[f'/repos/{repo}/pulls/1']
+        }
+    }, indent=4))
+    exit(0)
+
+temp = tempfile.TemporaryDirectory()
+cache_dir = os.path.join(temp.name, 'cache')
+os.environ['XDG_CACHE_HOME'] = cache_dir
+server = MockServer(ADDRESS, Handler, GITHUB_DATA)
+server.start()
+print(f'export GITHUB_API=http://{ADDRESS[0]}:{ADDRESS[1]}')

tasks/run-local.sh

@@ -14,6 +14,8 @@ S3_PORT=${S3_PORT:-9000}
 S3_URL_POD=https://localhost.localdomain:9000
 # S3 address from host
 S3_URL_HOST=https://localhost.localdomain:$S3_PORT
+# AMQP address from inside the cockpituous pod
+AMQP_POD=localhost:5671
 
 # CLI option defaults/values
 PR=
@@ -152,7 +154,7 @@ EOF
         [ -z "$TOKEN" ] || cp -fv "$TOKEN" "$SECRETS"/webhook/.config--github-token
         podman run -d --name cockpituous-webhook --pod=cockpituous --user user \
             -v "$SECRETS"/webhook:/run/secrets/webhook:ro,z \
-            -e AMQP_SERVER=localhost:5671 \
+            -e AMQP_SERVER=$AMQP_POD \
             quay.io/cockpit/tasks:${TASKS_TAG:-latest} webhook
     fi
 
@@ -165,16 +167,18 @@ EOF
 
     # Run tasks container in the backgroud
     podman run -d -it --name cockpituous-tasks --pod=cockpituous \
-        -v "$SECRETS"/tasks:/secrets:ro,z \
+        -v "$SECRETS"/tasks:/run/secrets/tasks:ro,z \
         -v "$SECRETS"/webhook:/run/secrets/webhook:ro,z \
         -e COCKPIT_CA_PEM=/run/secrets/webhook/ca.pem \
         -e COCKPIT_BOTS_REPO=${COCKPIT_BOTS_REPO:-} \
         -e COCKPIT_BOTS_BRANCH=${COCKPIT_BOTS_BRANCH:-} \
         -e COCKPIT_TESTMAP_INJECT=main/unit-tests \
-        -e AMQP_SERVER=localhost:5671 \
+        -e AMQP_SERVER=$AMQP_POD \
         -e S3_LOGS_URL=$S3_URL_POD/logs/ \
         -e SKIP_STATIC_CHECK=1 \
-        quay.io/cockpit/tasks:${TASKS_TAG:-latest} ${INTERACTIVE:+sleep infinity}
+        quay.io/cockpit/tasks:${TASKS_TAG:-latest} bash
+
+    podman exec -i cockpituous-tasks setup-tasks
 }
 
 cleanup_containers() {
@@ -183,20 +187,12 @@ cleanup_containers() {
     # clean up dummy token, so that image-prune does not try to use it
     rm "$SECRETS"/webhook/.config--github-token
 
-    if [ -n "$INTERACTIVE" ]; then
-        podman stop --time=0 cockpituous-tasks
-    else
-        # tell the tasks container iteration that we are done
-        podman exec cockpituous-tasks kill -TERM 1
-    fi
+    podman stop --time=0 cockpituous-tasks
 }
 
 test_image() {
     # test image upload
     podman exec -i cockpituous-tasks timeout 30 sh -euxc '
-        # wait until tasks container has set up itself and checked out bots
-        until [ -f bots/tests-trigger ]; do echo "waiting for tasks to initialize"; sleep 5; done
-
         cd bots
 
         # fake an image
@@ -236,21 +232,66 @@ test_image() {
     '
 }
 
+test_mock_pr() {
+    podman cp "$MYDIR/mock-github-pr" cockpituous-tasks:/work/bots/mock-github-pr
+    podman exec -i cockpituous-tasks sh -euxc "
+        cd bots
+        # test mock PR against our checkout, so that cloning will work
+        SHA=\$(git rev-parse HEAD)
+
+        # start mock GH server
+        PYTHONPATH=. ./mock-github-pr cockpit-project/bots \$SHA &
+        GH_MOCK_PID=\$!
+        export GITHUB_API=http://127.0.0.7:8443
+        until curl --silent \$GITHUB_API; do sleep 0.1; done
+
+        # simulate GitHub webhook event, put that into the webhook queue
+        PYTHONPATH=. ./mock-github-pr --print-event cockpit-project/bots \$SHA | \
+            ./publish-queue --amqp $AMQP_POD --create --queue webhook
+
+        ./inspect-queue --amqp $AMQP_POD
+
+        # first run-queue processes webhook → tests-scan → public queue
+        ./run-queue --amqp $AMQP_POD
+        ./inspect-queue --amqp $AMQP_POD
+
+        # second run-queue actually runs the test
+        ./run-queue --amqp $AMQP_POD
+
+        kill \$GH_MOCK_PID
+    "
+
+    LOGS_URL="$S3_URL_HOST/logs/"
+    CURL="curl --cacert $SECRETS/ca.pem --silent --fail --show-error"
+    LOG_MATCH="$($CURL $LOGS_URL| grep -o "pull-1-[[:alnum:]-]*-unit-tests/log<")"
+    LOG="$($CURL "${LOGS_URL}${LOG_MATCH%<}")"
+    echo "--------------- mock PR test log -----------------"
+    echo  "$LOG"
+    echo "--------------- mock PR test log end -------------"
+    assert_in 'Test run finished' "$LOG"
+}
+
 test_pr() {
     # need to use real GitHub token for this
     [ -z "$TOKEN" ] || cp -fv "$TOKEN" "$SECRETS"/webhook/.config--github-token
 
+    # run the main loop in the background; we could do this with a single run-queue invocation,
+    # but we want to test the cockpit-tasks script
+    podman exec -i cockpituous-tasks cockpit-tasks &
+    TASKS_PID=$!
+
     podman exec -i cockpituous-tasks sh -euxc "
-    cd bots;
-    ./tests-scan -p $PR --amqp 'localhost:5671' --repo $PR_REPO;
+    cd bots
+
+    ./tests-scan -p $PR --amqp '$AMQP_POD' --repo $PR_REPO;
     for retry in \$(seq 10); do
         ./tests-scan --repo $PR_REPO --human-readable --dry;
         OUT=\$(./tests-scan --repo $PR_REPO -p $PR --human-readable --dry);
         [ \"\${OUT%unit-tests*}\" = \"\$OUT\" ] || break;
         echo waiting until the status is visible;
         sleep 10;
     done;
-    ./inspect-queue --amqp localhost:5671;"
+    ./inspect-queue --amqp $AMQP_POD;"
 
     LOGS_URL="$S3_URL_HOST/logs/"
     CURL="curl --cacert $SECRETS/ca.pem --silent --fail --show-error"
@@ -261,6 +302,11 @@ test_pr() {
         echo waiting for unit-tests run to finish...
         sleep 10
     done
+
+    # tell the tasks container iteration that we are done
+    kill -TERM $TASKS_PID
+    wait $TASKS_PID || true
+
     LOG_PATH="${LOG_MATCH%<}"
 
     # spot-checks that it produced sensible logs in S3
@@ -282,8 +328,8 @@ test_pr() {
 
 test_queue() {
     # tasks can connect to queue
-    OUT=$(podman exec -i cockpituous-tasks bots/inspect-queue --amqp localhost:5671)
-    echo "$OUT" | grep -q 'queue public is empty'
+    OUT=$(podman exec -i cockpituous-tasks bots/inspect-queue --amqp $AMQP_POD)
+    echo "$OUT" | grep -q 'queue public does not exist'
 }
 
 #
@@ -298,16 +344,14 @@ launch_containers
 podman logs -f cockpituous-tasks &
 
 if [ -n "$INTERACTIVE" ]; then
-    # check out the correct bots, as part of what cockpit-tasks would usually do
-    podman exec cockpituous-tasks sh -euc \
-        'git clone --quiet --depth=1 -b "${COCKPIT_BOTS_BRANCH:-main}" "${COCKPIT_BOTS_REPO:-https://github.com/cockpit-project/bots}"'
-
     echo "Starting a tasks container shell; exit it to clean up the deployment"
     podman exec -it cockpituous-tasks bash
 else
     # tests which don't need GitHub interaction
     test_image
     test_queue
+    # "almost" end-to-end, starting with GitHub webhook JSON payload injection; fully localy, no privs
+    test_mock_pr
     # if we have a PR number, run a unit test inside local deployment, and update PR status
     [ -z "$PR" ] || test_pr
 fi

tasks/setup-tasks

@@ -21,17 +21,27 @@ npm config set fetch-timeout 600000
 npm config set fetch-retry-mintimeout 60000
 npm config set maxsockets 3
 
-# set up S3 keys for OpenShift secrets volume
-if [ ! -d /secrets/s3-keys ]; then
-    # then our container symlink will point into the void, replace it with a directory and set up all files that we can find
-    rm ~/.config/cockpit-dev/s3-keys
-    mkdir ~/.config/cockpit-dev/s3-keys
-    for f in /secrets/s3-keys--*; do
-        [ -e "$f" ] || continue # non-matching glob
-        ln -s "$f" ~/.config/cockpit-dev/s3-keys/"${f#*--}"
-    done
+# Set up secrets
+if [ -d /run/secrets/tasks ]; then
+    ls -l ~/.config/cockpit-dev/
+    ln -snf /run/secrets/tasks/s3-keys ~/.config/cockpit-dev/s3-keys
+    ln -snf /run/secrets/webhook/.config--github-token ~/.config/github-token
+    git config --global credential.helper store
+    echo "https://cockpituous:$(cat ~/.config/github-token)@github.com" > ~/.git-credentials
+
+    # set up S3 keys for OpenShift secrets volume, where there is just a flat hierarchy with "--" encoding
+    if [ ! -d /run/secrets/tasks/s3-keys ] && [ ! -d ~/.config/cockpit-dev/s3-keys ]; then
+        # then our container symlink will point into the void, replace it with a directory and set up all files that we can find
+        rm ~/.config/cockpit-dev/s3-keys
+        mkdir ~/.config/cockpit-dev/s3-keys
+        for f in /run/secrets/tasks/s3-keys--*; do
+            [ -e "$f" ] || continue # non-matching glob
+            ln -s "$f" ~/.config/cockpit-dev/s3-keys/"${f#*--}"
+        done
+    fi
 fi
 
-# Set up github user and token
-git config --global credential.helper store
-echo "https://cockpituous:$(cat ~/.config/github-token)@github.com" > ~/.git-credentials
+# Get bots
+if [ ! -d ~/bots ]; then
+    git clone --quiet -b "${COCKPIT_BOTS_BRANCH:-main}" "${COCKPIT_BOTS_REPO:-https://github.com/cockpit-project/bots}" ~/bots
+fi