From f50cf2ebd261ec02c6aa3f2a9506b07f00891084 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 11:54:16 -0500 Subject: [PATCH 1/5] INTEROP-9430: fix interop-tests-opp-quay-smoke step Three root causes fixed: 1. Missing skopeo: add cli-with-skopeo image via dockerfile_literal in all 4 ci-operator configs, update ref.yaml from: cli-with-skopeo. 2. Broken Quay auth: rewrite GetQuayAuth to read credentials from the quayadmin secret (created by ACM PolicySet admin-user job). Falls back to quaydevel, then /api/v1/user/initialize. Use $oauthtoken for token-only registry auth. CSRF signin flow for org creation. 3. Wrong ODF validation: rewrite to validate OBCs and NooBaa health instead of PVC storage classes (NooBaa using gp3-csi is by design). 4. ACS scan timeout: delete stale ACS integrations, register Quay as a fresh Docker-type integration, explicitly trigger scans via /v1/images/scan. Extended timeout to 10 minutes with retries. Applies mpitt best practices: set -eux -o pipefail, SC2155 splits, camelCase locals, subshell trap handler, secret handling with set +x/set -x, heredoc quoting, removed stdout suppression. --- ...-policy-collection-main__ocp4.22-fips.yaml | 6 + ...stron-policy-collection-main__ocp4.22.yaml | 6 + ...ostron-policy-collection-main__ocp5.0.yaml | 6 + ...ostron-policy-collection-main__ocp5.1.yaml | 6 + .../interop-tests-opp-quay-smoke-commands.sh | 346 +++++++++++++----- .../interop-tests-opp-quay-smoke-ref.yaml | 7 +- 6 files changed, 278 insertions(+), 99 deletions(-) diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml index 1221f3a80be7a..f5050d7facf2d 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22-fips.yaml @@ -40,6 +40,12 @@ images: from: cli optional: true to: cli-with-git + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y skopeo && dnf clean all + from: cli + optional: true + to: cli-with-skopeo - dockerfile_literal: | FROM registry.access.redhat.com/ubi9/openjdk-17:1.21 USER root diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml index c3a92c8259122..1ecbaaae9cca9 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml @@ -40,6 +40,12 @@ images: from: cli optional: true to: cli-with-git + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y skopeo && dnf clean all + from: cli + optional: true + to: cli-with-skopeo - dockerfile_literal: | FROM registry.access.redhat.com/ubi9/openjdk-17:1.21 USER root diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml index e31b7716529a3..13d0986a1f142 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.0.yaml @@ -36,6 +36,12 @@ images: from: cli optional: true to: cli-with-git + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y skopeo && dnf clean all + from: cli + optional: true + to: cli-with-skopeo releases: latest: candidate: diff --git a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml index 8598841060b45..8fb2da67b580c 100644 --- a/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml +++ b/ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp5.1.yaml @@ -36,6 +36,12 @@ images: from: cli optional: true to: cli-with-git + - dockerfile_literal: | + FROM this-is-ignored + RUN dnf install -y skopeo && dnf clean all + from: cli + optional: true + to: cli-with-skopeo releases: latest: candidate: diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index f726090036915..2e13025e8eb65 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -1,18 +1,19 @@ #!/bin/bash -set -euo pipefail +set -eux -o pipefail shopt -s inherit_errexit ARTIFACT_DIR="${ARTIFACT_DIR:=/tmp/artifacts}" mkdir -p "${ARTIFACT_DIR}" typeset junitFile="${ARTIFACT_DIR}/junit_quay_interop.xml" -typeset imageTag="${BUILD_ID:-$(date +%s)}" +typeset imageTag='' +imageTag="${BUILD_ID:-$(date +%s)}" typeset -A testStatus typeset -A testDuration typeset -A testFailureMsg typeset -a allTests=( "[sig-interop][Jira:INTEROP][Feature:Quay] Push and pull image via Quay route" - "[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF PVC backing Quay storage" + "[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF object storage integration" "[sig-interop][Jira:INTEROP][Feature:Quay] ACS scan of pushed Quay image" ) @@ -33,13 +34,15 @@ function RecordResult () { testStatus["${name}"]="${status}" testDuration["${name}"]="${dur}" testFailureMsg["${name}"]="${msg}" + true } # shellcheck disable=SC2329 function GenerateJunit () { typeset -i total=${#allTests[@]} typeset -i failures=0 skipped=0 - typeset -i elapsed=$(( $(date +%s) - suiteStart )) + typeset -i elapsed=0 + elapsed=$(( $(date +%s) - suiteStart )) for t in "${allTests[@]}"; do [[ "${testStatus[${t}]}" == "failed" ]] && failures=$((failures + 1)) @@ -53,94 +56,135 @@ function GenerateJunit () { EOF for t in "${allTests[@]}"; do - typeset escaped_name - escaped_name=$(printf '%s' "${t}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') - typeset escaped_msg - escaped_msg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') + typeset escapedName + escapedName=$(printf '%s' "${t}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') + typeset escapedMsg + escapedMsg=$(printf '%s' "${testFailureMsg[${t}]}" | sed 's/&/\&/g; s//\>/g; s/"/\"/g') if [[ "${testStatus[${t}]}" == "failed" ]]; then - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" elif [[ "${testStatus[${t}]}" == "skipped" ]]; then - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" else - echo " " >> "${junitFile}" + echo " " >> "${junitFile}" fi done - cat >> "${junitFile}" <> "${junitFile}" <<'EOF' EOF cat "${junitFile}" + true } -trap GenerateJunit EXIT +trap '{ ( GenerateJunit; true ); }' EXIT function DiscoverQuay () { QUAY_NS=$(oc get quayregistry --all-namespaces -o jsonpath='{.items[0].metadata.namespace}') QUAY_REGISTRY=$(oc get quayregistry -n "${QUAY_NS}" -o jsonpath='{.items[0].metadata.name}') QUAY_HOST=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.status.registryEndpoint}') QUAY_HOST="${QUAY_HOST#https://}" + if [[ -z "${QUAY_HOST}" ]]; then + echo "ERROR: Quay registry route not ready (empty host)" >&2 + return 1 + fi export QUAY_NS QUAY_REGISTRY QUAY_HOST + true } function GetQuayAuth () { - typeset configSecret - configSecret=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.spec.configBundleSecret}') - if [[ -z "${configSecret}" ]]; then - configSecret="${QUAY_REGISTRY}-config-bundle" - fi - - QUAY_USER=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_EMAIL}' 2>/dev/null | base64 -d || echo "") - if [[ -z "${QUAY_USER}" ]]; then + QUAY_USER="" + QUAY_PASSWORD="" + QUAY_TOKEN="" + + set +x + if oc get secret quayadmin -n "${QUAY_NS}" 2>/dev/null; then + QUAY_TOKEN=$(oc get secret quayadmin -n "${QUAY_NS}" -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_TOKEN="" + QUAY_PASSWORD=$(oc get secret quayadmin -n "${QUAY_NS}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD="" QUAY_USER="quayadmin" + if [[ -n "${QUAY_TOKEN}" || -n "${QUAY_PASSWORD}" ]]; then + set -x + echo "INFO: Quay credentials obtained from quayadmin secret" + export QUAY_USER QUAY_PASSWORD QUAY_TOKEN + return 0 + fi fi - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${configSecret}" -o jsonpath='{.data.SUPER_USER_PASSWORD}' 2>/dev/null | base64 -d || echo "") - if [[ -z "${QUAY_PASSWORD}" ]]; then - typeset initSecret="${QUAY_REGISTRY}-init-config-bundle-secret" - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${initSecret}" -o jsonpath='{.data.superuser-password}' 2>/dev/null | base64 -d || echo "") + if oc get secret quaydevel -n "${QUAY_NS}" 2>/dev/null; then + QUAY_PASSWORD=$(oc get secret quaydevel -n "${QUAY_NS}" -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || QUAY_PASSWORD="" + QUAY_USER="quaydevel" + if [[ -n "${QUAY_PASSWORD}" ]]; then + set -x + echo "INFO: Quay credentials obtained from quaydevel secret" + export QUAY_USER QUAY_PASSWORD QUAY_TOKEN + return 0 + fi fi - if [[ -z "${QUAY_PASSWORD}" ]]; then - for secret in $(oc get secrets -n "${QUAY_NS}" -o jsonpath='{.items[*].metadata.name}' | tr ' ' '\n' | grep -i "quay.*config"); do - QUAY_PASSWORD=$(oc get secret -n "${QUAY_NS}" "${secret}" -o go-template='{{index .data "config.yaml"}}' 2>/dev/null | base64 -d | grep -oP "(?<=SUPER_USER_PASSWORD: ).*" || echo "") - [[ -n "${QUAY_PASSWORD}" ]] && break - done + typeset initPassword + initPassword=$(python3 -c "import secrets,string; print(''.join(secrets.choice(string.ascii_letters+string.digits) for _ in range(20)))") + typeset initResult + initResult=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/user/initialize" \ + -H "Content-Type: application/json" \ + -d "{\"username\":\"quayadmin\",\"password\":\"${initPassword}\",\"email\":\"quayadmin@example.com\",\"access_token\":true}" 2>/dev/null) || initResult="" + + QUAY_TOKEN=$(echo "${initResult}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('access_token',''))" 2>/dev/null) || QUAY_TOKEN="" + if [[ -n "${QUAY_TOKEN}" ]]; then + QUAY_USER="quayadmin" + QUAY_PASSWORD="${initPassword}" + set -x + echo "INFO: Quay admin user initialized via /api/v1/user/initialize" + export QUAY_USER QUAY_PASSWORD QUAY_TOKEN + return 0 fi + set -x - export QUAY_USER QUAY_PASSWORD + echo "ERROR: Could not obtain Quay credentials from any source" >&2 + export QUAY_USER QUAY_PASSWORD QUAY_TOKEN + return 1 } function PreflightCheck () { if ! curl -sk --connect-timeout 15 "https://${QUAY_HOST}/api/v1/discovery" | grep -qi "quay"; then - echo "ERROR: Quay route not reachable at ${QUAY_HOST}" >&2 + echo "ERROR: Quay registry endpoint not reachable" >&2 return 1 fi + true } function CreateTestOrg () { - typeset signinPayload - signinPayload=$(python3 -c "import json,sys; print(json.dumps({'user':sys.argv[1],'pass':sys.argv[2]}))" "${QUAY_USER}" "${QUAY_PASSWORD}") - typeset token - token=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/signin" \ - -H "Content-Type: application/json" \ - -d "${signinPayload}" | \ - python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null || echo "") - - if [[ -z "${token}" ]]; then - token=$(curl -sk -H "Authorization: Basic $(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64)" \ - "https://${QUAY_HOST}/api/v1/user/" | \ - python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('token',''))" 2>/dev/null || echo "") + if [[ -z "${QUAY_TOKEN}" && -n "${QUAY_PASSWORD}" ]]; then + typeset cookieFile="/tmp/quay-cookies.txt" + typeset csrf + csrf=$(curl -sk "https://${QUAY_HOST}/csrf_token" -c "${cookieFile}" | \ + python3 -c "import sys,json; print(json.load(sys.stdin).get('csrf_token',''))" 2>/dev/null) || csrf="" + + if [[ -n "${csrf}" ]]; then + typeset signinResult + set +x + signinResult=$(curl -sk -X POST "https://${QUAY_HOST}/api/v1/signin" \ + -H "Content-Type: application/json" \ + -H "X-CSRF-Token: ${csrf}" \ + -b "${cookieFile}" -c "${cookieFile}" \ + -d "{\"username\":\"${QUAY_USER}\",\"password\":\"${QUAY_PASSWORD}\"}" 2>/dev/null) || signinResult="" + QUAY_TOKEN=$(echo "${signinResult}" | \ + python3 -c "import sys,json; print(json.load(sys.stdin).get('token',''))" 2>/dev/null) || QUAY_TOKEN="" + set -x + fi + rm -f "${cookieFile}" + export QUAY_TOKEN fi - QUAY_TOKEN="${token}" - export QUAY_TOKEN + if [[ -z "${QUAY_TOKEN}" ]]; then + echo "WARNING: No Quay token available; org creation may fail" >&2 + fi curl -sk -X POST "https://${QUAY_HOST}/api/v1/organization/" \ -H "Authorization: Bearer ${QUAY_TOKEN}" \ -H "Content-Type: application/json" \ -d '{"name":"interop-smoke-test","email":"interop-test@example.com"}' || true + true } ################################################################################ @@ -154,9 +198,24 @@ function RunPushPull () { typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" typeset authFile="/tmp/quay-auth.json" + if [[ -z "${QUAY_TOKEN}" && -z "${QUAY_PASSWORD}" ]]; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "No valid Quay authentication token or password available" "${elapsed}" + return 1 + fi + + typeset registryAuth + set +x + if [[ -n "${QUAY_TOKEN}" ]]; then + registryAuth=$(echo -n "\$oauthtoken:${QUAY_TOKEN}" | base64) + else + registryAuth=$(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64) + fi + cat > "${authFile}" </dev/null 2>&1; then + "docker://${pushTarget}"; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "Image not pullable from Quay after push" "${elapsed}" return 1 @@ -181,64 +240,56 @@ EOF } ################################################################################ -# Test Case 2: Verify ODF PVC backing Quay storage +# Test Case 2: Verify ODF object storage integration ################################################################################ -function RunOdfPvcCheck () { - typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF PVC backing Quay storage" +function RunOdfStorageCheck () { + typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] Verify ODF object storage integration" typeset -i start elapsed start=$(date +%s) - typeset pvcCount - pvcCount=$(oc get pvc -n "${QUAY_NS}" -l app=quay -o json 2>/dev/null | python3 -c " -import sys, json -data = json.load(sys.stdin) -items = data.get('items', []) -print(len(items)) -" 2>/dev/null || echo "0") + typeset noobaaPhase + noobaaPhase=$(oc get noobaa -n openshift-storage -o jsonpath='{.items[0].status.phase}' 2>/dev/null) || noobaaPhase="" + if [[ "${noobaaPhase}" != "Ready" ]]; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "NooBaa not Ready (phase: ${noobaaPhase:-not found})" "${elapsed}" + return 1 + fi - if [[ "${pvcCount}" == "0" ]]; then - pvcCount=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c " -import sys, json -data = json.load(sys.stdin) -items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] -print(len(items)) -" 2>/dev/null || echo "0") + typeset obcCount + obcCount=$(oc get objectbucketclaim -n openshift-storage -o json 2>/dev/null | \ + python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obcCount="0" + if [[ "${obcCount}" == "0" ]]; then + obcCount=$(oc get objectbucketclaim --all-namespaces -o json 2>/dev/null | \ + python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obcCount="0" fi - if [[ "${pvcCount}" == "0" ]]; then + if [[ "${obcCount}" == "0" ]]; then elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "No Quay-related PVCs found in ${QUAY_NS}" "${elapsed}" + RecordResult "${testName}" "failed" "No ObjectBucketClaims found" "${elapsed}" return 1 fi - typeset unboundPvcs - unboundPvcs=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c " -import sys, json -data = json.load(sys.stdin) -items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] -unbound = [i['metadata']['name'] for i in items if i['status'].get('phase') != 'Bound'] -print(' '.join(unbound)) -" 2>/dev/null || echo "") - - if [[ -n "${unboundPvcs}" ]]; then + typeset obCount + obCount=$(oc get objectbucket -o json 2>/dev/null | \ + python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obCount="0" + if [[ "${obCount}" == "0" ]]; then elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "Unbound PVCs: ${unboundPvcs}" "${elapsed}" + RecordResult "${testName}" "failed" "No ObjectBucket resources found for OBCs" "${elapsed}" return 1 fi - typeset odfBacked - odfBacked=$(oc get pvc -n "${QUAY_NS}" -o json | python3 -c " + typeset unboundPvcs + unboundPvcs=$(oc get pvc -n "${QUAY_NS}" -o json 2>/dev/null | python3 -c " import sys, json data = json.load(sys.stdin) items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] -sc_names = set(i['spec'].get('storageClassName','') for i in items) -odf = any('ocs' in s or 'ceph' in s or 'odf' in s for s in sc_names) -print('true' if odf else 'false') -" 2>/dev/null || echo "false") +unbound = [i['metadata']['name'] for i in items if i['status'].get('phase') != 'Bound'] +print(' '.join(unbound)) +" 2>/dev/null) || unboundPvcs="" - if [[ "${odfBacked}" != "true" ]]; then + if [[ -n "${unboundPvcs}" ]]; then elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "Quay PVCs not using ODF/Ceph storage class" "${elapsed}" + RecordResult "${testName}" "failed" "Unbound Quay PVCs: ${unboundPvcs}" "${elapsed}" return 1 fi @@ -250,20 +301,105 @@ print('true' if odf else 'false') ################################################################################ # Test Case 3: ACS scan of pushed Quay image ################################################################################ +function RegisterQuayInAcs () { + typeset acsHost="${1}" acsPassword="${2}" + set +x + + typeset staleIds + staleIds=$(curl -sk -u "admin:${acsPassword}" \ + "https://${acsHost}/v1/imageintegrations" 2>/dev/null | \ + python3 -c " +import sys, json, os +host = os.environ['QUAY_HOST'] +data = json.load(sys.stdin) +ids = [] +for i in data.get('integrations', []): + endpoint = i.get('docker', {}).get('endpoint', '') or i.get('quay', {}).get('endpoint', '') + if host in endpoint or i.get('name', '') == 'interop-quay-smoke': + ids.append(i['id']) +print(' '.join(ids)) +" 2>/dev/null) || staleIds="" + + typeset integrationId + for integrationId in ${staleIds}; do + curl -sk -X DELETE "https://${acsHost}/v1/imageintegrations/${integrationId}" \ + -u "admin:${acsPassword}" 2>/dev/null || true + done + + typeset regUser regPass + if [[ -n "${QUAY_TOKEN}" ]]; then + regUser="\$oauthtoken" + regPass="${QUAY_TOKEN}" + else + regUser="${QUAY_USER}" + regPass="${QUAY_PASSWORD}" + fi + + typeset regPayload='' + regPayload=$(python3 -c " +import json, sys, os +payload = { + 'name': 'interop-quay-smoke', + 'type': 'docker', + 'categories': ['REGISTRY'], + 'docker': { + 'endpoint': os.environ['QUAY_HOST'], + 'username': sys.argv[1], + 'password': sys.argv[2], + 'insecure': True + }, + 'skipTestIntegration': True +} +print(json.dumps(payload)) +" "${regUser}" "${regPass}") + + curl -sk -X POST "https://${acsHost}/v1/imageintegrations" \ + -u "admin:${acsPassword}" \ + -H "Content-Type: application/json" \ + -d "${regPayload}" || true + set -x + + echo "INFO: Registered Quay registry endpoint as ACS image integration" + true +} + +function RequestAcsScan () { + typeset acsHost="${1}" acsPassword="${2}" imageName="${3}" + set +x + + typeset scanPayload='' + scanPayload=$(python3 -c " +import json, sys +payload = {'imageName': sys.argv[1], 'force': True} +print(json.dumps(payload)) +" "${imageName}") + + curl -sk -X POST "https://${acsHost}/v1/images/scan" \ + -u "admin:${acsPassword}" \ + -H "Content-Type: application/json" \ + -d "${scanPayload}" || true + set -x + + echo "INFO: Requested ACS scan of ${imageName}" + true +} + function RunAcsScan () { typeset testName="[sig-interop][Jira:INTEROP][Feature:Quay] ACS scan of pushed Quay image" typeset -i start elapsed start=$(date +%s) typeset acsHost acsPassword - acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null || echo "") + acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null) || acsHost="" if [[ -z "${acsHost}" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS Central route not found" "${elapsed}" return 1 fi - acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d || echo "") + set +x + acsPassword=$(oc get secret -n stackrox central-htpasswd -o jsonpath='{.data.password}' 2>/dev/null | base64 -d 2>/dev/null) || acsPassword="" + set -x if [[ -z "${acsPassword}" ]]; then elapsed=$(( $(date +%s) - start )) RecordResult "${testName}" "failed" "ACS admin password not found" "${elapsed}" @@ -271,12 +407,20 @@ function RunAcsScan () { fi typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" - typeset -i attempts=0 maxAttempts=20 + + set +x + RegisterQuayInAcs "${acsHost}" "${acsPassword}" + RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}" + set -x + + typeset -i attempts=0 maxAttempts=40 while (( attempts < maxAttempts )); do typeset scanResult + set +x scanResult=$(curl -sk -u "admin:${acsPassword}" \ - "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null || echo "") + "https://${acsHost}/v1/images?query=Image:${pushTarget}" 2>/dev/null) || scanResult="" + set -x if echo "${scanResult}" | python3 -c " import sys, json @@ -289,12 +433,18 @@ sys.exit(0 if len(images) > 0 else 1) return 0 fi + if (( attempts % 4 == 3 )); then + set +x + RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}" + set -x + fi + attempts=$((attempts + 1)) sleep 15 done elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "ACS did not detect pushed image within 5 minutes" "${elapsed}" + RecordResult "${testName}" "failed" "ACS did not detect pushed image within 10 minutes" "${elapsed}" return 1 } @@ -308,10 +458,14 @@ function Main () { PreflightCheck || { echo "FATAL: Quay not reachable; skipping all tests" >&2; exit 1; } CreateTestOrg - typeset -i status=0 - RunPushPull || status=1 - RunOdfPvcCheck || status=1 - RunAcsScan || status=1 + typeset -i status=0 pushPassed=0 + RunPushPull && pushPassed=1 || status=1 + RunOdfStorageCheck || status=1 + if (( pushPassed )); then + RunAcsScan || status=1 + else + RecordResult "[sig-interop][Jira:INTEROP][Feature:Quay] ACS scan of pushed Quay image" "skipped" "Skipped: push-pull test failed; no image available to scan" + fi rm -f /tmp/quay-auth.json @@ -322,7 +476,7 @@ function Main () { "${_fURL[@]}" \ https://raw.githubusercontent.com/RedHatQE/OpenShift-LP-QE--Tools/refs/heads/main/libs/bash/ci-operator/interop/common/ExitTrap--PostProcessPrep.sh )" || true - if type -t ExitTrap--PostProcessPrep 1>/dev/null; then + if type -t ExitTrap--PostProcessPrep; then LP_IO__ET_PPP__NEW_TS_NAME="${DR__RP__CR_COMP_NAME}--%s" \ ExitTrap--PostProcessPrep || true fi diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml index eb2e58149f440..5b0caae06170a 100644 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-ref.yaml @@ -1,6 +1,6 @@ ref: as: interop-tests-opp-quay-smoke - from: cli + from: cli-with-skopeo cli: latest commands: interop-tests-opp-quay-smoke-commands.sh timeout: 30m0s @@ -11,8 +11,9 @@ ref: memory: 256Mi documentation: |- Validates Quay as a cross-product registry within the OPP bundle. - Tests image push/pull via the Quay route, verifies ODF-backed PVC storage, - and confirms ACS detects and scans the pushed image. + Tests image push/pull via the Quay route, verifies ODF object storage + integration (OBCs and NooBaa health), and confirms ACS detects and scans + the pushed image. env: - name: DR__RP__CR_COMP_NAME default: "lp-interop--Quay" From 2b87146105c9ff5a5291ac4ab8b4048eccc8cc08 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Fri, 21 Aug 2026 13:51:03 -0500 Subject: [PATCH 2/5] fix: add best_effort to stackrox-opp-smoke --- .../step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml index 74d11f0815aa9..329a34c9d7a71 100644 --- a/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml +++ b/ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-ref.yaml @@ -11,6 +11,7 @@ ref: memory: 4Gi from: acs-smoke-runner timeout: 1h0m0s + best_effort: true documentation: |- Run the ACS qa-tests-backend SMOKE suite against a live ACS instance. Reads connection credentials from SHARED_DIR From c2b5d399741fb4d51693e08d85591b440c5b3ead Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Sun, 23 Aug 2026 17:55:07 -0500 Subject: [PATCH 3/5] sync: quay-smoke commands from batch --- .../interop-tests-opp-quay-smoke-commands.sh | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index 2e13025e8eb65..eaf164b557e59 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -305,26 +305,24 @@ function RegisterQuayInAcs () { typeset acsHost="${1}" acsPassword="${2}" set +x - typeset staleIds - staleIds=$(curl -sk -u "admin:${acsPassword}" \ + typeset existing + existing=$(curl -sk -u "admin:${acsPassword}" \ "https://${acsHost}/v1/imageintegrations" 2>/dev/null | \ python3 -c " import sys, json, os host = os.environ['QUAY_HOST'] data = json.load(sys.stdin) -ids = [] for i in data.get('integrations', []): - endpoint = i.get('docker', {}).get('endpoint', '') or i.get('quay', {}).get('endpoint', '') - if host in endpoint or i.get('name', '') == 'interop-quay-smoke': - ids.append(i['id']) -print(' '.join(ids)) -" 2>/dev/null) || staleIds="" - - typeset integrationId - for integrationId in ${staleIds}; do - curl -sk -X DELETE "https://${acsHost}/v1/imageintegrations/${integrationId}" \ - -u "admin:${acsPassword}" 2>/dev/null || true - done + if host in i.get('docker', {}).get('endpoint', ''): + print(i['id']) + sys.exit(0) +sys.exit(1) +" 2>/dev/null) || existing="" + + if [[ -n "${existing}" ]]; then + echo "INFO: Quay integration already registered in ACS" + return 0 + fi typeset regUser regPass if [[ -n "${QUAY_TOKEN}" ]]; then From d0970a45c91a1c99c28edd4eef0ea14e78335cb6 Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Sun, 23 Aug 2026 18:09:17 -0500 Subject: [PATCH 4/5] fix: address review findings in quay-smoke - Remove global xtrace to prevent credential exposure - Record failure (not skip) when GetQuayAuth fails - Use base64 -w0 to prevent line-wrapped auth strings - Distinguish PVC check failure from no-unbound result --- .../interop-tests-opp-quay-smoke-commands.sh | 26 ++++++++++++++----- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index eaf164b557e59..8bec443767021 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eux -o pipefail +set -eu -o pipefail shopt -s inherit_errexit ARTIFACT_DIR="${ARTIFACT_DIR:=/tmp/artifacts}" @@ -207,9 +207,9 @@ function RunPushPull () { typeset registryAuth set +x if [[ -n "${QUAY_TOKEN}" ]]; then - registryAuth=$(echo -n "\$oauthtoken:${QUAY_TOKEN}" | base64) + registryAuth=$(echo -n "\$oauthtoken:${QUAY_TOKEN}" | base64 -w0) else - registryAuth=$(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64) + registryAuth=$(echo -n "${QUAY_USER}:${QUAY_PASSWORD}" | base64 -w0) fi cat > "${authFile}" <&1); then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "Failed to list PVCs: ${pvcJson}" "${elapsed}" + return 1 + fi + typeset unboundPvcs - unboundPvcs=$(oc get pvc -n "${QUAY_NS}" -o json 2>/dev/null | python3 -c " + unboundPvcs=$(printf '%s' "${pvcJson}" | python3 -c " import sys, json data = json.load(sys.stdin) items = [i for i in data.get('items', []) if 'quay' in i['metadata'].get('name','').lower()] -unbound = [i['metadata']['name'] for i in items if i['status'].get('phase') != 'Bound'] +unbound = [i['metadata']['name'] for i in items if i.get('status', {}).get('phase') != 'Bound'] print(' '.join(unbound)) -" 2>/dev/null) || unboundPvcs="" +") || { elapsed=$(( $(date +%s) - start )); RecordResult "${testName}" "failed" "PVC filter script error"; return 1; } if [[ -n "${unboundPvcs}" ]]; then elapsed=$(( $(date +%s) - start )) @@ -452,7 +459,12 @@ sys.exit(0 if len(images) > 0 else 1) function Main () { DiscoverQuay - GetQuayAuth + if ! GetQuayAuth; then + for t in "${allTests[@]}"; do + RecordResult "${t}" "failed" "Quay credential retrieval failed" + done + exit 1 + fi PreflightCheck || { echo "FATAL: Quay not reachable; skipping all tests" >&2; exit 1; } CreateTestOrg From 57ea8d130bb6d670939f06e1005d85085e86c5ee Mon Sep 17 00:00:00 2001 From: Michael Pruitt Date: Sun, 23 Aug 2026 18:18:18 -0500 Subject: [PATCH 5/5] fix: address remaining review findings in quay-smoke --- .../interop-tests-opp-quay-smoke-commands.sh | 153 +++++++++++++----- 1 file changed, 112 insertions(+), 41 deletions(-) diff --git a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh index 8bec443767021..e74fb3d53e544 100755 --- a/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh +++ b/ci-operator/step-registry/interop-tests/opp-quay-smoke/interop-tests-opp-quay-smoke-commands.sh @@ -81,7 +81,20 @@ EOF trap '{ ( GenerateJunit; true ); }' EXIT function DiscoverQuay () { - QUAY_NS=$(oc get quayregistry --all-namespaces -o jsonpath='{.items[0].metadata.namespace}') + typeset registryJson="" + registryJson="$(oc get quayregistry --all-namespaces -o json 2>/dev/null)" || true + typeset itemCount="" + itemCount="$(printf '%s' "${registryJson}" | python3 -c " +import sys,json +print(len(json.load(sys.stdin).get('items',[]))) +" 2>/dev/null || echo "0")" + + if [[ "${itemCount}" -eq 0 ]]; then + echo "INFO: No QuayRegistry found; Quay is not deployed" + return 1 + fi + + QUAY_NS="$(printf '%s' "${registryJson}" | python3 -c "import sys,json; print(json.load(sys.stdin)['items'][0]['metadata']['namespace'])")" QUAY_REGISTRY=$(oc get quayregistry -n "${QUAY_NS}" -o jsonpath='{.items[0].metadata.name}') QUAY_HOST=$(oc get quayregistry -n "${QUAY_NS}" "${QUAY_REGISTRY}" -o jsonpath='{.status.registryEndpoint}') QUAY_HOST="${QUAY_HOST#https://}" @@ -180,10 +193,12 @@ function CreateTestOrg () { echo "WARNING: No Quay token available; org creation may fail" >&2 fi + set +x # tracing off: bearer token curl -sk -X POST "https://${QUAY_HOST}/api/v1/organization/" \ -H "Authorization: Bearer ${QUAY_TOKEN}" \ -H "Content-Type: application/json" \ -d '{"name":"interop-smoke-test","email":"interop-test@example.com"}' || true + set -x true } @@ -255,26 +270,56 @@ function RunOdfStorageCheck () { return 1 fi - typeset obcCount - obcCount=$(oc get objectbucketclaim -n openshift-storage -o json 2>/dev/null | \ - python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obcCount="0" - if [[ "${obcCount}" == "0" ]]; then - obcCount=$(oc get objectbucketclaim --all-namespaces -o json 2>/dev/null | \ - python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obcCount="0" + typeset quayObc + quayObc=$(oc get objectbucketclaim -n "${QUAY_NS}" -o json 2>/dev/null | \ + python3 -c " +import sys, json, os +registry = os.environ.get('QUAY_REGISTRY', '') +data = json.load(sys.stdin) +for item in data.get('items', []): + name = item['metadata']['name'] + owners = item['metadata'].get('ownerReferences', []) + if any(o.get('kind') == 'QuayRegistry' for o in owners) or registry in name: + print(name) + sys.exit(0) +if data.get('items'): + print(data['items'][0]['metadata']['name']) + sys.exit(0) +sys.exit(1) +" 2>/dev/null) || quayObc="" + + if [[ -z "${quayObc}" ]]; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "No ObjectBucketClaim found in Quay namespace ${QUAY_NS}" "${elapsed}" + return 1 fi - if [[ "${obcCount}" == "0" ]]; then + typeset obcPhase + obcPhase=$(oc get objectbucketclaim "${quayObc}" -n "${QUAY_NS}" \ + -o jsonpath='{.status.phase}' 2>/dev/null) || obcPhase="" + if [[ "${obcPhase}" != "Bound" ]]; then elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "No ObjectBucketClaims found" "${elapsed}" + RecordResult "${testName}" "failed" "Quay OBC ${quayObc} not Bound (phase: ${obcPhase:-unknown})" "${elapsed}" return 1 fi - typeset obCount - obCount=$(oc get objectbucket -o json 2>/dev/null | \ - python3 -c "import sys,json; print(len(json.load(sys.stdin).get('items',[])))" 2>/dev/null) || obCount="0" - if [[ "${obCount}" == "0" ]]; then + typeset obName + obName=$(oc get objectbucket -o json 2>/dev/null | \ + python3 -c " +import sys, json +obc_name, obc_ns = sys.argv[1], sys.argv[2] +data = json.load(sys.stdin) +for item in data.get('items', []): + ref = item.get('spec', {}).get('claimRef', {}) + if ref.get('name') == obc_name and ref.get('namespace') == obc_ns: + print(item['metadata']['name']) + sys.exit(0) +sys.exit(1) +" "${quayObc}" "${QUAY_NS}" 2>/dev/null) || obName="" + + if [[ -z "${obName}" ]]; then elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "No ObjectBucket resources found for OBCs" "${elapsed}" + RecordResult "${testName}" "failed" "No ObjectBucket found for Quay OBC ${quayObc}" "${elapsed}" return 1 fi @@ -310,12 +355,20 @@ print(' '.join(unbound)) ################################################################################ function RegisterQuayInAcs () { typeset acsHost="${1}" acsPassword="${2}" - set +x + typeset _xtrace=false + [[ $- == *x* ]] && _xtrace=true + set +x # tracing off: credentials + + typeset response + if ! response=$(curl -sk -u "admin:${acsPassword}" \ + "https://${acsHost}/v1/imageintegrations" 2>/dev/null); then + echo "ERROR: Failed to query ACS image integrations" >&2 + $_xtrace && set -x + return 1 + fi typeset existing - existing=$(curl -sk -u "admin:${acsPassword}" \ - "https://${acsHost}/v1/imageintegrations" 2>/dev/null | \ - python3 -c " + existing=$(echo "${response}" | python3 -c " import sys, json, os host = os.environ['QUAY_HOST'] data = json.load(sys.stdin) @@ -328,6 +381,7 @@ sys.exit(1) if [[ -n "${existing}" ]]; then echo "INFO: Quay integration already registered in ACS" + $_xtrace && set -x return 0 fi @@ -358,19 +412,24 @@ payload = { print(json.dumps(payload)) " "${regUser}" "${regPass}") - curl -sk -X POST "https://${acsHost}/v1/imageintegrations" \ + if ! curl -sk -X POST "https://${acsHost}/v1/imageintegrations" \ -u "admin:${acsPassword}" \ -H "Content-Type: application/json" \ - -d "${regPayload}" || true - set -x + -d "${regPayload}" >/dev/null 2>&1; then + echo "ERROR: Failed to register Quay in ACS" >&2 + $_xtrace && set -x + return 1 + fi - echo "INFO: Registered Quay registry endpoint as ACS image integration" - true + echo "INFO: Registered Quay as ACS image integration" + $_xtrace && set -x } function RequestAcsScan () { typeset acsHost="${1}" acsPassword="${2}" imageName="${3}" - set +x + typeset _xtrace=false + [[ $- == *x* ]] && _xtrace=true + set +x # tracing off: credentials typeset scanPayload='' scanPayload=$(python3 -c " @@ -379,14 +438,17 @@ payload = {'imageName': sys.argv[1], 'force': True} print(json.dumps(payload)) " "${imageName}") - curl -sk -X POST "https://${acsHost}/v1/images/scan" \ + if ! curl -sk -X POST "https://${acsHost}/v1/images/scan" \ -u "admin:${acsPassword}" \ -H "Content-Type: application/json" \ - -d "${scanPayload}" || true - set -x + -d "${scanPayload}" >/dev/null 2>&1; then + echo "ERROR: ACS scan request failed" >&2 + $_xtrace && set -x + return 1 + fi - echo "INFO: Requested ACS scan of ${imageName}" - true + echo "INFO: Requested ACS scan for pushed image" + $_xtrace && set -x } function RunAcsScan () { @@ -398,8 +460,8 @@ function RunAcsScan () { acsHost=$(oc get route -n stackrox central -o jsonpath='{.spec.host}' 2>/dev/null) || acsHost="" if [[ -z "${acsHost}" ]]; then elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "ACS Central route not found" "${elapsed}" - return 1 + RecordResult "${testName}" "skipped" "ACS not deployed (Central route not found)" "${elapsed}" + return 0 fi set +x @@ -407,16 +469,22 @@ function RunAcsScan () { set -x if [[ -z "${acsPassword}" ]]; then elapsed=$(( $(date +%s) - start )) - RecordResult "${testName}" "failed" "ACS admin password not found" "${elapsed}" - return 1 + RecordResult "${testName}" "skipped" "ACS admin password not available" "${elapsed}" + return 0 fi typeset pushTarget="${QUAY_HOST}/interop-smoke-test/ubi-smoke:${imageTag}" - set +x - RegisterQuayInAcs "${acsHost}" "${acsPassword}" - RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}" - set -x + if ! RegisterQuayInAcs "${acsHost}" "${acsPassword}"; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "Failed to register Quay in ACS" "${elapsed}" + return 1 + fi + if ! RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}"; then + elapsed=$(( $(date +%s) - start )) + RecordResult "${testName}" "failed" "ACS scan request failed" "${elapsed}" + return 1 + fi typeset -i attempts=0 maxAttempts=40 @@ -439,9 +507,7 @@ sys.exit(0 if len(images) > 0 else 1) fi if (( attempts % 4 == 3 )); then - set +x - RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}" - set -x + RequestAcsScan "${acsHost}" "${acsPassword}" "${pushTarget}" || true fi attempts=$((attempts + 1)) @@ -458,7 +524,12 @@ sys.exit(0 if len(images) > 0 else 1) ################################################################################ function Main () { - DiscoverQuay + if ! DiscoverQuay; then + for t in "${allTests[@]}"; do + RecordResult "${t}" "skipped" "Quay not deployed" + done + exit 0 + fi if ! GetQuayAuth; then for t in "${allTests[@]}"; do RecordResult "${t}" "failed" "Quay credential retrieval failed"