INTEROP-9236: Add ACS SMOKE test pipeline for OPP interop

ci-operator/config/stolostron/policy-collection/stolostron-policy-collection-main__ocp4.22.yaml

@@ -48,6 +48,17 @@ images:
     from: cli
     optional: true
     to: cli-with-git
+  - dockerfile_literal: |
+      FROM registry.access.redhat.com/ubi9/openjdk-17:1.21
+      USER root
+      RUN microdnf install -y git && microdnf clean all
+      RUN cd /tmp \
+          && curl -sLO https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz \
+          && curl -sL https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/sha256sum.txt | grep openshift-client-linux.tar.gz | sha256sum -c - \
+          && tar xzf openshift-client-linux.tar.gz -C /usr/local/bin oc kubectl \
+          && rm -f openshift-client-linux.tar.gz
+      USER 1001
+    to: acs-smoke-runner
 releases:
   latest:
     candidate:
@@ -122,6 +133,8 @@ tests:
     - ref: acm-policies-openshift-plus-setup
     - ref: acm-policies-openshift-plus
     - chain: cucushift-installer-check-cluster-health
+    - ref: stackrox-opp-readiness
+    - ref: stackrox-opp-smoke
     - ref: acm-tests-clc-create
     - ref: acm-fetch-managed-clusters
     - ref: acm-opp-app

ci-operator/step-registry/stackrox/opp-readiness/OWNERS

@@ -0,0 +1,4 @@
+approvers:
+- cspi-qe-ocp-lp
+reviewers:
+- cspi-qe-ocp-lp

ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-commands.sh

@@ -0,0 +1,209 @@
+#!/bin/bash
+set -eux -o pipefail
+shopt -s inherit_errexit
+
+# ---------------------------------------------------------------------------
+# ACS OPP Readiness Gate
+#
+# Verifies that ACS Central and SecuredCluster are operational before
+# running SMOKE tests.  Discovers namespaces dynamically via CRs.
+# Writes credentials and connection details to $SHARED_DIR for
+# downstream steps.
+#
+# Dependencies: oc, curl, python3 (all present in the `cli` image).
+# ---------------------------------------------------------------------------
+
+if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then
+    export KUBECONFIG="${SHARED_DIR}/kubeconfig"
+fi
+
+typeset -i pollInterval=30
+typeset -i timeout=600
+typeset -i elapsed=0
+
+function WaitFor () {
+    typeset description="$1"
+    shift
+    typeset checkFn="$1"
+    shift
+
+    elapsed=0
+    echo "[readiness] Waiting for: ${description}"
+    while true; do
+        if "${checkFn}" "$@"; then
+            echo "[readiness] OK: ${description}"
+            return 0
+        fi
+        elapsed=$((elapsed + pollInterval))
+        if [[ ${elapsed} -ge ${timeout} ]]; then
+            echo "[readiness] TIMEOUT after ${timeout}s waiting for: ${description}"
+            return 1
+        fi
+        echo "[readiness]   ...retrying in ${pollInterval}s (${elapsed}/${timeout}s)"
+        sleep "${pollInterval}"
+    done
+    true
+}
+
+function JsonLength () {
+    python3 -c "import json,sys; d=json.load(sys.stdin); print(len(d.get('$1',[])))"
+}
+
+# ---------------------------------------------------------------------------
+# Namespace discovery via CRs (never hardcode)
+# ---------------------------------------------------------------------------
+function DiscoverCentralNs () {
+    centralNs="$(oc get centrals.platform.stackrox.io --all-namespaces \
+        -o jsonpath='{.items[0].metadata.namespace}' 2>/dev/null)" \
+        && [[ -n "${centralNs}" ]]
+}
+
+function DiscoverScNs () {
+    scNs="$(oc get securedclusters.platform.stackrox.io --all-namespaces \
+        -o jsonpath='{.items[0].metadata.namespace}' 2>/dev/null)" \
+        && [[ -n "${scNs}" ]]
+}
+
+typeset centralNs=""
+typeset scNs=""
+
+WaitFor "Central CR namespace discovery" DiscoverCentralNs
+echo "[readiness] Central namespace: ${centralNs}"
+
+WaitFor "SecuredCluster CR namespace discovery" DiscoverScNs
+echo "[readiness] SecuredCluster namespace: ${scNs}"
+
+# ---------------------------------------------------------------------------
+# Check 1: Central route exists
+# ---------------------------------------------------------------------------
+typeset centralUrl=""
+
+function CheckCentralRoute () {
+    oc get route central -n "${centralNs}" -o jsonpath='{.spec.host}' 2>/dev/null
+}
+
+WaitFor "Central route" CheckCentralRoute
+
+set +x
+centralUrl="$(oc get route central -n "${centralNs}" -o jsonpath='{.spec.host}')"
+set -x
+echo "[readiness] Central route discovered"
+
+# ---------------------------------------------------------------------------
+# Extract ROX_ADMIN_PASSWORD before API checks
+# ---------------------------------------------------------------------------
+typeset roxAdminPassword=""
+echo "[readiness] Extracting roxAdminPassword..."
+set +x
+roxAdminPassword="$(oc get secret -n "${centralNs}" central-htpasswd \
+    -o jsonpath='{.data.password}' | base64 -d)"
+set -x
+
+if [[ -z "${roxAdminPassword}" ]]; then
+    echo "[readiness] FATAL: could not extract roxAdminPassword"
+    exit 1
+fi
+echo "[readiness] roxAdminPassword extracted successfully"
+
+# ---------------------------------------------------------------------------
+# Check 2: Central API health (authenticated v1/metadata)
+# ---------------------------------------------------------------------------
+function CheckCentralApi () {
+    set +x
+    typeset httpCode=""
+    httpCode="$(curl -sk -o /dev/null -w '%{http_code}' \
+        -u "admin:${roxAdminPassword}" \
+        "https://${centralUrl}/v1/metadata" --max-time 10)" || { set -x; return 1; }
+    set -x
+    [[ "${httpCode}" == "200" ]]
+}
+
+WaitFor "Central API health (v1/metadata)" CheckCentralApi
+
+# ---------------------------------------------------------------------------
+# Check 3: At least 1 secured cluster connected
+# ---------------------------------------------------------------------------
+function CheckClustersConnected () {
+    set +x
+    typeset clusterCount=""
+    clusterCount="$(curl -sk -u "admin:${roxAdminPassword}" \
+        "https://${centralUrl}/v1/clusters" --max-time 10 \
+        | JsonLength clusters)" || { set -x; return 1; }
+    set -x
+    [[ "${clusterCount}" -ge 1 ]]
+}
+
+WaitFor "secured cluster connected (v1/clusters)" CheckClustersConnected
+
+# ---------------------------------------------------------------------------
+# Check 4: Sensor pods Running (detect OOMKilled)
+# ---------------------------------------------------------------------------
+function CheckSensorPods () {
+    typeset podCount=""
+    podCount="$(oc get pods -n "${scNs}" -l app=sensor \
+        -o json 2>/dev/null | JsonLength items)" || return 1
+    if [[ "${podCount}" -eq 0 ]]; then
+        echo "[readiness]   no sensor pods found yet"
+        return 1
+    fi
+
+    typeset sensorJson=""
+    sensorJson="$(oc get pods -n "${scNs}" -l app=sensor -o json 2>/dev/null)" || return 1
+    typeset oomContainers=""
+    if [[ -n "${sensorJson}" ]]; then
+        oomContainers="$(echo "${sensorJson}" | python3 -c "
+import json,sys
+d=json.load(sys.stdin)
+for pod in d.get('items',[]):
+    for cs in pod.get('status',{}).get('containerStatuses',[]):
+        ls=cs.get('lastState',{}).get('terminated',{})
+        if ls.get('reason')=='OOMKilled':
+            print(cs['name'])
+")"
+    fi
+    if [[ -n "${oomContainers}" ]]; then
+        echo "[readiness] WARNING: OOMKilled detected in sensor containers: ${oomContainers}"
+    fi
+
+    typeset podConditions=""
+    podConditions="$(oc get pods -n "${scNs}" -l app=sensor \
+        -o jsonpath='{range .items[*]}{.metadata.name}{" "}{range .status.conditions[*]}{.type}={.status}{" "}{end}{"\n"}{end}' 2>/dev/null)" || return 1
+    typeset notReady=""
+    notReady="$(echo "${podConditions}" | while IFS= read -r line; do
+            [[ -z "${line}" ]] && continue
+            if ! echo "${line}" | grep -q 'Ready=True'; then
+                echo "${line%% *}:NotReady"
+            fi
+        done)"
+    [[ -z "${notReady}" ]]
+}
+
+WaitFor "sensor pods Running in ${scNs}" CheckSensorPods
+
+# ---------------------------------------------------------------------------
+# Check 5: Default policies loaded (count > 80)
+# ---------------------------------------------------------------------------
+function CheckPoliciesLoaded () {
+    set +x
+    typeset policyCount=""
+    policyCount="$(curl -sk -u "admin:${roxAdminPassword}" \
+        "https://${centralUrl}/v1/policies?query=" --max-time 10 \
+        | JsonLength policies)" || { set -x; return 1; }
+    set -x
+    echo "[readiness]   policy count: ${policyCount}"
+    [[ "${policyCount}" -gt 80 ]]
+}
+
+WaitFor "default policies loaded (>80)" CheckPoliciesLoaded
+
+echo "[readiness] Writing connection details to SHARED_DIR..."
+
+set +x
+echo "${roxAdminPassword}" > "${SHARED_DIR}/ROX_ADMIN_PASSWORD"
+echo "${centralUrl}"       > "${SHARED_DIR}/CENTRAL_URL"
+set -x
+
+echo "${centralNs}"  > "${SHARED_DIR}/CENTRAL_NS"
+echo "${scNs}"       > "${SHARED_DIR}/SC_NS"
+
+echo "[readiness] All checks passed. ACS is ready for SMOKE tests."

ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-ref.metadata.json

@@ -0,0 +1,11 @@
+{
+	"path": "stackrox/opp-readiness/stackrox-opp-readiness-ref.yaml",
+	"owners": {
+		"approvers": [
+			"cspi-qe-ocp-lp"
+		],
+		"reviewers": [
+			"cspi-qe-ocp-lp"
+		]
+	}
+}

ci-operator/step-registry/stackrox/opp-readiness/stackrox-opp-readiness-ref.yaml

@@ -0,0 +1,16 @@
+ref:
+  as: stackrox-opp-readiness
+  commands: stackrox-opp-readiness-commands.sh
+  resources:
+    requests:
+      cpu: 100m
+      memory: 200Mi
+  from: cli
+  timeout: 1h15m0s
+  documentation: |-
+    Verify ACS Central and SecuredCluster are operational before running
+    SMOKE tests.  Discovers namespaces dynamically via Central and
+    SecuredCluster CRs, then polls Central API health, secured-cluster
+    connectivity, sensor pod status, and default policy count.  Writes
+    ROX_ADMIN_PASSWORD, CENTRAL_URL, CENTRAL_NS, and SC_NS to SHARED_DIR
+    for downstream steps.

ci-operator/step-registry/stackrox/opp-smoke/OWNERS

@@ -0,0 +1,4 @@
+approvers:
+- cspi-qe-ocp-lp
+reviewers:
+- cspi-qe-ocp-lp

ci-operator/step-registry/stackrox/opp-smoke/stackrox-opp-smoke-commands.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+set -eux -o pipefail
+shopt -s inherit_errexit
+
+if [[ -f "${SHARED_DIR}/kubeconfig" ]]; then
+    export KUBECONFIG="${SHARED_DIR}/kubeconfig"
+fi
+
+echo "[smoke] Reading connection details from SHARED_DIR..."
+
+set +x
+CENTRAL_URL="$(cat "${SHARED_DIR}/CENTRAL_URL")"
+ROX_ADMIN_PASSWORD="$(cat "${SHARED_DIR}/ROX_ADMIN_PASSWORD")"
+set -x
+
+echo "[smoke] Connection details loaded from SHARED_DIR"
+
+STACKROX_REF="${STACKROX_REF:-master}"
+SCANNER_REF="${SCANNER_REF:-master}"
+
+echo "[smoke] Sparse-cloning stackrox/stackrox..."
+cd /tmp
+rm -rf stackrox scanner
+git clone --depth 1 --filter=blob:none --sparse --branch "${STACKROX_REF}" \
+    https://github.com/stackrox/stackrox.git stackrox
+cd stackrox
+git sparse-checkout set qa-tests-backend/ proto/
+
+echo "[smoke] Fetching scanner protos..."
+git clone --depth 1 --filter=blob:none --sparse --branch "${SCANNER_REF}" \
+    https://github.com/stackrox/scanner.git /tmp/scanner
+cd /tmp/scanner
+git sparse-checkout set proto/scanner
+cp -r proto/scanner /tmp/stackrox/qa-tests-backend/src/main/proto/scanner
+chmod -R u+w /tmp/stackrox/qa-tests-backend/src/main/proto/scanner
+
+echo "[smoke] Materializing proto sources (replace symlinks with copies)..."
+cd /tmp/stackrox/qa-tests-backend/src/main/proto
+for link in api internalapi storage test tools; do
+    if [[ -L "${link}" ]]; then
+        target="$(readlink -f "${link}")"
+        rm "${link}"
+        cp -r "${target}" "${link}"
+    fi
+done
+
+echo "[smoke] Patching DEFAULT_CLUSTER_NAME to 'local-cluster'..."
+sed -i 's/DEFAULT_CLUSTER_NAME = "remote"/DEFAULT_CLUSTER_NAME = "local-cluster"/' \
+    /tmp/stackrox/qa-tests-backend/src/main/groovy/services/ClusterService.groovy
+grep -q 'DEFAULT_CLUSTER_NAME = "local-cluster"' \
+    /tmp/stackrox/qa-tests-backend/src/main/groovy/services/ClusterService.groovy \
+    || { echo "[smoke] FATAL: DEFAULT_CLUSTER_NAME patch failed"; exit 1; }
+
+set +x
+export API_HOSTNAME="${CENTRAL_URL}"
+export API_PORT="443"
+export ROX_USERNAME="admin"
+export ROX_ADMIN_PASSWORD
+export CLUSTER="OPENSHIFT"
+export CI="true"
+export POD_SECURITY_POLICIES="false"
+export TEST_TARGET="smoke-test"
+REGISTRY_USERNAME="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/QUAY_RHACS_ENG_RO_USERNAME)"
+export REGISTRY_USERNAME
+REGISTRY_PASSWORD="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/QUAY_RHACS_ENG_RO_PASSWORD)"
+export REGISTRY_PASSWORD
+if [[ -f /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_CREDENTIALS_GCR_SCANNER_V2 ]]; then
+    GOOGLE_CREDENTIALS_GCR_SCANNER_V2="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_CREDENTIALS_GCR_SCANNER_V2)"
+    export GOOGLE_CREDENTIALS_GCR_SCANNER_V2
+fi
+if [[ -f /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2 ]]; then
+    GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2="$(cat /tmp/vault/stackrox-stackrox-e2e-tests/GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2)"
+    export GOOGLE_ARTIFACT_REGISTRY_SERVICE_ACCOUNT_V2
+fi
+set -x
+
+cd /tmp/stackrox/qa-tests-backend
+
+cat > /tmp/fix-proto-deps.gradle <<'INIT'
+allprojects {
+    afterEvaluate {
+        tasks.matching { it.name == 'compileGroovy' }.configureEach {
+            dependsOn tasks.matching { it.name == 'generateProto' }
+        }
+    }
+}
+INIT
+
+echo "[smoke] Running testSMOKE..."
+typeset -i testExit=0
+./gradlew testSMOKE --no-daemon --init-script /tmp/fix-proto-deps.gradle \
+    -Dorg.gradle.jvmargs="-Xmx2g" || testExit=$?
+
+echo "[smoke] Copying JUnit results to ARTIFACT_DIR..."
+if [[ -d build/test-results/testSMOKE ]]; then
+    find build/test-results/testSMOKE -name '*.xml' -exec cp -v {} "${ARTIFACT_DIR}/" \;
+fi
+
+if [[ -d build/reports/tests/testSMOKE ]]; then
+    mkdir -p "${ARTIFACT_DIR}/smoke-report"
+    find build/reports/tests/testSMOKE -mindepth 1 -maxdepth 1 \
+        -exec cp -r {} "${ARTIFACT_DIR}/smoke-report/" \;
+fi
+
+echo "[smoke] Test run finished with exit code: ${testExit}"
+if [[ "${testExit}" -ne 0 ]] && [[ -d build/test-results/testSMOKE ]]; then
+    typeset total=""
+    total="$(find build/test-results/testSMOKE -name '*.xml' -exec grep -l 'testcase' {} \; | wc -l)"
+    if [[ "${total}" -gt 0 ]]; then
+        echo "[smoke] Tests executed and results captured; treating as informational (exit 0)."
+        echo "[smoke] Review JUnit XML in ARTIFACT_DIR for individual test failures."
+        exit 0
+    fi
+fi
+exit "${testExit}"