test: Add Docker integration tests

.github/workflows/pre-release.yaml

@@ -49,4 +49,27 @@ jobs:
 
       - uses: actions/checkout@v4
 
-      - run: .bin/pre-release-docker
+      - uses: actions/setup-java@v4
+        with:
+          java-version: '21'
+          distribution: 'graalvm'
+          cache: maven
+
+      - name: Build Docker images
+        run: |
+          export VERSION=0.0.0-latest-master+$(git rev-parse --short HEAD)
+          export IMAGE_TAG="latest-master"
+          export SKIP_VERIFY=1
+          export PATH="$(pwd)/.bin:$PATH"
+          docker-build-ice
+          docker-build-ice-rest-catalog
+
+      - name: Run Docker integration tests
+        run: >
+          ./mvnw -pl ice-rest-catalog install -Dmaven.test.skip=true -Pno-check &&
+          ./mvnw -pl ice-rest-catalog failsafe:integration-test failsafe:verify
+          -Dit.test=DockerScenarioBasedIT
+          -Ddocker.image=altinity/ice-rest-catalog:debug-with-ice-latest-master-amd64
+
+      - name: Push Docker images
+        run: .bin/pre-release-docker

.github/workflows/verify.yaml

@@ -15,9 +15,45 @@ jobs:
           distribution: 'graalvm'
           cache: maven
       - run: ./mvnw clean verify
-      - name: Install 
+      - name: Install
         run: ./mvnw install
       # TODO: check native-image can build ice
       - name: Run Scenario-Based Integration Tests
         run: ../mvnw test -Dtest=ScenarioBasedIT
-        working-directory: ice-rest-catalog
+        working-directory: ice-rest-catalog
+  docker-integration:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      # https://github.com/docker/setup-buildx-action
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Install regctl
+        uses: regclient/actions/regctl-installer@ce5fd131e371ffcdd7508b478cb223b3511a9183
+      - name: regctl login
+        uses: regclient/actions/regctl-login@ce5fd131e371ffcdd7508b478cb223b3511a9183
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
+        with:
+          java-version: '21'
+          distribution: 'graalvm'
+          cache: maven
+      - name: Build Docker images
+        run: |
+          export VERSION=0.0.0-latest-master+$(git rev-parse --short HEAD)
+          export IMAGE_TAG="latest-master"
+          export SKIP_VERIFY=1
+          export PATH="$(pwd)/.bin:$PATH"
+          docker-build-ice
+          docker-build-ice-rest-catalog
+      - name: Run Docker integration tests
+        run: >
+          ./mvnw -pl ice-rest-catalog install -DskipTests=true -Pno-check &&
+          ./mvnw -pl ice-rest-catalog failsafe:integration-test failsafe:verify
+          -Dit.test=DockerScenarioBasedIT
+          -Ddocker.image=altinity/ice-rest-catalog:debug-with-ice-latest-master-amd64

ice-rest-catalog/pom.xml

@@ -393,7 +393,6 @@
       <groupId>com.fasterxml.jackson.dataformat</groupId>
       <artifactId>jackson-dataformat-yaml</artifactId>
       <version>${jackson.version}</version>
-      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>io.etcd</groupId>

ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/DockerScenarioBasedIT.java

@@ -0,0 +1,176 @@
+/*
+ * Copyright (c) 2025 Altinity Inc and/or its affiliates. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ */
+package com.altinity.ice.rest.catalog;
+
+import java.io.File;
+import java.net.URI;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.Network;
+import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.MountableFile;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+
+/**
+ * Docker-based integration tests for ICE REST Catalog.
+ *
+ * <p>Runs the ice-rest-catalog Docker image (specified via system property {@code docker.image})
+ * alongside a MinIO container, then executes scenario-based tests against it.
+ */
+public class DockerScenarioBasedIT extends RESTCatalogTestBase {
+
+  private Network network;
+
+  private GenericContainer<?> minio;
+
+  private GenericContainer<?> catalog;
+
+  @Override
+  @BeforeClass
+  @SuppressWarnings("resource")
+  public void setUp() throws Exception {
+    String dockerImage =
+        System.getProperty("docker.image", "altinity/ice-rest-catalog:debug-with-ice-0.12.0");
+    logger.info("Using Docker image: {}", dockerImage);
+
+    network = Network.newNetwork();
+
+    // Start MinIO
+    minio =
+        new GenericContainer<>("minio/minio:latest")
+            .withNetwork(network)
+            .withNetworkAliases("minio")
+            .withExposedPorts(9000)
+            .withEnv("MINIO_ACCESS_KEY", "minioadmin")
+            .withEnv("MINIO_SECRET_KEY", "minioadmin")
+            .withCommand("server", "/data")
+            .waitingFor(Wait.forHttp("/minio/health/live").forPort(9000));
+    minio.start();
+
+    // Create test bucket via MinIO's host-mapped port
+    String minioHostEndpoint = "http://" + minio.getHost() + ":" + minio.getMappedPort(9000);
+    try (var s3Client =
+        S3Client.builder()
+            .endpointOverride(URI.create(minioHostEndpoint))
+            .region(Region.US_EAST_1)
+            .credentialsProvider(
+                StaticCredentialsProvider.create(
+                    AwsBasicCredentials.create("minioadmin", "minioadmin")))
+            .forcePathStyle(true)
+            .build()) {
+      s3Client.createBucket(CreateBucketRequest.builder().bucket("test-bucket").build());
+      logger.info("Created test-bucket in MinIO");
+    }
+
+    // Load YAML config for the catalog container (MinIO via Docker network alias "minio")
+    URL configResource = getClass().getClassLoader().getResource("docker-catalog-config.yaml");
+    if (configResource == null) {
+      throw new IllegalStateException("docker-catalog-config.yaml not found on classpath");
+    }
+    String catalogConfig = Files.readString(Paths.get(configResource.toURI()));
+
+    Path scenariosDir = getScenariosDirectory().toAbsolutePath();
+    if (!Files.exists(scenariosDir) || !Files.isDirectory(scenariosDir)) {
+      throw new IllegalStateException(
+          "Scenarios directory must exist at "
+              + scenariosDir
+              + ". Run 'mvn test-compile' or run the test from Maven (e.g. mvn failsafe:integration-test).");
+    }
+    Path insertScanInput = scenariosDir.resolve("insert-scan").resolve("input.parquet");
+    if (!Files.exists(insertScanInput)) {
+      throw new IllegalStateException(
+          "Scenario input not found at "
+              + insertScanInput
+              + ". Ensure test resources are on the classpath and scenarios/insert-scan/input.parquet exists.");
+    }
+
+    // Start the ice-rest-catalog container (debug-with-ice has ice CLI at /usr/local/bin/ice)
+    catalog =
+        new GenericContainer<>(dockerImage)
+            .withNetwork(network)
+            .withExposedPorts(5000)
+            .withEnv("ICE_REST_CATALOG_CONFIG", "")
+            .withEnv("ICE_REST_CATALOG_CONFIG_YAML", catalogConfig)
+            .withCopyFileToContainer(MountableFile.forHostPath(scenariosDir), "/scenarios")
+            .waitingFor(Wait.forHttp("/v1/config").forPort(5000).forStatusCode(200));
+
+    try {
+      catalog.start();
+    } catch (Exception e) {
+      if (catalog != null) {
+        logger.error("Catalog container logs (stdout): {}", catalog.getLogs());
+      }
+      throw e;
+    }
+
+    // Copy CLI config into container so ice CLI can talk to co-located REST server
+    File cliConfigHost = File.createTempFile("ice-docker-cli-", ".yaml");
+    try {
+      Files.write(cliConfigHost.toPath(), "uri: http://localhost:5000\n".getBytes());
+      catalog.copyFileToContainer(
+          MountableFile.forHostPath(cliConfigHost.toPath()), "/tmp/ice-cli.yaml");
+    } finally {
+      cliConfigHost.delete();
+    }
+
+    logger.info(
+        "Catalog container started at {}:{}", catalog.getHost(), catalog.getMappedPort(5000));
+  }
+
+  @Override
+  @AfterClass
+  public void tearDown() {
+    if (catalog != null) {
+      catalog.close();
+    }
+    if (minio != null) {
+      minio.close();
+    }
+    if (network != null) {
+      network.close();
+    }
+  }
+
+  @Override
+  protected ScenarioTestRunner createScenarioRunner(String scenarioName) throws Exception {
+    Path scenariosDir = getScenariosDirectory();
+
+    String containerId = catalog.getContainerId();
+
+    // Wrapper script on host: docker exec <container> ice "$@" (CLI runs inside container)
+    File wrapperScript = File.createTempFile("ice-docker-exec-", ".sh");
+    wrapperScript.deleteOnExit();
+    String wrapperContent = "#!/bin/sh\n" + "exec docker exec " + containerId + " ice \"$@\"\n";
+    Files.write(wrapperScript.toPath(), wrapperContent.getBytes());
+    if (!wrapperScript.setExecutable(true)) {
+      throw new IllegalStateException("Could not set wrapper script executable: " + wrapperScript);
+    }
+
+    Map<String, String> templateVars = new HashMap<>();
+    templateVars.put("ICE_CLI", wrapperScript.getAbsolutePath());
+    templateVars.put("CLI_CONFIG", "/tmp/ice-cli.yaml");
+    templateVars.put("SCENARIO_DIR", "/scenarios/" + scenarioName);
+    templateVars.put("MINIO_ENDPOINT", "");
+    templateVars.put("CATALOG_URI", "http://localhost:5000");
+
+    return new ScenarioTestRunner(scenariosDir, templateVars);
+  }
+}

ice-rest-catalog/src/test/java/com/altinity/ice/rest/catalog/RESTCatalogTestBase.java

@@ -14,7 +14,12 @@
 import com.altinity.ice.rest.catalog.internal.config.Config;
 import java.io.File;
 import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
 import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
 import java.util.Map;
 import org.apache.iceberg.catalog.Catalog;
 import org.eclipse.jetty.server.Server;
@@ -23,6 +28,8 @@
 import org.testcontainers.containers.GenericContainer;
 import org.testng.annotations.AfterClass;
 import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
 import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
 import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
 import software.amazon.awssdk.regions.Region;
@@ -152,4 +159,94 @@ protected String getMinioEndpoint() {
   protected String getCatalogUri() {
     return "http://localhost:8080";
   }
+
+  /**
+   * Get the path to the scenarios directory.
+   *
+   * @return Path to scenarios directory
+   * @throws URISyntaxException If the resource URL cannot be converted to a path
+   */
+  protected Path getScenariosDirectory() throws URISyntaxException {
+    URL scenariosUrl = getClass().getClassLoader().getResource("scenarios");
+    if (scenariosUrl == null) {
+      return Paths.get("src/test/resources/scenarios");
+    }
+    return Paths.get(scenariosUrl.toURI());
+  }
+
+  /**
+   * Create a ScenarioTestRunner for the given scenario. Subclasses provide host or container-based
+   * CLI and config.
+   *
+   * @param scenarioName Name of the scenario (e.g. for container path resolution)
+   * @return Configured ScenarioTestRunner
+   * @throws Exception If there's an error creating the runner
+   */
+  protected abstract ScenarioTestRunner createScenarioRunner(String scenarioName) throws Exception;
+
+  /** Data provider that discovers all test scenarios. */
+  @DataProvider(name = "scenarios")
+  public Object[][] scenarioProvider() throws Exception {
+    Path scenariosDir = getScenariosDirectory();
+    ScenarioTestRunner runner = new ScenarioTestRunner(scenariosDir, Map.of());
+    List<String> scenarios = runner.discoverScenarios();
+
+    if (scenarios.isEmpty()) {
+      logger.warn("No test scenarios found in: {}", scenariosDir);
+      return new Object[0][0];
+    }
+
+    logger.info("Discovered {} test scenario(s): {}", scenarios.size(), scenarios);
+
+    Object[][] data = new Object[scenarios.size()][1];
+    for (int i = 0; i < scenarios.size(); i++) {
+      data[i][0] = scenarios.get(i);
+    }
+    return data;
+  }
+
+  /** Parameterized test that executes a single scenario. */
+  @Test(dataProvider = "scenarios")
+  public void testScenario(String scenarioName) throws Exception {
+    logger.info("====== Starting scenario test: {} ======", scenarioName);
+
+    ScenarioTestRunner runner = createScenarioRunner(scenarioName);
+    ScenarioTestRunner.ScenarioResult result = runner.executeScenario(scenarioName);
+
+    if (result.runScriptResult() != null) {
+      logger.info("Run script exit code: {}", result.runScriptResult().exitCode());
+    }
+    if (result.verifyScriptResult() != null) {
+      logger.info("Verify script exit code: {}", result.verifyScriptResult().exitCode());
+    }
+
+    assertScenarioSuccess(scenarioName, result);
+    logger.info("====== Scenario test passed: {} ======", scenarioName);
+  }
+
+  /** Assert that the scenario result indicates success; otherwise throw AssertionError. */
+  protected void assertScenarioSuccess(
+      String scenarioName, ScenarioTestRunner.ScenarioResult result) {
+    if (result.isSuccess()) {
+      return;
+    }
+    StringBuilder errorMessage = new StringBuilder();
+    errorMessage.append("Scenario '").append(scenarioName).append("' failed:\n");
+
+    if (result.runScriptResult() != null && result.runScriptResult().exitCode() != 0) {
+      errorMessage.append("\nRun script failed with exit code: ");
+      errorMessage.append(result.runScriptResult().exitCode());
+      errorMessage.append("\nStdout:\n").append(result.runScriptResult().stdout());
+      errorMessage.append("\nStderr:\n").append(result.runScriptResult().stderr());
+    }
+
+    if (result.verifyScriptResult() != null && result.verifyScriptResult().exitCode() != 0) {
+      errorMessage.append("\nVerify script failed with exit code: ");
+      errorMessage.append(result.verifyScriptResult().exitCode());
+      errorMessage.append("\nStdout:\n").append(result.verifyScriptResult().stdout());
+      errorMessage.append("\nStderr:\n").append(result.verifyScriptResult().stderr());
+    }
+
+    throw new AssertionError(errorMessage.toString());
+  }
 }