fix(ci): stabilize PXF regression tests

automation/src/main/java/org/apache/cloudberry/pxf/automation/components/common/ShellSystemObject.java

@@ -43,6 +43,8 @@ public class ShellSystemObject extends BaseSystemObject {
     public static final long _5_MINUTES = (_1_MINUTE * 5);
     public static final long _10_MINUTES = _5_MINUTES * 2;
     public static final long _30_MINUTES = _10_MINUTES * 3;
+    public static final long _40_MINUTES = _10_MINUTES * 4;
+    public static final long _60_MINUTES = _30_MINUTES * 2;
 
     public static final int MIN_COMMAND_TIMEOUT = 100;
 

automation/src/main/java/org/apache/cloudberry/pxf/automation/components/gpdb/Gpdb.java

@@ -417,22 +417,40 @@ public void copyFromStdin(Table from, Table to, String delim, boolean csv) throw
 	 * @throws Exception
 	 */
 	public void copyFromFile(Table to, File path, String delim, boolean csv) throws Exception {
+		copyFromFile(to, path, delim, csv, null);
+	}
+
+	/**
+	 * Copy data file to "to" table from file "path" with an optional command timeout.
+	 *
+	 * @param to copy to required table
+	 * @param path file to copy
+	 * @param delim delimiter
+	 * @param csv is csv format - if it is, delimiter is not used.
+	 * @param commandTimeout command timeout in milliseconds, or null for the default
+	 * @throws Exception if the operation fails
+	 */
+	public void copyFromFile(Table to, File path, String delim, boolean csv, Long commandTimeout) throws Exception {
 		String from = "'" + path.getAbsolutePath() + "'";
 		copyLocalFileToRemoteGpdb(from);
-		copyWithOptionalCTAS(from, to, null, delim, null, csv);
+		copyWithOptionalCTAS(from, to, null, delim, null, csv, commandTimeout);
 	}
 
 	private void copyWithOptionalCTAS(String from, Table to, String dataToCopy, String delim, String nullChar, boolean csv) throws Exception {
+		copyWithOptionalCTAS(from, to, dataToCopy, delim, nullChar, csv, null);
+	}
+
+	private void copyWithOptionalCTAS(String from, Table to, String dataToCopy, String delim, String nullChar, boolean csv, Long commandTimeout) throws Exception {
 		// COPY TO <foreign table> is not supported in PXF FDW with GP6, so we will have to do a workaround by
 		// creating a native table, copying data from the file into it and then performing a CTAS into the foreign table
 		if (FDWUtils.useFDW && getVersion() < 7) {
 			Table nativeTable = createTableLike(to.getName() + "_native", to);
 			// copy data into the native table
-			copy(nativeTable.getName(), from, dataToCopy, delim, null, csv);
+			copy(nativeTable.getName(), from, dataToCopy, delim, null, csv, commandTimeout);
 			// CTAS into the foreign table
 			copyData(nativeTable, to, true);
 		} else {
-			copy(to.getName(), from, dataToCopy, delim, null, csv);
+			copy(to.getName(), from, dataToCopy, delim, null, csv, commandTimeout);
 		}
 	}
 
@@ -466,6 +484,10 @@ public void copyFromFile(Table to, File path, String delim, String nullChar, boo
 	}
 
 	private void copy(String to, String from, String dataToCopy, String delim, String nullChar, boolean csv) throws Exception {
+		copy(to, from, dataToCopy, delim, nullChar, csv, null);
+	}
+
+	private void copy(String to, String from, String dataToCopy, String delim, String nullChar, boolean csv, Long commandTimeout) throws Exception {
 
 		ReportUtils.startLevel(report, getClass(), "Copy from " + from + " to " + to);
 
@@ -486,9 +508,8 @@ private void copy(String to, String from, String dataToCopy, String delim, Strin
 		String copyCmd = "\\COPY " + to + " FROM " + from + " " + copyParams + ";";
 
 		ShellSystemObject sso = openPsql();
-		// in case data from file, the COPY command can be long. increasing the timeout time.
-		if (dataToCopy == null) {
-			sso.setCommandTimeout(ShellSystemObject._10_MINUTES);
+		if (commandTimeout != null) {
+			sso.setCommandTimeout(commandTimeout);
 		}
 
 		runSqlCmd(sso, copyCmd, true);

automation/src/test/java/org/apache/cloudberry/pxf/automation/features/writable/HdfsWritableTextTest.java

@@ -4,6 +4,7 @@
 import annotations.SkipForFDW;
 import annotations.WorksWithFDW;
 import org.apache.commons.lang.StringUtils;
+import org.apache.cloudberry.pxf.automation.components.common.ShellSystemObject;
 import org.apache.cloudberry.pxf.automation.datapreparer.writable.WritableDataPreparer;
 import org.apache.cloudberry.pxf.automation.enums.EnumCompressionTypes;
 import org.apache.cloudberry.pxf.automation.features.BaseWritableFeature;
@@ -17,6 +18,7 @@
 import org.apache.cloudberry.pxf.automation.utils.system.ProtocolUtils;
 import org.apache.cloudberry.pxf.automation.utils.tables.ComparisonUtils;
 import org.junit.Assert;
+import org.postgresql.util.PSQLException;
 import org.testng.annotations.Test;
 
 import java.io.File;
@@ -40,6 +42,14 @@ public class HdfsWritableTextTest extends BaseWritableFeature {
     private String hdfsWorkingDataDir;
     private ProtocolEnum protocol;
     private static final Comparator<List<String>> ROW_COMPARATOR = Comparator.comparing(row -> String.join("|", row));
+    private static final int LARGE_MULTI_BLOCK_REPEAT_COUNT = 15_000;
+    // 2,000 repetitions of the 1,000-row fixture are over 128 MiB, preserving
+    // multi-HDFS-block coverage without making an uncompressed COPY take an hour.
+    private static final int UNCOMPRESSED_MULTI_BLOCK_REPEAT_COUNT = 2_000;
+    private static final int FDW_MULTI_BLOCK_REPEAT_COUNT = 1_000;
+    private static final int BZIP2_MULTI_BLOCK_REPEAT_COUNT = 1_000;
+    private static final int OUTPUT_VISIBILITY_MAX_ATTEMPTS = 90;
+    private static final long OUTPUT_VISIBILITY_RETRY_DELAY_MS = 10_000;
 
     private enum InsertionMethod {
         INSERT,
@@ -70,6 +80,16 @@ protected void beforeClass() throws Exception {
         removeBlankRows(dataTable);
     }
 
+    private int multiBlockRepeatCount() {
+        // FDW writes through a native-table INSERT workaround, so use the bounded
+        // one-million-row fixture while external tables retain the stress coverage.
+        return FDWUtils.useFDW ? FDW_MULTI_BLOCK_REPEAT_COUNT : LARGE_MULTI_BLOCK_REPEAT_COUNT;
+    }
+
+    private int uncompressedMultiBlockRepeatCount() {
+        return FDWUtils.useFDW ? FDW_MULTI_BLOCK_REPEAT_COUNT : UNCOMPRESSED_MULTI_BLOCK_REPEAT_COUNT;
+    }
+
     /**
      * Insert data to Writable table using specific plugins in the table
      * location instead of using profile.
@@ -396,7 +416,9 @@ public void textFormatBZip2CopyFromStdinShortname() throws Exception {
      *
      * @throws Exception if test fails to run
      */
-    @Test(groups = {"features", "gpdb", "hcfs", "security"}, timeOut = 120000)
+    // This writes and reads roughly 100 MB of data. Rocky 9 CI runners can take
+    // longer than two minutes while copying the HDFS result back for validation.
+    @Test(groups = {"features", "gpdb", "hcfs", "security"}, timeOut = 600000)
     public void textFormatWideRowsInsert() throws Exception {
 
         int rows = 10;
@@ -466,12 +488,14 @@ public void copyFromFileMultiBlockedDataNoCompression() throws Exception {
         FileFormatsUtils.prepareData(new WritableDataPreparer(), 1000, data);
         // multiple it to file
         String multiBlockedLocalFilePath = dataTempFolder + "/multiBlockedData";
-        FileFormatsUtils.prepareDataFile(data, 15000, multiBlockedLocalFilePath);
+        int repeatCount = uncompressedMultiBlockRepeatCount();
+        FileFormatsUtils.prepareDataFile(data, repeatCount, multiBlockedLocalFilePath);
 
         String hdfsPath = hdfsWritePath + "/copy_from_file_multi_block_no_compression";
         writableExTable = prepareWritableTable("pxf_text_multi_block_no_compression_w", hdfsPath, null);
 
-        gpdb.copyFromFile(writableExTable, new File(multiBlockedLocalFilePath), ",", false);
+        gpdb.copyFromFile(writableExTable, new File(multiBlockedLocalFilePath), ",", false,
+                ShellSystemObject._30_MINUTES);
 
         // for HCFS on Cloud, wait a bit for async write in previous steps to finish
         if (protocol != ProtocolEnum.HDFS && protocol != ProtocolEnum.FILE) {
@@ -480,7 +504,7 @@ public void copyFromFileMultiBlockedDataNoCompression() throws Exception {
 
         readableExTable = prepareReadableTable("pxf_text_multi_block_no_compression_r", hdfsPath);
         gpdb.runAnalyticQuery("SELECT COUNT(*) FROM " + readableExTable.getName(),
-                String.valueOf(1000 * 15000));
+                String.valueOf(1000 * repeatCount));
     }
 
     /**
@@ -495,12 +519,14 @@ public void copyFromFileMultiBlockedDataGZip() throws Exception {
         FileFormatsUtils.prepareData(new WritableDataPreparer(), 1000, data);
         // multiple it to file
         String multiBlockedLocalFilePath = (dataTempFolder + "/multiBlockedData_gzip");
-        FileFormatsUtils.prepareDataFile(data, 15000, multiBlockedLocalFilePath);
+        int repeatCount = multiBlockRepeatCount();
+        FileFormatsUtils.prepareDataFile(data, repeatCount, multiBlockedLocalFilePath);
 
         String hdfsPath = hdfsWritePath + "/copy_from_file_multi_block_gzip";
         writableExTable = prepareWritableGzipTable("pxf_text_multi_block_gzip_w", hdfsPath);
 
-        gpdb.copyFromFile(writableExTable, new File(multiBlockedLocalFilePath), ",", false);
+        gpdb.copyFromFile(writableExTable, new File(multiBlockedLocalFilePath), ",", false,
+                ShellSystemObject._60_MINUTES);
 
         // for HCFS on Cloud, wait a bit for async write in previous steps to finish
         if (protocol != ProtocolEnum.HDFS && protocol != ProtocolEnum.FILE) {
@@ -509,7 +535,7 @@ public void copyFromFileMultiBlockedDataGZip() throws Exception {
 
         readableExTable = prepareReadableTable("pxf_text_multi_block_gzip_r", hdfsPath);
         gpdb.runAnalyticQuery("SELECT COUNT(*) FROM " + readableExTable.getName(),
-                String.valueOf(1000 * 15000));
+                String.valueOf(1000 * repeatCount));
     }
 
     /**
@@ -518,14 +544,18 @@ public void copyFromFileMultiBlockedDataGZip() throws Exception {
      *
      * @throws Exception if test fails to run
      */
+    // PXF FDW on GPDB 6 does not create BZip2 output through its native-table INSERT workaround.
+    @SkipForFDW
     @Test(groups = {"features", "gpdb", "hcfs", "security"})
     public void copyFromFileMultiBlockedDataBZip2() throws Exception {
 
         Table data = new Table("data", null);
         FileFormatsUtils.prepareData(new WritableDataPreparer(), 1000, data);
         // multiple it to file
         String multiBlockedLocalFilePath = dataTempFolder + "/multiBlockedData_bzip";
-        FileFormatsUtils.prepareDataFile(data, 15000, multiBlockedLocalFilePath);
+        // One million rows still exercises many 900 KiB BZip2 blocks without the
+        // unrecoverable partial write produced by the former 15-million-row fixture.
+        FileFormatsUtils.prepareDataFile(data, BZIP2_MULTI_BLOCK_REPEAT_COUNT, multiBlockedLocalFilePath);
 
         String hdfsPath = hdfsWritePath + "/copy_from_file_multi_block_bzip2";
         writableExTable = prepareWritableBZip2Table("pxf_text_multi_block_bzip2_w", hdfsPath);
@@ -538,8 +568,26 @@ public void copyFromFileMultiBlockedDataBZip2() throws Exception {
         }
 
         readableExTable = prepareReadableTable("pxf_text_multi_block_bzip2_r", hdfsPath);
-        gpdb.runAnalyticQuery("SELECT COUNT(*) FROM " + readableExTable.getName(),
-                String.valueOf(1000 * 15000));
+        String countQuery = "SELECT COUNT(*) FROM " + readableExTable.getName();
+        String expectedCount = String.valueOf(1000 * BZIP2_MULTI_BLOCK_REPEAT_COUNT);
+        // The BZip2 output can become visible to a freshly created readable table
+        // after COPY returns on loaded CI runners. Retry the assertion for up to
+        // fifteen minutes instead of treating that transient visibility race as data loss.
+        for (int attempt = 1; attempt <= OUTPUT_VISIBILITY_MAX_ATTEMPTS; attempt++) {
+            try {
+                gpdb.runAnalyticQuery(countQuery, expectedCount);
+                return;
+            } catch (AssertionError | PSQLException error) {
+                if (error instanceof PSQLException
+                        && !StringUtils.contains(error.getMessage(), "Input path does not exist")) {
+                    throw error;
+                }
+                if (attempt == OUTPUT_VISIBILITY_MAX_ATTEMPTS) {
+                    throw error;
+                }
+                sleep(OUTPUT_VISIBILITY_RETRY_DELAY_MS);
+            }
+        }
     }
 
     /**
@@ -607,9 +655,7 @@ private void verifyResult(String hdfsPath, Table data, EnumCompressionTypes comp
             throws Exception {
 
         String localResultFile = dataTempFolder + "/" + hdfsPath.replaceAll("/", "_");
-        // wait a bit for async write in previous steps to finish
-        hdfs.waitForFile(hdfsPath, 120);
-        List<String> files = hdfs.list(hdfsPath);
+        List<String> files = waitForOutputFiles(hdfsPath);
         Table resultTable = new Table("result_table", null);
         int index = 0;
         for (String file : files) {
@@ -626,6 +672,26 @@ private void verifyResult(String hdfsPath, Table data, EnumCompressionTypes comp
         ComparisonUtils.compareTables(data, resultTable, null, "\\\\", "\"");
     }
 
+    /**
+     * Wait until a writable-table operation publishes at least one output file.
+     * The target directory is created before INSERT/COPY runs, so waiting only
+     * for the directory would allow a transient empty listing to be treated as
+     * an empty result table.
+     */
+    private List<String> waitForOutputFiles(String hdfsPath) throws Exception {
+        hdfs.waitForFile(hdfsPath, 120);
+        for (int attempt = 1; attempt <= OUTPUT_VISIBILITY_MAX_ATTEMPTS; attempt++) {
+            List<String> files = hdfs.list(hdfsPath);
+            if (!files.isEmpty()) {
+                return files;
+            }
+            if (attempt < OUTPUT_VISIBILITY_MAX_ATTEMPTS) {
+                sleep(OUTPUT_VISIBILITY_RETRY_DELAY_MS);
+            }
+        }
+        throw new AssertionError("No output files became visible in " + hdfsPath);
+    }
+
     /**
      * Get HDFS path to text file (no compression), load it to Table Object and
      * compare with given data Table.

ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh

@@ -38,7 +38,7 @@ if command -v apt-get >/dev/null 2>&1; then
   sudo apt update
   sudo apt install -y openjdk-11-jdk maven
 elif command -v dnf >/dev/null 2>&1; then
-  sudo dnf install -y --nobest java-11-openjdk-devel maven
+  sudo dnf --disablerepo=hpc-common install -y --nobest java-11-openjdk-devel maven
 fi
 
 cd /home/gpadmin/workspace/cloudberry-pxf
@@ -79,11 +79,29 @@ export PXF_HOME=/usr/local/pxf
 sudo mkdir -p "$PXF_HOME"
 sudo chown -R gpadmin:gpadmin "$PXF_HOME"
 
+retry_make() {
+  local attempt=1
+  local max_attempts=3
+  local delay_seconds=30
+
+  until make "$@"; do
+    if (( attempt >= max_attempts )); then
+      echo "Build command failed after ${max_attempts} attempts: make $*" >&2
+      return 1
+    fi
+
+    echo "Build command failed (attempt ${attempt}/${max_attempts}); retrying in ${delay_seconds}s: make $*" >&2
+    sleep "$delay_seconds"
+    ((attempt++))
+    ((delay_seconds *= 2))
+  done
+}
+
 # Build and Install PXF
-make -C external-table install
-make -C fdw install
-make -C cli install
-make -C server install-server
+retry_make -C external-table install
+retry_make -C fdw install
+retry_make -C cli install
+retry_make -C server install-server
 
 # Set up PXF environment