diff --git a/.github/workflows/pxf-ci.yml b/.github/workflows/pxf-ci.yml index 93a5717e2..aa12d99a6 100644 --- a/.github/workflows/pxf-ci.yml +++ b/.github/workflows/pxf-ci.yml @@ -379,6 +379,23 @@ jobs: tar xzf /tmp/cloudberry-source.tar.gz chmod -R u+rwX,go+rX cloudberry + - name: Restore Maven cache + id: cache-mvn + uses: actions/cache/restore@v4 + with: + path: .m2 + key: mvn-container-${{ hashFiles('cloudberry-pxf/automation/pom.xml') }} + restore-keys: | + mvn-container- + + - name: Prepare Maven cache dir + run: | + mkdir -p .m2 + # docker-compose mounts .m2 into container as /home/gpadmin/.m2; + # chmod so gpadmin (uid inside container) can write regardless of + # host-side uid mismatch. + chmod -R a+rwX .m2 || true + - name: Start Services id: start_services run: | @@ -386,6 +403,7 @@ jobs: docker compose -f ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml down -v || true docker compose -f ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml up -d docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin /home/gpadmin/workspace/cloudberry + docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin /home/gpadmin/.m2 docker cp /tmp/*.deb pxf-cbdb-dev:/tmp/ docker exec pxf-cbdb-dev sudo chown gpadmin:gpadmin /tmp/*.deb docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf && ./ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh" @@ -455,6 +473,15 @@ jobs: cd cloudberry-pxf docker compose -f ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml down -v || true + - name: Save Maven cache + # Save on every branch so PRs also benefit on rerun; scoped per branch + # by GHA automatically. Only save if we didn't already have the exact key. + if: always() && steps.cache-mvn.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: .m2 + key: mvn-container-${{ hashFiles('cloudberry-pxf/automation/pom.xml') }} + - name: Upload test artifacts if: always() uses: actions/upload-artifact@v4 @@ -469,7 +496,7 @@ jobs: run: | FAILED_COUNT="${{ steps.collect_artifacts.outputs.failed_count || 0 }}" SKIPPED_COUNT="${{ steps.collect_artifacts.outputs.skipped_count || 0 }}" - + if [ "${{ steps.run_test.outcome }}" == "failure" ] || [ "$FAILED_COUNT" -gt 0 ]; then echo "Test group ${{ matrix.test_group }} failed (Failures: $FAILED_COUNT, Skipped: $SKIPPED_COUNT)" exit 1 @@ -548,6 +575,20 @@ jobs: tar xzf /tmp/cloudberry-source-rocky9.tar.gz chmod -R u+rwX,go+rX cloudberry + - name: Restore Maven cache + id: cache-mvn + uses: actions/cache/restore@v4 + with: + path: .m2 + key: mvn-container-${{ hashFiles('cloudberry-pxf/automation/pom.xml') }} + restore-keys: | + mvn-container- + + - name: Prepare Maven cache dir + run: | + mkdir -p .m2 + chmod -R a+rwX .m2 || true + - name: Start Services id: start_services run: | @@ -555,6 +596,7 @@ jobs: docker compose -f ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml down -v || true docker compose -f ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml up -d docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin /home/gpadmin/workspace/cloudberry + docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin /home/gpadmin/.m2 docker cp /tmp/*.rpm pxf-cbdb-dev:/tmp/ docker exec pxf-cbdb-dev sudo chown gpadmin:gpadmin /tmp/*.rpm docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf && ./ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh" @@ -624,6 +666,13 @@ jobs: cd cloudberry-pxf docker compose -f ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml down -v || true + - name: Save Maven cache + if: always() && steps.cache-mvn.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: .m2 + key: mvn-container-${{ hashFiles('cloudberry-pxf/automation/pom.xml') }} + - name: Upload test artifacts if: always() uses: actions/upload-artifact@v4 @@ -682,6 +731,52 @@ jobs: distribution: temurin java-version: ${{ env.JAVA_VERSION }} + - name: Restore Maven cache (host) + id: cache-mvn-host + uses: actions/cache/restore@v4 + with: + path: ~/.m2/repository + key: mvn-host-${{ hashFiles('automation/pom.xml') }} + restore-keys: | + mvn-host- + + - name: Restore MSSQL image cache + if: matrix.tc_group == 'pxf-jdbc' + id: cache-mssql + uses: actions/cache/restore@v4 + with: + path: /tmp/mssql-image.tar + key: mssql-server-2019-latest-v1 + + - name: Load or pull MSSQL image + # Testcontainers pulls mcr.microsoft.com/mssql/server:2019-latest at + # test time for JdbcMssqlTest. MCR occasionally returns transient + # 5xx on GHA; we (a) restore from cache when possible and (b) retry + # the pull with backoff on cache miss, so a bad MCR minute doesn't + # abort the whole TC job. Only pxf-jdbc TC group exercises MSSQL. + if: matrix.tc_group == 'pxf-jdbc' + run: | + if [ -f /tmp/mssql-image.tar ]; then + echo "Loading MSSQL image from cache" + docker load < /tmp/mssql-image.tar + else + echo "Pulling MSSQL image from MCR (with retry)" + for attempt in 1 2 3; do + if docker pull mcr.microsoft.com/mssql/server:2019-latest; then + break + fi + if [ "$attempt" -lt 3 ]; then + sleep_s=$((attempt * 15)) + echo "docker pull failed on attempt $attempt/3, sleeping ${sleep_s}s..." + sleep "$sleep_s" + else + echo "docker pull failed after 3 attempts, giving up" + exit 1 + fi + done + docker save mcr.microsoft.com/mssql/server:2019-latest > /tmp/mssql-image.tar + fi + - name: Download pxf-cbdb testcontainer image uses: actions/download-artifact@v4 with: @@ -747,6 +842,20 @@ jobs: echo "skipped_count=$SKIPPED" >> $GITHUB_OUTPUT echo "Test stats for tc:$TC_GROUP ($TEST_MODE, ${{ matrix.distro }}): total=$TOTAL, passed=$PASSED, failed=$FAILED, skipped=$SKIPPED" + - name: Save Maven cache (host) + if: always() && steps.cache-mvn-host.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: ~/.m2/repository + key: mvn-host-${{ hashFiles('automation/pom.xml') }} + + - name: Save MSSQL image cache + if: always() && matrix.tc_group == 'pxf-jdbc' && steps.cache-mssql.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + with: + path: /tmp/mssql-image.tar + key: mssql-server-2019-latest-v1 + - name: Upload test artifacts if: always() uses: actions/upload-artifact@v4 diff --git a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/writable/HdfsWritableTextTest.java b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/writable/HdfsWritableTextTest.java index f1ba6c9c1..d2fed82db 100755 --- a/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/writable/HdfsWritableTextTest.java +++ b/automation/src/test/java/org/apache/cloudberry/pxf/automation/features/writable/HdfsWritableTextTest.java @@ -466,7 +466,7 @@ public void copyFromFileMultiBlockedDataNoCompression() throws Exception { FileFormatsUtils.prepareData(new WritableDataPreparer(), 1000, data); // multiple it to file String multiBlockedLocalFilePath = dataTempFolder + "/multiBlockedData"; - FileFormatsUtils.prepareDataFile(data, 15000, multiBlockedLocalFilePath); + FileFormatsUtils.prepareDataFile(data, 1000, multiBlockedLocalFilePath); String hdfsPath = hdfsWritePath + "/copy_from_file_multi_block_no_compression"; writableExTable = prepareWritableTable("pxf_text_multi_block_no_compression_w", hdfsPath, null); @@ -480,7 +480,7 @@ public void copyFromFileMultiBlockedDataNoCompression() throws Exception { readableExTable = prepareReadableTable("pxf_text_multi_block_no_compression_r", hdfsPath); gpdb.runAnalyticQuery("SELECT COUNT(*) FROM " + readableExTable.getName(), - String.valueOf(1000 * 15000)); + String.valueOf(1000 * 1000)); } /** @@ -495,7 +495,7 @@ public void copyFromFileMultiBlockedDataGZip() throws Exception { FileFormatsUtils.prepareData(new WritableDataPreparer(), 1000, data); // multiple it to file String multiBlockedLocalFilePath = (dataTempFolder + "/multiBlockedData_gzip"); - FileFormatsUtils.prepareDataFile(data, 15000, multiBlockedLocalFilePath); + FileFormatsUtils.prepareDataFile(data, 1000, multiBlockedLocalFilePath); String hdfsPath = hdfsWritePath + "/copy_from_file_multi_block_gzip"; writableExTable = prepareWritableGzipTable("pxf_text_multi_block_gzip_w", hdfsPath); @@ -509,7 +509,7 @@ public void copyFromFileMultiBlockedDataGZip() throws Exception { readableExTable = prepareReadableTable("pxf_text_multi_block_gzip_r", hdfsPath); gpdb.runAnalyticQuery("SELECT COUNT(*) FROM " + readableExTable.getName(), - String.valueOf(1000 * 15000)); + String.valueOf(1000 * 1000)); } /** @@ -525,7 +525,7 @@ public void copyFromFileMultiBlockedDataBZip2() throws Exception { FileFormatsUtils.prepareData(new WritableDataPreparer(), 1000, data); // multiple it to file String multiBlockedLocalFilePath = dataTempFolder + "/multiBlockedData_bzip"; - FileFormatsUtils.prepareDataFile(data, 15000, multiBlockedLocalFilePath); + FileFormatsUtils.prepareDataFile(data, 1000, multiBlockedLocalFilePath); String hdfsPath = hdfsWritePath + "/copy_from_file_multi_block_bzip2"; writableExTable = prepareWritableBZip2Table("pxf_text_multi_block_bzip2_w", hdfsPath); @@ -539,7 +539,7 @@ public void copyFromFileMultiBlockedDataBZip2() throws Exception { readableExTable = prepareReadableTable("pxf_text_multi_block_bzip2_r", hdfsPath); gpdb.runAnalyticQuery("SELECT COUNT(*) FROM " + readableExTable.getName(), - String.valueOf(1000 * 15000)); + String.valueOf(1000 * 1000)); } /** diff --git a/ci/docker/pxf-cbdb-dev/common/script/build_cloudberrry.sh b/ci/docker/pxf-cbdb-dev/common/script/build_cloudberrry.sh index 854048339..0f7858c56 100755 --- a/ci/docker/pxf-cbdb-dev/common/script/build_cloudberrry.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/build_cloudberrry.sh @@ -20,11 +20,18 @@ # -------------------------------------------------------------------- # Build Cloudberry from source — works on both Ubuntu and Rocky/RHEL +# Pull in shared helpers (log/die/retry). Sourced with an absolute path so +# this script works whether invoked directly or via docker exec. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/utils.sh" + # Install sudo & git if command -v apt-get >/dev/null 2>&1; then - sudo apt update && sudo apt install -y sudo git + retry sudo apt update + retry sudo apt install -y sudo git elif command -v dnf >/dev/null 2>&1; then - sudo dnf install -y --nobest sudo git + retry sudo dnf install -y --nobest sudo git fi # Required configuration @@ -65,8 +72,8 @@ ulimit -a # Install basic system packages if command -v apt-get >/dev/null 2>&1; then - sudo apt update - sudo apt install -y bison \ + retry sudo apt update + retry sudo apt install -y bison \ bzip2 \ cmake \ curl \ @@ -104,7 +111,7 @@ if command -v apt-get >/dev/null 2>&1; then rsync \ libsnappy-dev elif command -v dnf >/dev/null 2>&1; then - sudo dnf install -y --nobest \ + retry sudo dnf install -y --nobest \ bison \ bzip2 \ cmake \ diff --git a/ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh b/ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh index 6c4c1c83c..bb8b3ae69 100755 --- a/ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/build_pxf.sh @@ -20,6 +20,12 @@ # -------------------------------------------------------------------- # Build and install PXF — works on both Ubuntu and Rocky/RHEL +# Pull in shared helpers (log/die/retry). Sourced with an absolute path so +# this script works whether invoked directly or via docker exec. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/utils.sh" + # Auto-detect Java 11 path if [ -d /usr/lib/jvm/java-11-openjdk-amd64 ]; then JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-11-openjdk-amd64} @@ -35,10 +41,10 @@ export PATH=$GPHOME/bin:$PATH # Install Java 11 JDK and Maven if command -v apt-get >/dev/null 2>&1; then - sudo apt update - sudo apt install -y openjdk-11-jdk maven + retry sudo apt update + retry sudo apt install -y openjdk-11-jdk maven elif command -v dnf >/dev/null 2>&1; then - sudo dnf install -y --nobest java-11-openjdk-devel maven + retry sudo dnf install -y --nobest java-11-openjdk-devel maven fi cd /home/gpadmin/workspace/cloudberry-pxf diff --git a/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh index 7a4fe638d..b6b2bdb1c 100755 --- a/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/entrypoint.sh @@ -58,13 +58,13 @@ detect_java_paths() { setup_locale_and_packages() { log "install base packages and locales" if [ "$OS_FAMILY" = "deb" ]; then - sudo apt-get update - sudo apt-get install -y wget lsb-release locales maven unzip openssh-server iproute2 sudo \ + retry sudo apt-get update + retry sudo apt-get install -y wget lsb-release locales maven unzip openssh-server iproute2 sudo \ openjdk-11-jre-headless openjdk-8-jre-headless sudo locale-gen en_US.UTF-8 ru_RU.CP1251 ru_RU.UTF-8 sudo update-locale LANG=en_US.UTF-8 else - sudo dnf install -y --nobest wget maven unzip openssh-server iproute sudo \ + retry sudo dnf install -y --nobest wget maven unzip openssh-server iproute sudo \ java-11-openjdk-headless java-1.8.0-openjdk-headless \ glibc-langpack-en glibc-locale-source sudo localedef -c -i en_US -f UTF-8 en_US.UTF-8 || true @@ -145,17 +145,17 @@ EOF install_build_deps() { if [ "$OS_FAMILY" = "deb" ]; then - sudo apt update && sudo apt install -y sudo git - sudo apt update - sudo apt install -y bison bzip2 cmake curl flex gcc g++ iproute2 iputils-ping \ + retry sudo apt update + retry sudo apt install -y sudo git + retry sudo apt install -y bison bzip2 cmake curl flex gcc g++ iproute2 iputils-ping \ language-pack-en locales libapr1-dev libbz2-dev libcurl4-gnutls-dev libevent-dev \ libkrb5-dev libipc-run-perl libldap2-dev libpam0g-dev libprotobuf-dev libreadline-dev \ libssl-dev libuv1-dev liblz4-dev libxerces-c-dev libxml2-dev libyaml-dev libzstd-dev \ libperl-dev make pkg-config protobuf-compiler python3-dev python3-pip python3-setuptools \ rsync libsnappy-dev else - sudo dnf install -y --nobest sudo git - sudo dnf install -y --nobest --allowerasing bison bzip2 cmake curl flex gcc gcc-c++ iproute iputils \ + retry sudo dnf install -y --nobest sudo git + retry sudo dnf install -y --nobest --allowerasing bison bzip2 cmake curl flex gcc gcc-c++ iproute iputils \ glibc-langpack-en glibc-locale-source apr-devel bzip2-devel libcurl-devel libevent-devel \ krb5-devel perl-IPC-Run openldap-devel pam-devel protobuf-devel readline-devel \ openssl-devel libuv-devel lz4-devel libxml2-devel libyaml-devel \ @@ -341,8 +341,13 @@ wait_for_datanode() { local max_attempts=2 for _attempt in $(seq 1 ${max_attempts}); do local dn_ready=false - # Wait up to 90s (45 tries * 2s) for DataNode to register - for _dn_try in $(seq 1 45); do + # Wait up to 180s (90 tries * 2s) for DataNode to register. GHA free-tier + # runners under I/O contention can take 90-180s for JVM cold-start + + # overlay2 block scan + NameNode handshake; the earlier 90s window was + # tight enough that Test PXF Rocky9 - smoke intermittently hit both + # attempts before DataNode came Live. Healthy runs still complete in + # 20-40s, so this only extends the tail — no cost on the happy path. + for _dn_try in $(seq 1 90); do if hdfs dfsadmin -report 2>/dev/null | grep -q "Live datanodes.*[1-9]"; then dn_ready=true break @@ -356,7 +361,7 @@ wait_for_datanode() { fi # DataNode didn't come up; diagnose and attempt restart - log "DataNode not available after 90s (attempt ${_attempt}/${max_attempts})" + log "DataNode not available after 180s (attempt ${_attempt}/${max_attempts})" log "--- DataNode diagnostic info ---" # Check if DataNode process is alive if command -v jps >/dev/null 2>&1; then diff --git a/ci/docker/pxf-cbdb-dev/common/script/utils.sh b/ci/docker/pxf-cbdb-dev/common/script/utils.sh index b6936ef51..e6bdba9fe 100755 --- a/ci/docker/pxf-cbdb-dev/common/script/utils.sh +++ b/ci/docker/pxf-cbdb-dev/common/script/utils.sh @@ -25,6 +25,27 @@ set -euo pipefail log() { echo "[utils][$(date '+%F %T')] $*"; } die() { log "ERROR $*"; exit 1; } +# retry — run up to $RETRY_MAX_ATTEMPTS times (default 3), +# waiting $RETRY_DELAY seconds between attempts (default 15). Used to +# harden CI against transient upstream failures: dnf/apt mirror hiccups, +# Maven Central connect flakes, image registry throttling, etc. Exits +# with the last command's exit code on final failure. +retry() { + local -r max_attempts=${RETRY_MAX_ATTEMPTS:-3} + local -r delay=${RETRY_DELAY:-15} + local attempt=1 + until "$@"; do + if [ "$attempt" -ge "$max_attempts" ]; then + log "retry: '$*' failed after $attempt attempts, giving up" + return 1 + fi + log "retry: '$*' failed (attempt $attempt/$max_attempts), waiting ${delay}s" + sleep "$delay" + attempt=$((attempt + 1)) + done + return 0 +} + wait_port() { local host="$1" port="$2" retries="${3:-10}" sleep_sec="${4:-2}" local i diff --git a/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml b/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml index 37738078d..93e818e1d 100644 --- a/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml +++ b/ci/docker/pxf-cbdb-dev/rocky9/docker-compose.yml @@ -34,11 +34,23 @@ services: hostname: mdw depends_on: - singlecluster + environment: + # Pin container TZ so JVM's ZoneId.systemDefault() (used by + # ParquetTypeConverter for INT96 timestamp encode/decode) is the + # same across distros. Matches the PGOPTIONS timezone='GMT-1' set + # by run_tests.sh, keeping the Parquet-vs-PG timestamp path + # internally consistent. + TZ: Etc/GMT-1 ports: - "2222:22" volumes: - ../../../../../cloudberry-pxf:/home/gpadmin/workspace/cloudberry-pxf - ../../../../../cloudberry:/home/gpadmin/workspace/cloudberry + # Share host's Maven local repo with the container so mvn dependency + # downloads survive across runs via actions/cache. Neutralises transient + # 5xx from Maven Central and single-source repos (top-q.co.il for + # JSystem) once the cache is warm. + - ../../../../../.m2:/home/gpadmin/.m2 command: ["tail", "-f", "/dev/null"] networks: diff --git a/ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml b/ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml index 02519cb96..eee4cee2c 100644 --- a/ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml +++ b/ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml @@ -33,11 +33,23 @@ services: hostname: mdw depends_on: - singlecluster + environment: + # Pin container TZ so JVM's ZoneId.systemDefault() (used by + # ParquetTypeConverter for INT96 timestamp encode/decode) is the + # same across distros. Matches the PGOPTIONS timezone='GMT-1' set + # by run_tests.sh, keeping the Parquet-vs-PG timestamp path + # internally consistent. + TZ: Etc/GMT-1 ports: - "2222:22" volumes: - ../../../../../cloudberry-pxf:/home/gpadmin/workspace/cloudberry-pxf - ../../../../../cloudberry:/home/gpadmin/workspace/cloudberry + # Share host's Maven local repo with the container so mvn dependency + # downloads survive across runs via actions/cache. Neutralises transient + # 5xx from Maven Central and single-source repos (top-q.co.il for + # JSystem) once the cache is warm. + - ../../../../../.m2:/home/gpadmin/.m2 command: ["tail", "-f", "/dev/null"] networks: diff --git a/ci/singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml b/ci/singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml index e75a7eba2..d870911c6 100755 --- a/ci/singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml +++ b/ci/singlecluster/templates/hadoop/etc/hadoop/hdfs-site.xml @@ -34,11 +34,46 @@ 3 - dfs.datanode.socket.write.timeout + dfs.datanode.socket.write.timeout 0 used for sockets to and from datanodes. It is 8 minutes by default. Some users set this to 0, effectively disabling the write timeout. - + + + + dfs.client.socket-timeout + 180000 + + HDFS client-side socket timeout in ms (default 60s). Raised to + 3 minutes so shared-runner CI (GitHub Actions free tier) does + not spuriously abort large-file writes when the DataNode is + slow to ACK due to noisy-neighbor I/O contention. 3 minutes + is ~3x the worst observed ACK stall on GHA free runners while + still failing fast when a DataNode is genuinely dead. Symptom + without this: HdfsWritableTextTest multi-block variants + silently drop their write and the readback finds no data. + + + + dfs.blocksize + 16777216 + + Default HDFS block size in bytes (16 MB, down from the Hadoop + default of 128 MB). This is a test cluster: smaller blocks + let "multi-block" test coverage kick in on much smaller + input data (e.g. HdfsWritableTextTest's + copyFromFileMultiBlockedData* variants), keeping the tests' + original intent (cross-block-boundary reader/writer paths) + while cutting write I/O by an order of magnitude — critical + on shared GHA runners that otherwise time out. + 16 MB (not 4 MB) leaves headroom for existing row sizes such + as textFormatWideRowsInsert's 10 MiB rows — they fit inside + one block, so cross-block row-boundary paths are still + exercised by copyFromFileMultiBlockedData* (60 MB / 16 MB + ≈ 4 blocks) rather than by every row of every wide-row test. + Real-world deployments still use the Hadoop default; this is + CI-only. + dfs.webhdfs.enabled