GH-926: Shade arrow-compression with Flight JDBC driver

dataset/pom.xml

@@ -33,7 +33,7 @@ under the License.
   <properties>
     <arrow.cpp.build.dir>../../../cpp/release-build/</arrow.cpp.build.dir>
     <parquet.version>1.16.0</parquet.version>
-    <avro.version>1.12.0</avro.version>
+    <avro.version>1.12.1</avro.version>
   </properties>
 
   <dependencies>

flight/flight-sql-jdbc-core/pom.xml

@@ -79,6 +79,12 @@ under the License.
       <classifier>${arrow.vector.classifier}</classifier>
     </dependency>
 
+    <dependency>
+      <groupId>org.apache.arrow</groupId>
+      <artifactId>arrow-compression</artifactId>
+      <scope>runtime</scope>
+    </dependency>
+
     <dependency>
       <groupId>com.google.guava</groupId>
       <artifactId>guava</artifactId>
@@ -151,7 +157,7 @@ under the License.
     <dependency>
       <groupId>com.github.ben-manes.caffeine</groupId>
       <artifactId>caffeine</artifactId>
-      <version>3.2.0</version>
+      <version>3.2.3</version>
     </dependency>
   </dependencies>
 

flight/flight-sql-jdbc-driver/pom.xml

@@ -101,6 +101,8 @@ under the License.
                   <shadedPattern>org.apache.arrow.driver.jdbc.shaded.com.</shadedPattern>
                   <excludes>
                     <exclude>com.sun.**</exclude>
+                    <!-- zstd-jni uses JNI native methods bound to original class names -->
+                    <exclude>com.github.luben.**</exclude>
                   </excludes>
                 </relocation>
                 <relocation>

flight/flight-sql-jdbc-driver/src/shade/LICENSE.txt

@@ -365,3 +365,43 @@ License text:
 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
 | OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.**
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains zstd-jni 1.5.7-6.
+
+Copyright: Copyright (c) 2015-present, Luben Karavelov
+Home page: https://github.com/luben/zstd-jni
+License: BSD 2-Clause License
+License text:
+
+| Copyright (c) 2015-present, Luben Karavelov. All rights reserved.
+|
+| Redistribution and use in source and binary forms, with or without modification,
+| are permitted provided that the following conditions are met:
+|
+| * Redistributions of source code must retain the above copyright notice, this
+|   list of conditions and the following disclaimer.
+|
+| * Redistributions in binary form must reproduce the above copyright notice, this
+|   list of conditions and the following disclaimer in the documentation and/or
+|   other materials provided with the distribution.
+|
+| THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+| ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+| WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+| DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+| ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+| (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+| LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+| ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+| (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+| SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+
+This binary artifact contains Apache Commons Compress 1.27.1.
+
+Copyright: Copyright 2002-2024 The Apache Software Foundation
+Home page: https://commons.apache.org/proper/commons-compress/
+License: https://www.apache.org/licenses/LICENSE-2.0

flight/flight-sql-jdbc-driver/src/shade/NOTICE.txt

@@ -338,3 +338,34 @@ This product includes Netty 4.1.119.Final, with the following in its NOTICE:
 |     * license/LICENSE.brotli4j.txt (Apache License 2.0)
 |   * HOMEPAGE:
 |     * https://github.com/hyperxpro/Brotli4j
+
+---------------------------------------------------
+
+This product includes zstd-jni 1.5.7-6, with the following in its NOTICE:
+
+| zstd-jni - JNI bindings for Zstd
+| =================================
+|
+| Copyright (c) 2015-present, Luben Karavelov. All rights reserved.
+|
+| This product provides JNI bindings for the Zstd compression library.
+|
+|   * LICENSE:
+|     * BSD 2-Clause License
+|   * HOMEPAGE:
+|     * https://github.com/luben/zstd-jni
+
+---------------------------------------------------
+
+This product includes Apache Commons Compress 1.27.1, with the following in its NOTICE:
+
+| Apache Commons Compress
+| Copyright 2002-2024 The Apache Software Foundation
+|
+| This product includes software developed at
+| The Apache Software Foundation (https://www.apache.org/).
+|
+|   * LICENSE:
+|     * Apache License 2.0
+|   * HOMEPAGE:
+|     * https://commons.apache.org/proper/commons-compress/

flight/flight-sql/pom.xml

@@ -113,7 +113,7 @@ under the License.
     <dependency>
       <groupId>org.apache.commons</groupId>
       <artifactId>commons-text</artifactId>
-      <version>1.13.1</version>
+      <version>1.15.0</version>
       <scope>test</scope>
     </dependency>
     <dependency>

pom.xml

@@ -98,20 +98,20 @@ under the License.
     <dep.slf4j.version>2.0.17</dep.slf4j.version>
     <dep.guava-bom.version>33.4.8-jre</dep.guava-bom.version>
     <dep.netty-bom.version>4.2.9.Final</dep.netty-bom.version>
-    <dep.grpc-bom.version>1.73.0</dep.grpc-bom.version>
+    <dep.grpc-bom.version>1.78.0</dep.grpc-bom.version>
     <dep.protobuf-bom.version>4.33.1</dep.protobuf-bom.version>
     <dep.jackson-bom.version>2.18.3</dep.jackson-bom.version>
     <dep.hadoop.version>3.4.2</dep.hadoop.version>
     <dep.fbs.version>25.2.10</dep.fbs.version>
-    <dep.avro.version>1.12.0</dep.avro.version>
+    <dep.avro.version>1.12.1</dep.avro.version>
     <dep.mockito-bom.version>5.17.0</dep.mockito-bom.version>
     <arrow.vector.classifier></arrow.vector.classifier>
     <forkCount>2</forkCount>
     <checkstyle.version>10.23.0</checkstyle.version>
     <checkstyle.failOnViolation>true</checkstyle.failOnViolation>
     <error_prone_core.version>2.42.0</error_prone_core.version>
     <checker.framework.version>3.53.0</checker.framework.version>
-    <logback.version>1.5.21</logback.version>
+    <logback.version>1.5.24</logback.version>
     <doclint>none</doclint>
     <additionalparam>-Xdoclint:none</additionalparam>
     <!-- List of add-opens arg line arguments for tests -->
@@ -505,7 +505,7 @@ under the License.
         <plugin>
           <groupId>org.codehaus.mojo</groupId>
           <artifactId>exec-maven-plugin</artifactId>
-          <version>3.5.0</version>
+          <version>3.6.3</version>
         </plugin>
         <plugin>
           <groupId>org.codehaus.mojo</groupId>

vector/src/main/codegen/templates/AbstractFieldReader.java

@@ -109,10 +109,6 @@ public void copyAsField(String name, ${name}Writer writer) {
 
   </#list></#list>
 
-  public void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory) {
-    fail("CopyAsValue StructWriter");
-  }
-
   public void read(ExtensionHolder holder) {
     fail("Extension");
   }
@@ -147,4 +143,5 @@ public int size() {
   private void fail(String name) {
     throw new IllegalArgumentException(String.format("You tried to read a [%s] type when you are using a field reader of type [%s].", name, this.getClass().getSimpleName()));
   }
+
 }

vector/src/main/codegen/templates/AbstractFieldWriter.java

@@ -107,14 +107,17 @@ public void endEntry() {
     throw new IllegalStateException(String.format("You tried to end a map entry when you are using a ValueWriter of type %s.", this.getClass().getSimpleName()));
   }
 
+  @Override
   public void write(ExtensionHolder var1)  {
-    this.fail("ExtensionType");
+    this.fail("Cannot write ExtensionHolder");
   }
+  @Override
   public void writeExtension(Object var1)  {
-    this.fail("ExtensionType");
+    this.fail("Cannot write extension object");
   }
-  public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) {
-    this.fail("ExtensionType");
+  @Override
+  public void writeExtension(Object var1, ArrowType type)  {
+    this.fail("Cannot write extension with type " + type);
   }
 
   <#list vv.types as type><#list type.minor as minor><#assign name = minor.class?cap_first />

vector/src/main/codegen/templates/ArrowType.java

@@ -27,8 +27,10 @@
 
 import org.apache.arrow.flatbuf.Type;
 import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.complex.writer.FieldWriter;
 import org.apache.arrow.vector.types.*;
 import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.ValueVector;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -331,6 +333,10 @@ public boolean equals(Object obj) {
     public <T> T accept(ArrowTypeVisitor<T> visitor) {
       return visitor.visit(this);
     }
+
+    public FieldWriter getNewFieldWriter(ValueVector vector) {
+      throw new UnsupportedOperationException("WriterImpl not yet implemented.");
+    }
   }
 
   private static final int defaultDecimalBitWidth = 128;

vector/src/main/codegen/templates/BaseReader.java

@@ -49,7 +49,6 @@ public interface RepeatedStructReader extends StructReader{
     boolean next();
     int size();
     void copyAsValue(StructWriter writer);
-    void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory);
   }
 
   public interface ListReader extends BaseReader{
@@ -60,7 +59,6 @@ public interface RepeatedListReader extends ListReader{
     boolean next();
     int size();
     void copyAsValue(ListWriter writer);
-    void copyAsValue(ListWriter writer, ExtensionTypeWriterFactory writerFactory);
   }
 
   public interface MapReader extends BaseReader{
@@ -71,7 +69,6 @@ public interface RepeatedMapReader extends MapReader{
     boolean next();
     int size();
     void copyAsValue(MapWriter writer);
-    void copyAsValue(MapWriter writer, ExtensionTypeWriterFactory writerFactory);
   }
 
   public interface ScalarReader extends

vector/src/main/codegen/templates/BaseWriter.java

@@ -125,11 +125,12 @@ public interface ExtensionWriter extends BaseWriter {
     void writeExtension(Object value);
 
     /**
-     * Adds the given extension type factory. This factory allows configuring writer implementations for specific ExtensionTypeVector.
+     * Writes the given extension type value.
      *
-     * @param factory the extension type factory to add
+     * @param value the extension type value to write
+     * @param type of the extension
      */
-    void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory);
+    void writeExtension(Object value, ArrowType type);
   }
 
   public interface ScalarWriter extends

vector/src/main/codegen/templates/ComplexCopier.java

@@ -41,15 +41,8 @@ public class ComplexCopier {
    * @param input field to read from
    * @param output field to write to
    */
-  public static void copy(FieldReader input, FieldWriter output) {
-    writeValue(input, output, null);
-  }
-
-  public static void copy(FieldReader input, FieldWriter output, ExtensionTypeWriterFactory extensionTypeWriterFactory) {
-    writeValue(input, output, extensionTypeWriterFactory);
-  }
+  public static void copy(FieldReader reader, FieldWriter writer) {
 
-  private static void writeValue(FieldReader reader, FieldWriter writer, ExtensionTypeWriterFactory extensionTypeWriterFactory) {
     final MinorType mt = reader.getMinorType();
 
       switch (mt) {
@@ -65,7 +58,7 @@ private static void writeValue(FieldReader reader, FieldWriter writer, Extension
             FieldReader childReader = reader.reader();
             FieldWriter childWriter = getListWriterForReader(childReader, writer);
             if (childReader.isSet()) {
-              writeValue(childReader, childWriter, extensionTypeWriterFactory);
+              copy(childReader, childWriter);
             } else {
               childWriter.writeNull();
             }
@@ -83,8 +76,8 @@ private static void writeValue(FieldReader reader, FieldWriter writer, Extension
             FieldReader structReader = reader.reader();
             if (structReader.isSet()) {
               writer.startEntry();
-              writeValue(mapReader.key(), getMapWriterForReader(mapReader.key(), writer.key()), extensionTypeWriterFactory);
-              writeValue(mapReader.value(), getMapWriterForReader(mapReader.value(), writer.value()), extensionTypeWriterFactory);
+              copy(mapReader.key(), getMapWriterForReader(mapReader.key(), writer.key()));
+              copy(mapReader.value(), getMapWriterForReader(mapReader.value(), writer.value()));
               writer.endEntry();
             } else {
               writer.writeNull();
@@ -103,7 +96,7 @@ private static void writeValue(FieldReader reader, FieldWriter writer, Extension
             if (childReader.getMinorType() != Types.MinorType.NULL) {
               FieldWriter childWriter = getStructWriterForReader(childReader, writer, name);
               if (childReader.isSet()) {
-                writeValue(childReader, childWriter, extensionTypeWriterFactory);
+                copy(childReader, childWriter);
               } else {
                 childWriter.writeNull();
               }
@@ -115,14 +108,10 @@ private static void writeValue(FieldReader reader, FieldWriter writer, Extension
         }
         break;
       case EXTENSIONTYPE:
-        if (extensionTypeWriterFactory == null) {
-          throw new IllegalArgumentException("Must provide ExtensionTypeWriterFactory");
-        }
         if (reader.isSet()) {
           Object value = reader.readObject();
           if (value != null) {
-            writer.addExtensionTypeWriterFactory(extensionTypeWriterFactory);
-            writer.writeExtension(value);
+            writer.writeExtension(value, reader.getField().getType());
           }
         } else {
           writer.writeNull();

vector/src/main/codegen/templates/NullReader.java

@@ -86,7 +86,6 @@ public void read(int arrayIndex, Nullable${name}Holder holder){
   }
   </#list></#list>
 
-  public void copyAsValue(StructWriter writer, ExtensionTypeWriterFactory writerFactory){}
   public void read(ExtensionHolder holder) {
     holder.isSet = 0;
   }

vector/src/main/codegen/templates/PromotableWriter.java

@@ -286,7 +286,7 @@ protected void setWriter(ValueVector v) {
         writer = new UnionWriter((UnionVector) vector, nullableStructWriterFactory);
         break;
       case EXTENSIONTYPE:
-        writer = new UnionExtensionWriter((ExtensionTypeVector) vector);
+        writer = ((ExtensionType) vector.getField().getType()).getNewFieldWriter(vector);
         break;
       default:
         writer = type.getNewFieldWriter(vector);
@@ -541,17 +541,13 @@ public void writeLargeVarChar(String value) {
   }
 
   @Override
-  public void writeExtension(Object value) {
-    getWriter(MinorType.EXTENSIONTYPE).writeExtension(value);
+  public void writeExtension(Object value, ArrowType arrowType) {
+    getWriter(MinorType.EXTENSIONTYPE, arrowType).writeExtension(value, arrowType);
   }
 
   @Override
-  public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory) {
-    getWriter(MinorType.EXTENSIONTYPE).addExtensionTypeWriterFactory(factory);
-  }
-
-  public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory factory, ArrowType arrowType) {
-    getWriter(MinorType.EXTENSIONTYPE, arrowType).addExtensionTypeWriterFactory(factory);
+  public void write(ExtensionHolder holder) {
+    getWriter(MinorType.EXTENSIONTYPE, holder.type()).write(holder);
   }
 
   @Override

vector/src/main/codegen/templates/UnionListWriter.java

@@ -204,13 +204,13 @@ public MapWriter map(String name, boolean keysSorted) {
 
   @Override
   public ExtensionWriter extension(ArrowType arrowType) {
-    this.extensionType = arrowType;
+    extensionType = arrowType;
     return this;
   }
+
   @Override
   public ExtensionWriter extension(String name, ArrowType arrowType) {
-    ExtensionWriter extensionWriter = writer.extension(name, arrowType);
-    return extensionWriter;
+    return writer.extension(name, arrowType);
   }
 
   <#if listName == "LargeList">
@@ -337,13 +337,13 @@ public void writeNull() {
 
   @Override
   public void writeExtension(Object value) {
-    writer.writeExtension(value);
+    writer.writeExtension(value, extensionType);
     writer.setPosition(writer.idx() + 1);
   }
 
   @Override
-  public void addExtensionTypeWriterFactory(ExtensionTypeWriterFactory var1) {
-    writer.addExtensionTypeWriterFactory(var1, extensionType);
+  public void writeExtension(Object value, ArrowType type) {
+    writeExtension(value);
   }
 
   public void write(ExtensionHolder var1) {