[AURON #2172] Add parquet suite wrappers for Spark versions

.github/workflows/tpcds.yml

@@ -46,21 +46,23 @@ jobs:
     with:
       sparkver: spark-3.1
       hadoop-profile: 'hadoop2.7'
+      sparktests: 'true'
 
   test-spark-32:
     name: Test spark-3.2
     uses: ./.github/workflows/tpcds-reusable.yml
     with:
       sparkver: spark-3.2
       hadoop-profile: 'hadoop2.7'
+      sparktests: 'true'
 
   test-spark-33:
     name: Test spark-3.3
     uses: ./.github/workflows/tpcds-reusable.yml
     with:
       sparkver: spark-3.3
       hadoop-profile: 'hadoop3'
-      sparktests: true
+      sparktests: 'true'
 
   test-spark-34-jdk11:
     name: Test spark-3.4
@@ -69,6 +71,7 @@ jobs:
       sparkver: spark-3.4
       javaver: '11'
       hadoop-profile: 'hadoop3'
+      sparktests: 'true'
 
   test-spark-35-jdk17-scala-2-13:
     name: Test spark-3.5 JDK17 Scala-2.13
@@ -78,6 +81,7 @@ jobs:
       javaver: '17'
       scalaver: '2.13'
       hadoop-profile: 'hadoop3'
+      sparktests: 'true'
 
   test-spark-35-jdk21-scala-2-13:
     name: Test spark-3.5 JDK21 Scala-2.13
@@ -87,6 +91,7 @@ jobs:
       javaver: '21'
       scalaver: '2.13'
       hadoop-profile: 'hadoop3'
+      sparktests: 'true'
 
   test-spark-40-jdk21-scala-2-13:
     name: Test spark-4.0 JDK21 Scala-2.13
@@ -96,6 +101,7 @@ jobs:
       javaver: '21'
       scalaver: '2.13'
       hadoop-profile: 'hadoop3'
+      sparktests: 'true'
 
   test-spark-41-jdk21-scala-2-13:
     name: Test spark-4.1 JDK21 Scala-2.13
@@ -104,4 +110,5 @@ jobs:
       sparkver: spark-4.1
       javaver: '21'
       scalaver: '2.13'
-      hadoop-profile: 'hadoop3'
+      hadoop-profile: 'hadoop3'
+      sparktests: 'true'

auron-spark-tests/pom.xml

@@ -58,6 +58,33 @@
       <version>${sparkVersion}</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-avro</artifactId>
+      <version>${parquetVersion}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-common</artifactId>
+      <version>${parquetVersion}</version>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-column</artifactId>
+      <version>${parquetVersion}</version>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.parquet</groupId>
+      <artifactId>parquet-encoding</artifactId>
+      <version>${parquetVersion}</version>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
   </dependencies>
 
   <build>

auron-spark-tests/spark31/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala

@@ -17,6 +17,7 @@
 package org.apache.auron.utils
 
 import org.apache.spark.sql._
+import org.apache.spark.sql.execution.datasources.parquet._
 
 class AuronSparkTestSettings extends SparkTestSettings {
   {
@@ -94,6 +95,79 @@ class AuronSparkTestSettings extends SparkTestSettings {
     .exclude("SPARK-22271: mean overflows and returns null for some decimal variables")
     .exclude("SPARK-32764: -0.0 and 0.0 should be equal")
 
+  enableSuite[AuronParquetAvroCompatibilitySuite]
+  enableSuite[AuronParquetCompatibilityTest]
+  enableSuite[AuronParquetCompressionCodecPrecedenceSuite]
+  enableSuite[AuronParquetEncodingSuite]
+  enableSuite[AuronParquetFileFormatSuite]
+  enableSuite[AuronParquetFileFormatV1Suite]
+  enableSuite[AuronParquetFileFormatV2Suite]
+  enableSuite[AuronParquetIOSuite]
+    .exclude("read dictionary encoded decimals written as INT32")
+    .exclude("read dictionary encoded decimals written as INT64")
+    .exclude("read dictionary encoded decimals written as FIXED_LEN_BYTE_ARRAY")
+    .exclude("read dictionary and plain encoded timestamp_millis written as INT64")
+    .exclude("SPARK-31159: compatibility with Spark 2.4 in reading dates/timestamps")
+    .exclude("SPARK-31159: rebasing timestamps in write")
+    .exclude("SPARK-31159: rebasing dates in write")
+  enableSuite[AuronParquetInteroperabilitySuite]
+    .exclude("parquet timestamp conversion")
+  enableSuite[AuronParquetPartitionDiscoverySuite]
+  enableSuite[AuronParquetProtobufCompatibilitySuite]
+    .exclude("unannotated array of primitive type")
+    .exclude("unannotated array of struct")
+    .exclude("struct with unannotated array")
+    .exclude("unannotated array of struct with unannotated array")
+    .exclude("unannotated array of string")
+  enableSuite[AuronParquetQuerySuite]
+    .exclude("SPARK-10634 timestamp written and read as INT64 - truncation")
+    .exclude("Enabling/disabling ignoreCorruptFiles")
+    .exclude("SPARK-26677: negated null-safe equality comparison should not filter matched row groups")
+    .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type")
+    .exclude("SPARK-34212 Parquet should read decimals correctly")
+  enableSuite[AuronParquetSchemaInferenceSuite]
+  enableSuite[AuronParquetSchemaPruningSuite]
+  enableSuite[AuronParquetSchemaSuite]
+    .exclude("schema mismatch failure error message for parquet reader")
+    .exclude("schema mismatch failure error message for parquet vectorized reader")
+  enableSuite[AuronParquetTest]
+  enableSuite[AuronParquetThriftCompatibilitySuite]
+    .exclude("Read Parquet file generated by parquet-thrift")
+  enableSuite[AuronParquetV1FilterSuite]
+    .excludeByPrefix("filter pushdown -")
+    .exclude("Filters should be pushed down for vectorized Parquet reader at row group level")
+    .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names")
+    .exclude("Filters should be pushed down for Parquet readers at row group level")
+    .exclude("SPARK-23852: Broken Parquet push-down for partially-written stats")
+    .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down")
+    .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode")
+  enableSuite[AuronParquetV1PartitionDiscoverySuite]
+    .exclude("read partitioned table - partition key included in Parquet file")
+    .exclude("read partitioned table - with nulls and partition keys are included in Parquet file")
+    .exclude("SPARK-18108 Parquet reader fails when data column types conflict with partition ones")
+    .exclude("SPARK-21463: MetadataLogFileIndex should respect userSpecifiedSchema for partition cols")
+  enableSuite[AuronParquetV1QuerySuite]
+    .exclude("SPARK-10634 timestamp written and read as INT64 - truncation")
+    .exclude("Enabling/disabling ignoreCorruptFiles")
+    .exclude("SPARK-26677: negated null-safe equality comparison should not filter matched row groups")
+    .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type")
+    .exclude("SPARK-34212 Parquet should read decimals correctly")
+    .exclude("returning batch for wide table")
+  enableSuite[AuronParquetV1SchemaPruningSuite]
+  enableSuite[AuronParquetV2FilterSuite]
+    .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names")
+    .exclude("Filters should be pushed down for Parquet readers at row group level")
+    .exclude("SPARK-23852: Broken Parquet push-down for partially-written stats")
+    .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down")
+    .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode")
+  enableSuite[AuronParquetV2PartitionDiscoverySuite]
+  enableSuite[AuronParquetV2QuerySuite]
+    .exclude("SPARK-10634 timestamp written and read as INT64 - truncation")
+    .exclude("SPARK-26677: negated null-safe equality comparison should not filter matched row groups")
+    .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type")
+    .exclude("returning batch for wide table")
+  enableSuite[AuronParquetV2SchemaPruningSuite]
+
   override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings {
     override def getResourceFilePath: String = ""
     override def getSupportedSQLQueryTests: Set[String] = Set.empty

auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala

@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.parquet
+
+import org.apache.spark.sql.SparkQueryTestsBase
+
+class AuronParquetAvroCompatibilitySuite
+    extends ParquetAvroCompatibilitySuite
+    with SparkQueryTestsBase {}

auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.parquet
+
+import org.apache.spark.sql.SparkQueryTestsBase
+
+class AuronParquetCompatibilityTest extends ParquetCompatibilityTest with SparkQueryTestsBase {}

auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala

@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.parquet
+
+import org.apache.spark.sql.SparkTestsSharedSessionBase
+
+class AuronParquetCompressionCodecPrecedenceSuite
+    extends ParquetCompressionCodecPrecedenceSuite
+    with SparkTestsSharedSessionBase {}

auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.parquet
+
+import org.apache.spark.sql.SparkQueryTestsBase
+
+class AuronParquetEncodingSuite extends ParquetEncodingSuite with SparkQueryTestsBase {}

auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.parquet
+
+import org.apache.spark.sql.SparkQueryTestsBase
+
+class AuronParquetFileFormatSuite extends ParquetFileFormatSuite with SparkQueryTestsBase {}

auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.parquet
+
+import org.apache.spark.sql.SparkQueryTestsBase
+
+class AuronParquetFileFormatV1Suite extends ParquetFileFormatV1Suite with SparkQueryTestsBase {}

auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.parquet
+
+import org.apache.spark.sql.SparkQueryTestsBase
+
+class AuronParquetFileFormatV2Suite extends ParquetFileFormatV2Suite with SparkQueryTestsBase {}

auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala

@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.parquet
+
+import org.apache.spark.sql.SparkQueryTestsBase
+
+class AuronParquetIOSuite extends ParquetIOSuite with SparkQueryTestsBase {}

auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala

@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.sql.execution.datasources.parquet
+
+import org.apache.spark.sql.SparkQueryTestsBase
+
+class AuronParquetInteroperabilitySuite
+    extends ParquetInteroperabilitySuite
+    with SparkQueryTestsBase {}