diff --git a/.github/workflows/tpcds.yml b/.github/workflows/tpcds.yml index f4c9c2cd6..d2711dee5 100644 --- a/.github/workflows/tpcds.yml +++ b/.github/workflows/tpcds.yml @@ -46,6 +46,7 @@ jobs: with: sparkver: spark-3.1 hadoop-profile: 'hadoop2.7' + sparktests: 'true' test-spark-32: name: Test spark-3.2 @@ -53,6 +54,7 @@ jobs: with: sparkver: spark-3.2 hadoop-profile: 'hadoop2.7' + sparktests: 'true' test-spark-33: name: Test spark-3.3 @@ -60,7 +62,7 @@ jobs: with: sparkver: spark-3.3 hadoop-profile: 'hadoop3' - sparktests: true + sparktests: 'true' test-spark-34-jdk11: name: Test spark-3.4 @@ -69,6 +71,7 @@ jobs: sparkver: spark-3.4 javaver: '11' hadoop-profile: 'hadoop3' + sparktests: 'true' test-spark-35-jdk17-scala-2-13: name: Test spark-3.5 JDK17 Scala-2.13 @@ -78,6 +81,7 @@ jobs: javaver: '17' scalaver: '2.13' hadoop-profile: 'hadoop3' + sparktests: 'true' test-spark-35-jdk21-scala-2-13: name: Test spark-3.5 JDK21 Scala-2.13 @@ -87,6 +91,7 @@ jobs: javaver: '21' scalaver: '2.13' hadoop-profile: 'hadoop3' + sparktests: 'true' test-spark-40-jdk21-scala-2-13: name: Test spark-4.0 JDK21 Scala-2.13 @@ -96,6 +101,7 @@ jobs: javaver: '21' scalaver: '2.13' hadoop-profile: 'hadoop3' + sparktests: 'true' test-spark-41-jdk21-scala-2-13: name: Test spark-4.1 JDK21 Scala-2.13 @@ -104,4 +110,5 @@ jobs: sparkver: spark-4.1 javaver: '21' scalaver: '2.13' - hadoop-profile: 'hadoop3' \ No newline at end of file + hadoop-profile: 'hadoop3' + sparktests: 'true' diff --git a/auron-spark-tests/pom.xml b/auron-spark-tests/pom.xml index 795f888dc..c6edba887 100644 --- a/auron-spark-tests/pom.xml +++ b/auron-spark-tests/pom.xml @@ -58,6 +58,33 @@ ${sparkVersion} test + + org.apache.parquet + parquet-avro + ${parquetVersion} + test + + + org.apache.parquet + parquet-common + ${parquetVersion} + tests + test + + + org.apache.parquet + parquet-column + ${parquetVersion} + tests + test + + + org.apache.parquet + parquet-encoding + ${parquetVersion} + tests + test + diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala index 1a14da87e..fed23ed24 100644 --- a/auron-spark-tests/spark31/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala @@ -17,6 +17,7 @@ package org.apache.auron.utils import org.apache.spark.sql._ +import org.apache.spark.sql.execution.datasources.parquet._ class AuronSparkTestSettings extends SparkTestSettings { { @@ -94,6 +95,79 @@ class AuronSparkTestSettings extends SparkTestSettings { .exclude("SPARK-22271: mean overflows and returns null for some decimal variables") .exclude("SPARK-32764: -0.0 and 0.0 should be equal") + enableSuite[AuronParquetAvroCompatibilitySuite] + enableSuite[AuronParquetCompatibilityTest] + enableSuite[AuronParquetCompressionCodecPrecedenceSuite] + enableSuite[AuronParquetEncodingSuite] + enableSuite[AuronParquetFileFormatSuite] + enableSuite[AuronParquetFileFormatV1Suite] + enableSuite[AuronParquetFileFormatV2Suite] + enableSuite[AuronParquetIOSuite] + .exclude("read dictionary encoded decimals written as INT32") + .exclude("read dictionary encoded decimals written as INT64") + .exclude("read dictionary encoded decimals written as FIXED_LEN_BYTE_ARRAY") + .exclude("read dictionary and plain encoded timestamp_millis written as INT64") + .exclude("SPARK-31159: compatibility with Spark 2.4 in reading dates/timestamps") + .exclude("SPARK-31159: rebasing timestamps in write") + .exclude("SPARK-31159: rebasing dates in write") + enableSuite[AuronParquetInteroperabilitySuite] + .exclude("parquet timestamp conversion") + enableSuite[AuronParquetPartitionDiscoverySuite] + enableSuite[AuronParquetProtobufCompatibilitySuite] + .exclude("unannotated array of primitive type") + .exclude("unannotated array of struct") + .exclude("struct with unannotated array") + .exclude("unannotated array of struct with unannotated array") + .exclude("unannotated array of string") + enableSuite[AuronParquetQuerySuite] + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude("Enabling/disabling ignoreCorruptFiles") + .exclude("SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("SPARK-34212 Parquet should read decimals correctly") + enableSuite[AuronParquetSchemaInferenceSuite] + enableSuite[AuronParquetSchemaPruningSuite] + enableSuite[AuronParquetSchemaSuite] + .exclude("schema mismatch failure error message for parquet reader") + .exclude("schema mismatch failure error message for parquet vectorized reader") + enableSuite[AuronParquetTest] + enableSuite[AuronParquetThriftCompatibilitySuite] + .exclude("Read Parquet file generated by parquet-thrift") + enableSuite[AuronParquetV1FilterSuite] + .excludeByPrefix("filter pushdown -") + .exclude("Filters should be pushed down for vectorized Parquet reader at row group level") + .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") + .exclude("Filters should be pushed down for Parquet readers at row group level") + .exclude("SPARK-23852: Broken Parquet push-down for partially-written stats") + .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down") + .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode") + enableSuite[AuronParquetV1PartitionDiscoverySuite] + .exclude("read partitioned table - partition key included in Parquet file") + .exclude("read partitioned table - with nulls and partition keys are included in Parquet file") + .exclude("SPARK-18108 Parquet reader fails when data column types conflict with partition ones") + .exclude("SPARK-21463: MetadataLogFileIndex should respect userSpecifiedSchema for partition cols") + enableSuite[AuronParquetV1QuerySuite] + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude("Enabling/disabling ignoreCorruptFiles") + .exclude("SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("SPARK-34212 Parquet should read decimals correctly") + .exclude("returning batch for wide table") + enableSuite[AuronParquetV1SchemaPruningSuite] + enableSuite[AuronParquetV2FilterSuite] + .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") + .exclude("Filters should be pushed down for Parquet readers at row group level") + .exclude("SPARK-23852: Broken Parquet push-down for partially-written stats") + .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down") + .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode") + enableSuite[AuronParquetV2PartitionDiscoverySuite] + enableSuite[AuronParquetV2QuerySuite] + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude("SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("returning batch for wide table") + enableSuite[AuronParquetV2SchemaPruningSuite] + override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings { override def getResourceFilePath: String = "" override def getSupportedSQLQueryTests: Set[String] = Set.empty diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala new file mode 100644 index 000000000..aba4a3714 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetAvroCompatibilitySuite + extends ParquetAvroCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala new file mode 100644 index 000000000..a049cf2b6 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetCompatibilityTest extends ParquetCompatibilityTest with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala new file mode 100644 index 000000000..ffa5355d6 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetCompressionCodecPrecedenceSuite + extends ParquetCompressionCodecPrecedenceSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala new file mode 100644 index 000000000..379d2d122 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetEncodingSuite extends ParquetEncodingSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala new file mode 100644 index 000000000..6f20c5547 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatSuite extends ParquetFileFormatSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala new file mode 100644 index 000000000..2e8dc6e83 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatV1Suite extends ParquetFileFormatV1Suite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala new file mode 100644 index 000000000..850a4d8b8 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatV2Suite extends ParquetFileFormatV2Suite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala new file mode 100644 index 000000000..96fb78238 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetIOSuite extends ParquetIOSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala new file mode 100644 index 000000000..e25558ca5 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetInteroperabilitySuite + extends ParquetInteroperabilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala new file mode 100644 index 000000000..af37a69c0 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetPartitionDiscoverySuite + extends ParquetPartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala new file mode 100644 index 000000000..83ef15c77 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetProtobufCompatibilitySuite + extends ParquetProtobufCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala new file mode 100644 index 000000000..40fa45239 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetQuerySuite extends ParquetQuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala new file mode 100644 index 000000000..6be422dde --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetSchemaInferenceSuite + extends ParquetSchemaInferenceSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala new file mode 100644 index 000000000..301c12eb5 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetSchemaPruningSuite extends ParquetSchemaPruningSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala new file mode 100644 index 000000000..ada8eb92f --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetSchemaSuite extends ParquetSchemaSuite with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala new file mode 100644 index 000000000..cc937309c --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetTest extends ParquetTest with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala new file mode 100644 index 000000000..0f9ef6a44 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetThriftCompatibilitySuite + extends ParquetThriftCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala new file mode 100644 index 000000000..8e4638841 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1FilterSuite extends ParquetV1FilterSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala new file mode 100644 index 000000000..aa28b253e --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1PartitionDiscoverySuite + extends ParquetV1PartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala new file mode 100644 index 000000000..a0d750850 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1QuerySuite extends ParquetV1QuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala new file mode 100644 index 000000000..6aecd6666 --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1SchemaPruningSuite + extends ParquetV1SchemaPruningSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala new file mode 100644 index 000000000..05029c1da --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2FilterSuite extends ParquetV2FilterSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala new file mode 100644 index 000000000..677498f0b --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2PartitionDiscoverySuite + extends ParquetV2PartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala new file mode 100644 index 000000000..f8abbceca --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2QuerySuite extends ParquetV2QuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala new file mode 100644 index 000000000..30b1f47db --- /dev/null +++ b/auron-spark-tests/spark31/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2SchemaPruningSuite + extends ParquetV2SchemaPruningSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala index b881e1185..7deb2c978 100644 --- a/auron-spark-tests/spark32/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala @@ -17,6 +17,7 @@ package org.apache.auron.utils import org.apache.spark.sql._ +import org.apache.spark.sql.execution.datasources.parquet._ class AuronSparkTestSettings extends SparkTestSettings { { @@ -71,6 +72,7 @@ class AuronSparkTestSettings extends SparkTestSettings { "SPARK-19471: AggregationIterator does not initialize the generated result projection before using it") .exclude( "SPARK-24788: RelationalGroupedDataset.toString with unresolved exprs should not fail") + .exclude("SPARK-34837: Support ANSI SQL intervals by the aggregate function `avg`") enableSuite[AuronDatasetAggregatorSuite] @@ -91,6 +93,105 @@ class AuronSparkTestSettings extends SparkTestSettings { .exclude("SPARK-22271: mean overflows and returns null for some decimal variables") .exclude("SPARK-32764: -0.0 and 0.0 should be equal") + enableSuite[AuronParquetAvroCompatibilitySuite] + enableSuite[AuronParquetColumnIndexSuite] + enableSuite[AuronParquetCompatibilityTest] + enableSuite[AuronParquetCompressionCodecPrecedenceSuite] + enableSuite[AuronParquetEncodingSuite] + enableSuite[AuronParquetFileFormatSuite] + enableSuite[AuronParquetFileFormatV1Suite] + enableSuite[AuronParquetFileFormatV2Suite] + enableSuite[AuronParquetIOSuite] + .exclude("SPARK-34817: Read UINT_64 as Decimal from parquet") + .exclude("SPARK-35640: read binary as timestamp should throw schema incompatible error") + .exclude("SPARK-35640: int as long should throw schema incompatible error") + .exclude("read dictionary encoded decimals written as INT32") + .exclude("read dictionary encoded decimals written as INT64") + .exclude("read dictionary encoded decimals written as FIXED_LEN_BYTE_ARRAY") + .exclude("read dictionary and plain encoded timestamp_millis written as INT64") + .exclude("SPARK-36726: test incorrect Parquet row group file offset") + .exclude("SPARK-34167: read LongDecimals with precision < 10, VectorizedReader true") + .exclude("SPARK-34167: read LongDecimals with precision < 10, VectorizedReader false") + enableSuite[AuronParquetInteroperabilitySuite] + .exclude("parquet timestamp conversion") + enableSuite[AuronParquetPartitionDiscoverySuite] + enableSuite[AuronParquetProtobufCompatibilitySuite] + .exclude("unannotated array of primitive type") + .exclude("unannotated array of struct") + .exclude("struct with unannotated array") + .exclude("unannotated array of struct with unannotated array") + .exclude("unannotated array of string") + enableSuite[AuronParquetQuerySuite] + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude("Enabling/disabling ignoreCorruptFiles") + .exclude("SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("SPARK-34212 Parquet should read decimals correctly") + enableSuite[AuronParquetRebaseDatetimeSuite] + .exclude("SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps") + .exclude("SPARK-31159, SPARK-37705: rebasing timestamps in write") + .exclude("SPARK-31159: rebasing dates in write") + .exclude("SPARK-35427: datetime rebasing in the EXCEPTION mode") + enableSuite[AuronParquetRebaseDatetimeV1Suite] + .exclude("SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps") + .exclude("SPARK-31159, SPARK-37705: rebasing timestamps in write") + .exclude("SPARK-31159: rebasing dates in write") + .exclude("SPARK-35427: datetime rebasing in the EXCEPTION mode") + enableSuite[AuronParquetRebaseDatetimeV2Suite] + .exclude("SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps") + .exclude("SPARK-35427: datetime rebasing in the EXCEPTION mode") + enableSuite[AuronParquetSchemaInferenceSuite] + enableSuite[AuronParquetSchemaPruningSuite] + enableSuite[AuronParquetSchemaSuite] + .exclude("schema mismatch failure error message for parquet reader") + .exclude("schema mismatch failure error message for parquet vectorized reader") + .exclude("SPARK-40819: parquet file with TIMESTAMP(NANOS, true) (with nanosAsLong=true)") + .exclude("SPARK-40819: parquet file with TIMESTAMP(NANOS, true) (with default nanosAsLong=false)") + enableSuite[AuronParquetTest] + enableSuite[AuronParquetThriftCompatibilitySuite] + .exclude("Read Parquet file generated by parquet-thrift") + enableSuite[AuronParquetV1FilterSuite] + .excludeByPrefix("SPARK-40280: filter pushdown -") + .excludeByPrefix("filter pushdown -") + .exclude("Filters should be pushed down for vectorized Parquet reader at row group level") + .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") + .exclude("Filters should be pushed down for Parquet readers at row group level") + .exclude("SPARK-23852: Broken Parquet push-down for partially-written stats") + .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down") + .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode") + .exclude("Support Parquet column index") + .exclude("SPARK-34562: Bloom filter push down") + enableSuite[AuronParquetV1PartitionDiscoverySuite] + .exclude("read partitioned table - partition key included in Parquet file") + .exclude("read partitioned table - with nulls and partition keys are included in Parquet file") + .exclude("SPARK-18108 Parquet reader fails when data column types conflict with partition ones") + .exclude("SPARK-21463: MetadataLogFileIndex should respect userSpecifiedSchema for partition cols") + enableSuite[AuronParquetV1QuerySuite] + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude("Enabling/disabling ignoreCorruptFiles") + .exclude("SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("SPARK-34212 Parquet should read decimals correctly") + .exclude("returning batch for wide table") + .exclude("SPARK-39833: pushed filters with count()") + .exclude("SPARK-39833: pushed filters with project without filter columns") + enableSuite[AuronParquetV1SchemaPruningSuite] + enableSuite[AuronParquetV2FilterSuite] + .excludeByPrefix("SPARK-40280: filter pushdown -") + .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") + .exclude("Filters should be pushed down for Parquet readers at row group level") + .exclude("SPARK-23852: Broken Parquet push-down for partially-written stats") + .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down") + .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode") + .exclude("Support Parquet column index") + enableSuite[AuronParquetV2PartitionDiscoverySuite] + enableSuite[AuronParquetV2QuerySuite] + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude("SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("returning batch for wide table") + enableSuite[AuronParquetV2SchemaPruningSuite] + override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings { override def getResourceFilePath: String = "" override def getSupportedSQLQueryTests: Set[String] = Set.empty diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala new file mode 100644 index 000000000..aba4a3714 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetAvroCompatibilitySuite + extends ParquetAvroCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala new file mode 100644 index 000000000..a25f44a08 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetColumnIndexSuite extends ParquetColumnIndexSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala new file mode 100644 index 000000000..a049cf2b6 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetCompatibilityTest extends ParquetCompatibilityTest with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala new file mode 100644 index 000000000..ffa5355d6 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetCompressionCodecPrecedenceSuite + extends ParquetCompressionCodecPrecedenceSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala new file mode 100644 index 000000000..379d2d122 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetEncodingSuite extends ParquetEncodingSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala new file mode 100644 index 000000000..6f20c5547 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatSuite extends ParquetFileFormatSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala new file mode 100644 index 000000000..2e8dc6e83 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatV1Suite extends ParquetFileFormatV1Suite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala new file mode 100644 index 000000000..850a4d8b8 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatV2Suite extends ParquetFileFormatV2Suite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala new file mode 100644 index 000000000..96fb78238 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetIOSuite extends ParquetIOSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala new file mode 100644 index 000000000..e25558ca5 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetInteroperabilitySuite + extends ParquetInteroperabilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala new file mode 100644 index 000000000..af37a69c0 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetPartitionDiscoverySuite + extends ParquetPartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala new file mode 100644 index 000000000..83ef15c77 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetProtobufCompatibilitySuite + extends ParquetProtobufCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala new file mode 100644 index 000000000..40fa45239 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetQuerySuite extends ParquetQuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala new file mode 100644 index 000000000..948187df2 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeSuite + extends ParquetRebaseDatetimeSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala new file mode 100644 index 000000000..ebe02e058 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeV1Suite + extends ParquetRebaseDatetimeV1Suite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala new file mode 100644 index 000000000..a5b6b6f5a --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeV2Suite + extends ParquetRebaseDatetimeV2Suite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala new file mode 100644 index 000000000..6be422dde --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetSchemaInferenceSuite + extends ParquetSchemaInferenceSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala new file mode 100644 index 000000000..301c12eb5 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetSchemaPruningSuite extends ParquetSchemaPruningSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala new file mode 100644 index 000000000..ada8eb92f --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetSchemaSuite extends ParquetSchemaSuite with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala new file mode 100644 index 000000000..cc937309c --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetTest extends ParquetTest with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala new file mode 100644 index 000000000..0f9ef6a44 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetThriftCompatibilitySuite + extends ParquetThriftCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala new file mode 100644 index 000000000..8e4638841 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1FilterSuite extends ParquetV1FilterSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala new file mode 100644 index 000000000..aa28b253e --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1PartitionDiscoverySuite + extends ParquetV1PartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala new file mode 100644 index 000000000..a0d750850 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1QuerySuite extends ParquetV1QuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala new file mode 100644 index 000000000..6aecd6666 --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1SchemaPruningSuite + extends ParquetV1SchemaPruningSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala new file mode 100644 index 000000000..05029c1da --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2FilterSuite extends ParquetV2FilterSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala new file mode 100644 index 000000000..677498f0b --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2PartitionDiscoverySuite + extends ParquetV2PartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala new file mode 100644 index 000000000..f8abbceca --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2QuerySuite extends ParquetV2QuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala new file mode 100644 index 000000000..30b1f47db --- /dev/null +++ b/auron-spark-tests/spark32/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2SchemaPruningSuite + extends ParquetV2SchemaPruningSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala index 963325121..cecc5ea43 100644 --- a/auron-spark-tests/spark34/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala @@ -17,6 +17,7 @@ package org.apache.auron.utils import org.apache.spark.sql._ +import org.apache.spark.sql.execution.datasources.parquet._ class AuronSparkTestSettings extends SparkTestSettings { { @@ -91,6 +92,129 @@ class AuronSparkTestSettings extends SparkTestSettings { .exclude("SPARK-20897: cached self-join should not fail") .exclude("SPARK-22271: mean overflows and returns null for some decimal variables") .exclude("SPARK-32764: -0.0 and 0.0 should be equal") + .exclude("SPARK-41048: Improve output partitioning and ordering with AQE cache") + + enableSuite[AuronParquetAvroCompatibilitySuite] + enableSuite[AuronParquetColumnIndexSuite] + enableSuite[AuronParquetCompatibilityTest] + enableSuite[AuronParquetCompressionCodecPrecedenceSuite] + enableSuite[AuronParquetEncodingSuite] + enableSuite[AuronParquetFieldIdIOSuite] + .exclude("Parquet reads infer fields using field ids correctly") + .exclude("absence of field ids") + .exclude("SPARK-38094: absence of field ids: reading nested schema") + .exclude("multiple id matches") + .exclude("read parquet file without ids") + .exclude("global read/write flag should work correctly") + enableSuite[AuronParquetFieldIdSchemaSuite] + enableSuite[AuronParquetFileFormatSuite] + enableSuite[AuronParquetFileFormatV1Suite] + enableSuite[AuronParquetFileFormatV2Suite] + enableSuite[AuronParquetIOSuite] + .exclude("vectorized reader: missing all struct fields") + .exclude("SPARK-34817: Read UINT_64 as Decimal from parquet") + .exclude("SPARK-35640: read binary as timestamp should throw schema incompatible error") + .exclude("SPARK-35640: int as long should throw schema incompatible error") + .exclude("read dictionary encoded decimals written as INT32") + .exclude("read dictionary encoded decimals written as INT64") + .exclude("read dictionary encoded decimals written as FIXED_LEN_BYTE_ARRAY") + .exclude("read dictionary and plain encoded timestamp_millis written as INT64") + .exclude("SPARK-40128 read DELTA_LENGTH_BYTE_ARRAY encoded strings") + .exclude("SPARK-36726: test incorrect Parquet row group file offset") + .exclude("SPARK-34167: read LongDecimals with precision < 10, VectorizedReader true") + .exclude("SPARK-34167: read LongDecimals with precision < 10, VectorizedReader false") + enableSuite[AuronParquetInteroperabilitySuite] + .exclude("parquet timestamp conversion") + enableSuite[AuronParquetPartitionDiscoverySuite] + enableSuite[AuronParquetProtobufCompatibilitySuite] + .exclude("unannotated array of primitive type") + .exclude("unannotated array of struct") + .exclude("struct with unannotated array") + .exclude("unannotated array of struct with unannotated array") + .exclude("unannotated array of string") + enableSuite[AuronParquetQuerySuite] + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude("Enabling/disabling ignoreCorruptFiles") + .exclude( + "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("SPARK-34212 Parquet should read decimals correctly") + .exclude("row group skipping doesn't overflow when reading into larger type") + enableSuite[AuronParquetRebaseDatetimeSuite] + .exclude( + "SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps") + .exclude("SPARK-31159, SPARK-37705: rebasing timestamps in write") + .exclude("SPARK-31159: rebasing dates in write") + .exclude("SPARK-35427: datetime rebasing in the EXCEPTION mode") + enableSuite[AuronParquetRebaseDatetimeV1Suite] + .exclude( + "SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps") + .exclude("SPARK-31159, SPARK-37705: rebasing timestamps in write") + .exclude("SPARK-31159: rebasing dates in write") + .exclude("SPARK-35427: datetime rebasing in the EXCEPTION mode") + enableSuite[AuronParquetRebaseDatetimeV2Suite] + .exclude( + "SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps") + .exclude("SPARK-35427: datetime rebasing in the EXCEPTION mode") + enableSuite[AuronParquetSchemaInferenceSuite] + enableSuite[AuronParquetSchemaPruningSuite] + enableSuite[AuronParquetSchemaSuite] + .exclude("schema mismatch failure error message for parquet reader") + .exclude("schema mismatch failure error message for parquet vectorized reader") + .exclude("SPARK-40819: parquet file with TIMESTAMP(NANOS, true) (with nanosAsLong=true)") + .exclude( + "SPARK-40819: parquet file with TIMESTAMP(NANOS, true) (with default nanosAsLong=false)") + enableSuite[AuronParquetTest] + enableSuite[AuronParquetThriftCompatibilitySuite] + .exclude("Read Parquet file generated by parquet-thrift") + enableSuite[AuronParquetV1FilterSuite] + .excludeByPrefix("SPARK-40280: filter pushdown -") + .excludeByPrefix("filter pushdown -") + .exclude("Filters should be pushed down for vectorized Parquet reader at row group level") + .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") + .exclude("Filters should be pushed down for Parquet readers at row group level") + .exclude("SPARK-23852: Broken Parquet push-down for partially-written stats") + .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down") + .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode") + .exclude("Support Parquet column index") + .exclude("SPARK-34562: Bloom filter push down") + enableSuite[AuronParquetV1PartitionDiscoverySuite] + .exclude("read partitioned table - partition key included in Parquet file") + .exclude( + "read partitioned table - with nulls and partition keys are included in Parquet file") + .exclude( + "SPARK-18108 Parquet reader fails when data column types conflict with partition ones") + .exclude( + "SPARK-21463: MetadataLogFileIndex should respect userSpecifiedSchema for partition cols") + enableSuite[AuronParquetV1QuerySuite] + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude("Enabling/disabling ignoreCorruptFiles") + .exclude( + "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("SPARK-34212 Parquet should read decimals correctly") + .exclude("row group skipping doesn't overflow when reading into larger type") + .exclude("returning batch for wide table") + .exclude("SPARK-39833: pushed filters with count()") + .exclude("SPARK-39833: pushed filters with project without filter columns") + enableSuite[AuronParquetV1SchemaPruningSuite] + enableSuite[AuronParquetV2FilterSuite] + .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") + .exclude("Filters should be pushed down for Parquet readers at row group level") + .exclude("SPARK-23852: Broken Parquet push-down for partially-written stats") + .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down") + .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode") + .excludeByPrefix("SPARK-40280: filter pushdown -") + .exclude("Support Parquet column index") + enableSuite[AuronParquetV2PartitionDiscoverySuite] + enableSuite[AuronParquetV2QuerySuite] + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude( + "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("returning batch for wide table") + enableSuite[AuronParquetV2SchemaPruningSuite] + enableSuite[AuronParquetVectorizedSuite] override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings { override def getResourceFilePath: String = "" diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala new file mode 100644 index 000000000..aba4a3714 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetAvroCompatibilitySuite + extends ParquetAvroCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala new file mode 100644 index 000000000..a25f44a08 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetColumnIndexSuite extends ParquetColumnIndexSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala new file mode 100644 index 000000000..a049cf2b6 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetCompatibilityTest extends ParquetCompatibilityTest with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala new file mode 100644 index 000000000..ffa5355d6 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetCompressionCodecPrecedenceSuite + extends ParquetCompressionCodecPrecedenceSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala new file mode 100644 index 000000000..379d2d122 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetEncodingSuite extends ParquetEncodingSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdIOSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdIOSuite.scala new file mode 100644 index 000000000..f8f20ba91 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdIOSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFieldIdIOSuite extends ParquetFieldIdIOSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdSchemaSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdSchemaSuite.scala new file mode 100644 index 000000000..288384bb6 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdSchemaSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetFieldIdSchemaSuite + extends ParquetFieldIdSchemaSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala new file mode 100644 index 000000000..6f20c5547 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatSuite extends ParquetFileFormatSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala new file mode 100644 index 000000000..2e8dc6e83 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatV1Suite extends ParquetFileFormatV1Suite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala new file mode 100644 index 000000000..850a4d8b8 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatV2Suite extends ParquetFileFormatV2Suite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala new file mode 100644 index 000000000..96fb78238 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetIOSuite extends ParquetIOSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala new file mode 100644 index 000000000..e25558ca5 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetInteroperabilitySuite + extends ParquetInteroperabilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala new file mode 100644 index 000000000..af37a69c0 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetPartitionDiscoverySuite + extends ParquetPartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala new file mode 100644 index 000000000..83ef15c77 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetProtobufCompatibilitySuite + extends ParquetProtobufCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala new file mode 100644 index 000000000..40fa45239 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetQuerySuite extends ParquetQuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala new file mode 100644 index 000000000..948187df2 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeSuite + extends ParquetRebaseDatetimeSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala new file mode 100644 index 000000000..ebe02e058 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeV1Suite + extends ParquetRebaseDatetimeV1Suite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala new file mode 100644 index 000000000..a5b6b6f5a --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeV2Suite + extends ParquetRebaseDatetimeV2Suite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala new file mode 100644 index 000000000..6be422dde --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetSchemaInferenceSuite + extends ParquetSchemaInferenceSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala new file mode 100644 index 000000000..301c12eb5 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetSchemaPruningSuite extends ParquetSchemaPruningSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala new file mode 100644 index 000000000..ada8eb92f --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetSchemaSuite extends ParquetSchemaSuite with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala new file mode 100644 index 000000000..cc937309c --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetTest extends ParquetTest with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala new file mode 100644 index 000000000..0f9ef6a44 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetThriftCompatibilitySuite + extends ParquetThriftCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala new file mode 100644 index 000000000..8e4638841 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1FilterSuite extends ParquetV1FilterSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala new file mode 100644 index 000000000..aa28b253e --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1PartitionDiscoverySuite + extends ParquetV1PartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala new file mode 100644 index 000000000..a0d750850 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1QuerySuite extends ParquetV1QuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala new file mode 100644 index 000000000..6aecd6666 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1SchemaPruningSuite + extends ParquetV1SchemaPruningSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala new file mode 100644 index 000000000..05029c1da --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2FilterSuite extends ParquetV2FilterSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala new file mode 100644 index 000000000..677498f0b --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2PartitionDiscoverySuite + extends ParquetV2PartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala new file mode 100644 index 000000000..f8abbceca --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2QuerySuite extends ParquetV2QuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala new file mode 100644 index 000000000..30b1f47db --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2SchemaPruningSuite + extends ParquetV2SchemaPruningSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetVectorizedSuite.scala b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetVectorizedSuite.scala new file mode 100644 index 000000000..f3a18e976 --- /dev/null +++ b/auron-spark-tests/spark34/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetVectorizedSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetVectorizedSuite extends ParquetVectorizedSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala index f98a0ee9d..7d0b47e8f 100644 --- a/auron-spark-tests/spark35/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala @@ -17,6 +17,7 @@ package org.apache.auron.utils import org.apache.spark.sql._ +import org.apache.spark.sql.execution.datasources.parquet._ class AuronSparkTestSettings extends SparkTestSettings { { @@ -74,6 +75,8 @@ class AuronSparkTestSettings extends SparkTestSettings { "SPARK-19471: AggregationIterator does not initialize the generated result projection before using it") .exclude( "SPARK-24788: RelationalGroupedDataset.toString with unresolved exprs should not fail") + .exclude("SPARK-16484: hll_*_agg + hll_union negative tests") + .exclude("SPARK-43876: Enable fast hashmap for distinct queries") enableSuite[AuronDatasetAggregatorSuite] @@ -93,6 +96,130 @@ class AuronSparkTestSettings extends SparkTestSettings { .exclude("SPARK-20897: cached self-join should not fail") .exclude("SPARK-22271: mean overflows and returns null for some decimal variables") .exclude("SPARK-32764: -0.0 and 0.0 should be equal") + .exclude("SPARK-41048: Improve output partitioning and ordering with AQE cache") + + enableSuite[AuronParquetAvroCompatibilitySuite] + enableSuite[AuronParquetColumnIndexSuite] + enableSuite[AuronParquetCompatibilityTest] + enableSuite[AuronParquetCompressionCodecPrecedenceSuite] + enableSuite[AuronParquetEncodingSuite] + enableSuite[AuronParquetFieldIdIOSuite] + .exclude("Parquet reads infer fields using field ids correctly") + .exclude("absence of field ids") + .exclude("SPARK-38094: absence of field ids: reading nested schema") + .exclude("multiple id matches") + .exclude("read parquet file without ids") + .exclude("global read/write flag should work correctly") + enableSuite[AuronParquetFieldIdSchemaSuite] + enableSuite[AuronParquetFileFormatSuite] + enableSuite[AuronParquetFileFormatV1Suite] + enableSuite[AuronParquetFileFormatV2Suite] + enableSuite[AuronParquetIOSuite] + .exclude("vectorized reader: missing all struct fields") + .exclude("SPARK-34817: Read UINT_64 as Decimal from parquet") + .exclude("SPARK-35640: read binary as timestamp should throw schema incompatible error") + .exclude("SPARK-35640: int as long should throw schema incompatible error") + .exclude("read dictionary encoded decimals written as INT32") + .exclude("explode nested lists crossing a rowgroup boundary") + .exclude("read dictionary encoded decimals written as INT64") + .exclude("read dictionary encoded decimals written as FIXED_LEN_BYTE_ARRAY") + .exclude("read dictionary and plain encoded timestamp_millis written as INT64") + .exclude("SPARK-40128 read DELTA_LENGTH_BYTE_ARRAY encoded strings") + .exclude("SPARK-36726: test incorrect Parquet row group file offset") + .exclude("SPARK-34167: read LongDecimals with precision < 10, VectorizedReader true") + .exclude("SPARK-34167: read LongDecimals with precision < 10, VectorizedReader false") + enableSuite[AuronParquetInteroperabilitySuite] + .exclude("parquet timestamp conversion") + enableSuite[AuronParquetPartitionDiscoverySuite] + enableSuite[AuronParquetProtobufCompatibilitySuite] + .exclude("unannotated array of primitive type") + .exclude("unannotated array of struct") + .exclude("struct with unannotated array") + .exclude("unannotated array of struct with unannotated array") + .exclude("unannotated array of string") + enableSuite[AuronParquetQuerySuite] + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude("Enabling/disabling ignoreCorruptFiles") + .exclude( + "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("SPARK-34212 Parquet should read decimals correctly") + .exclude("row group skipping doesn't overflow when reading into larger type") + enableSuite[AuronParquetRebaseDatetimeSuite] + .exclude( + "SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps") + .exclude("SPARK-31159, SPARK-37705: rebasing timestamps in write") + .exclude("SPARK-31159: rebasing dates in write") + .exclude("SPARK-35427: datetime rebasing in the EXCEPTION mode") + enableSuite[AuronParquetRebaseDatetimeV1Suite] + .exclude( + "SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps") + .exclude("SPARK-31159, SPARK-37705: rebasing timestamps in write") + .exclude("SPARK-31159: rebasing dates in write") + .exclude("SPARK-35427: datetime rebasing in the EXCEPTION mode") + enableSuite[AuronParquetRebaseDatetimeV2Suite] + .exclude( + "SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps") + .exclude("SPARK-35427: datetime rebasing in the EXCEPTION mode") + enableSuite[AuronParquetSchemaInferenceSuite] + enableSuite[AuronParquetSchemaPruningSuite] + enableSuite[AuronParquetSchemaSuite] + .exclude("schema mismatch failure error message for parquet reader") + .exclude("schema mismatch failure error message for parquet vectorized reader") + .exclude("SPARK-40819: parquet file with TIMESTAMP(NANOS, true) (with nanosAsLong=true)") + .exclude( + "SPARK-40819: parquet file with TIMESTAMP(NANOS, true) (with default nanosAsLong=false)") + enableSuite[AuronParquetTest] + enableSuite[AuronParquetThriftCompatibilitySuite] + .exclude("Read Parquet file generated by parquet-thrift") + enableSuite[AuronParquetV1FilterSuite] + .excludeByPrefix("SPARK-40280: filter pushdown -") + .excludeByPrefix("filter pushdown -") + .exclude("Filters should be pushed down for vectorized Parquet reader at row group level") + .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") + .exclude("Filters should be pushed down for Parquet readers at row group level") + .exclude("SPARK-23852: Broken Parquet push-down for partially-written stats") + .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down") + .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode") + .exclude("Support Parquet column index") + .exclude("SPARK-34562: Bloom filter push down") + enableSuite[AuronParquetV1PartitionDiscoverySuite] + .exclude("read partitioned table - partition key included in Parquet file") + .exclude( + "read partitioned table - with nulls and partition keys are included in Parquet file") + .exclude( + "SPARK-18108 Parquet reader fails when data column types conflict with partition ones") + .exclude( + "SPARK-21463: MetadataLogFileIndex should respect userSpecifiedSchema for partition cols") + enableSuite[AuronParquetV1QuerySuite] + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude("Enabling/disabling ignoreCorruptFiles") + .exclude( + "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("SPARK-34212 Parquet should read decimals correctly") + .exclude("row group skipping doesn't overflow when reading into larger type") + .exclude("returning batch for wide table") + .exclude("SPARK-39833: pushed filters with count()") + .exclude("SPARK-39833: pushed filters with project without filter columns") + enableSuite[AuronParquetV1SchemaPruningSuite] + enableSuite[AuronParquetV2FilterSuite] + .excludeByPrefix("SPARK-40280: filter pushdown -") + .exclude("SPARK-31026: Parquet predicate pushdown for fields having dots in the names") + .exclude("Filters should be pushed down for Parquet readers at row group level") + .exclude("SPARK-23852: Broken Parquet push-down for partially-written stats") + .exclude("SPARK-17091: Convert IN predicate to Parquet filter push-down") + .exclude("SPARK-25207: exception when duplicate fields in case-insensitive mode") + .exclude("Support Parquet column index") + enableSuite[AuronParquetV2PartitionDiscoverySuite] + enableSuite[AuronParquetV2QuerySuite] + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude( + "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("returning batch for wide table") + enableSuite[AuronParquetV2SchemaPruningSuite] + enableSuite[AuronParquetVectorizedSuite] override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings { override def getResourceFilePath: String = "" diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala new file mode 100644 index 000000000..aba4a3714 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetAvroCompatibilitySuite + extends ParquetAvroCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala new file mode 100644 index 000000000..a25f44a08 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetColumnIndexSuite extends ParquetColumnIndexSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala new file mode 100644 index 000000000..a049cf2b6 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetCompatibilityTest extends ParquetCompatibilityTest with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala new file mode 100644 index 000000000..ffa5355d6 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetCompressionCodecPrecedenceSuite + extends ParquetCompressionCodecPrecedenceSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala new file mode 100644 index 000000000..379d2d122 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetEncodingSuite extends ParquetEncodingSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdIOSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdIOSuite.scala new file mode 100644 index 000000000..f8f20ba91 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdIOSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFieldIdIOSuite extends ParquetFieldIdIOSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdSchemaSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdSchemaSuite.scala new file mode 100644 index 000000000..288384bb6 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdSchemaSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetFieldIdSchemaSuite + extends ParquetFieldIdSchemaSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala new file mode 100644 index 000000000..6f20c5547 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatSuite extends ParquetFileFormatSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala new file mode 100644 index 000000000..2e8dc6e83 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatV1Suite extends ParquetFileFormatV1Suite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala new file mode 100644 index 000000000..850a4d8b8 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatV2Suite extends ParquetFileFormatV2Suite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala new file mode 100644 index 000000000..96fb78238 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetIOSuite extends ParquetIOSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala new file mode 100644 index 000000000..e25558ca5 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetInteroperabilitySuite + extends ParquetInteroperabilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala new file mode 100644 index 000000000..af37a69c0 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetPartitionDiscoverySuite + extends ParquetPartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala new file mode 100644 index 000000000..83ef15c77 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetProtobufCompatibilitySuite + extends ParquetProtobufCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala new file mode 100644 index 000000000..40fa45239 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetQuerySuite extends ParquetQuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala new file mode 100644 index 000000000..948187df2 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeSuite + extends ParquetRebaseDatetimeSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala new file mode 100644 index 000000000..ebe02e058 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeV1Suite + extends ParquetRebaseDatetimeV1Suite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala new file mode 100644 index 000000000..a5b6b6f5a --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeV2Suite + extends ParquetRebaseDatetimeV2Suite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala new file mode 100644 index 000000000..6be422dde --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetSchemaInferenceSuite + extends ParquetSchemaInferenceSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala new file mode 100644 index 000000000..301c12eb5 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetSchemaPruningSuite extends ParquetSchemaPruningSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala new file mode 100644 index 000000000..ada8eb92f --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetSchemaSuite extends ParquetSchemaSuite with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala new file mode 100644 index 000000000..cc937309c --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetTest extends ParquetTest with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala new file mode 100644 index 000000000..0f9ef6a44 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetThriftCompatibilitySuite + extends ParquetThriftCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala new file mode 100644 index 000000000..8e4638841 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1FilterSuite extends ParquetV1FilterSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala new file mode 100644 index 000000000..aa28b253e --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1PartitionDiscoverySuite + extends ParquetV1PartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala new file mode 100644 index 000000000..a0d750850 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1QuerySuite extends ParquetV1QuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala new file mode 100644 index 000000000..6aecd6666 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1SchemaPruningSuite + extends ParquetV1SchemaPruningSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala new file mode 100644 index 000000000..05029c1da --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2FilterSuite extends ParquetV2FilterSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala new file mode 100644 index 000000000..677498f0b --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2PartitionDiscoverySuite + extends ParquetV2PartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala new file mode 100644 index 000000000..f8abbceca --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2QuerySuite extends ParquetV2QuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala new file mode 100644 index 000000000..30b1f47db --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2SchemaPruningSuite + extends ParquetV2SchemaPruningSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetVectorizedSuite.scala b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetVectorizedSuite.scala new file mode 100644 index 000000000..f3a18e976 --- /dev/null +++ b/auron-spark-tests/spark35/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetVectorizedSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetVectorizedSuite extends ParquetVectorizedSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala index 4bb49b98c..677d0c042 100644 --- a/auron-spark-tests/spark40/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala @@ -17,6 +17,7 @@ package org.apache.auron.utils import org.apache.spark.sql._ +import org.apache.spark.sql.execution.datasources.parquet._ class AuronSparkTestSettings extends SparkTestSettings { { @@ -25,18 +26,7 @@ class AuronSparkTestSettings extends SparkTestSettings { } enableSuite[AuronDataFrameFunctionsSuite] - // Native execution wraps SparkRuntimeException from validation failures in SparkException. - .exclude("map with arrays") - .exclude("map_concat function") - .exclude("SPARK-24734: Fix containsNull of Concat for array type") - .exclude("array_insert functions") - .exclude("transform keys function - Invalid lambda functions and exceptions") - // Native reverse only supports string inputs; array inputs fail with unsupported List type. - .exclude("reverse function - array for primitive type not containing null") - .exclude("reverse function - array for primitive type containing null") - .exclude("reverse function - array for non-primitive type") - // Native flatten can panic when child arrays have different containsNull metadata. - .exclude("flatten function") + .disable("Native execution can crash after ParquetQuery in Spark 4") enableSuite[AuronDateFunctionsSuite] // Native execution wraps Spark parsing/format validation exceptions in SparkException. @@ -50,49 +40,144 @@ class AuronSparkTestSettings extends SparkTestSettings { // Native date_trunc may produce incorrect results for historical timestamps with // non-UTC timezones due to timezone handling differences in the DataFusion engine. .exclude("SPARK-30766: date_trunc of old timestamps to hours and days") + .exclude("SPARK-30668: use legacy timestamp parser in to_timestamp") enableSuite[AuronMathFunctionsSuite] - // Native acosh uses a different floating-point formula than Spark's StrictMath.log, - // producing results that differ at the last ULP for certain edge-case inputs. - .exclude("acosh") + .disable("Native execution can crash in Spark 4") enableSuite[AuronMiscFunctionsSuite] + .exclude("reflect and java_method") enableSuite[AuronStringFunctionsSuite] + .exclude("string concat") + .exclude("string concat_ws") // Spark 4 adds the threshold argument, but native levenshtein currently supports only // two arguments. .exclude("string Levenshtein distance") // Native substr does not support BinaryType inputs. .exclude("string / binary substring function") + .exclude("UTF-8 string validate") + .exclude("RegExpReplace throws the right exception when replace fails on a particular row") enableSuite[AuronDataFrameAggregateSuite] - // See https://github.com/apache/auron/issues/1840 - .excludeByPrefix("collect functions") - // A custom version of the SPARK-19471 test has been added to AuronDataFrameAggregateSuite - // with modified plan checks for Auron's native aggregates, so we exclude the original here. - .exclude( - "SPARK-19471: AggregationIterator does not initialize the generated result projection before using it") - .exclude( - "SPARK-24788: RelationalGroupedDataset.toString with unresolved exprs should not fail") + .disable("Native execution can crash in Spark 4") enableSuite[AuronDatasetAggregatorSuite] + .disable("Native dataset aggregators fail in Spark 4") enableSuite[AuronTypedImperativeAggregateSuite] + .disable("Native execution can crash after ParquetQuery in Spark 4") enableSuite[AuronDataFrameSuite] - // Auron-specific implementations of these tests are provided above - .exclude("repartitionByRange") - .exclude("distributeBy and localSort") - .exclude("reuse exchange") - .exclude("SPARK-22520: support code generation for large CaseWhen") - .exclude("SPARK-27439: Explain result should match collected result after view change") - // These tests fail due to Auron native execution differences - .exclude("SPARK-28067: Aggregate sum should not return wrong results for decimal overflow") - .exclude("SPARK-35955: Aggregate avg should not return wrong results for decimal overflow") - .exclude("NaN is greater than all other non-NaN numeric values") - .exclude("SPARK-20897: cached self-join should not fail") - .exclude("SPARK-22271: mean overflows and returns null for some decimal variables") - .exclude("SPARK-32764: -0.0 and 0.0 should be equal") + .disable("Native execution can crash in Spark 4") + + enableSuite[AuronParquetAvroCompatibilitySuite] + .exclude("required primitives") + .exclude("optional primitives") + .exclude("non-nullable arrays") + .exclude("SPARK-10136 array of primitive array") + .exclude("map of primitive array") + .exclude("various complex types") + .exclude("SPARK-9407 Push down predicates involving Parquet ENUM columns") + enableSuite[AuronParquetColumnIndexSuite] + .exclude("reading from unaligned pages - test filters") + .exclude("test reading unaligned pages - test all types (dict encode)") + .exclude("SPARK-36123: reading from unaligned pages - test filters with nulls") + .exclude("test reading unaligned pages - test all types") + .exclude("reading unaligned pages - struct type") + enableSuite[AuronParquetCompatibilityTest] + enableSuite[AuronParquetCompressionCodecPrecedenceSuite] + enableSuite[AuronParquetEncodingSuite] + .disable("Native execution can crash in Spark 4") + enableSuite[AuronParquetFieldIdIOSuite] + .disable("Native parquet field id reads fail in Spark 4") + enableSuite[AuronParquetFieldIdSchemaSuite] + enableSuite[AuronParquetFileFormatSuite] + enableSuite[AuronParquetFileFormatV1Suite] + enableSuite[AuronParquetFileFormatV2Suite] + enableSuite[AuronParquetIOSuite] + .disable("Native execution can crash in Spark 4") + enableSuite[AuronParquetInteroperabilitySuite] + .disable("Native execution can crash in Spark 4") + enableSuite[AuronParquetPartitionDiscoverySuite] + .exclude("read partitioned table - normal case") + .exclude("Resolve type conflicts - decimals, dates and timestamps in partition column") + enableSuite[AuronParquetProtobufCompatibilitySuite] + .exclude("unannotated array of primitive type") + .exclude("unannotated array of struct") + .exclude("struct with unannotated array") + .exclude("unannotated array of struct with unannotated array") + .exclude("unannotated array of string") + enableSuite[AuronParquetQuerySuite] + .exclude("simple select queries") + .exclude("appending") + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude("Enabling/disabling ignoreCorruptFiles") + .exclude( + "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("SPARK-34212 Parquet should read decimals correctly") + enableSuite[AuronParquetRebaseDatetimeSuite] + .exclude( + "SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps") + .exclude("SPARK-31159, SPARK-37705: rebasing timestamps in write") + .exclude("SPARK-31159: rebasing dates in write") + .exclude("SPARK-35427: datetime rebasing in the EXCEPTION mode") + enableSuite[AuronParquetRebaseDatetimeV1Suite] + .disable("Spark 4 test resources use jar paths unsupported by Hadoop Path") + enableSuite[AuronParquetRebaseDatetimeV2Suite] + .disable("Spark 4 test resources use jar paths unsupported by Hadoop Path") + enableSuite[AuronParquetSchemaInferenceSuite] + enableSuite[AuronParquetSchemaPruningSuite] + .disable("Native parquet schema pruning reads fail in Spark 4") + enableSuite[AuronParquetSchemaSuite] + .disable("Native execution can crash in Spark 4") + enableSuite[AuronParquetTest] + enableSuite[AuronParquetThriftCompatibilitySuite] + .disable("Spark 4 test resources use jar paths unsupported by Hadoop Path") + enableSuite[AuronParquetV1FilterSuite] + .disable("Native execution can crash in Spark 4") + enableSuite[AuronParquetV1PartitionDiscoverySuite] + .exclude("read partitioned table - normal case") + .exclude("read partitioned table - partition key included in Parquet file") + .exclude( + "read partitioned table - with nulls and partition keys are included in Parquet file") + .exclude( + "SPARK-18108 Parquet reader fails when data column types conflict with partition ones") + .exclude( + "SPARK-21463: MetadataLogFileIndex should respect userSpecifiedSchema for partition cols") + enableSuite[AuronParquetV1QuerySuite] + .exclude("simple select queries") + .exclude("appending") + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude("Enabling/disabling ignoreCorruptFiles") + .exclude( + "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("SPARK-34212 Parquet should read decimals correctly") + .exclude("returning batch for wide table") + .exclude("SPARK-39833: pushed filters with count()") + .exclude("SPARK-39833: pushed filters with project without filter columns") + enableSuite[AuronParquetV1SchemaPruningSuite] + .disable("Native parquet schema pruning reads fail in Spark 4") + enableSuite[AuronParquetV2FilterSuite] + .disable("Native execution can crash in Spark 4") + enableSuite[AuronParquetV2PartitionDiscoverySuite] + .exclude("read partitioned table - normal case") + .exclude( + "SPARK-22109: Resolve type conflicts between strings and timestamps in partition column") + enableSuite[AuronParquetV2QuerySuite] + .exclude("simple select queries") + .exclude("appending") + .exclude("self-join") + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude( + "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("returning batch for wide table") + enableSuite[AuronParquetV2SchemaPruningSuite] + .disable("Native parquet schema pruning reads fail in Spark 4") + enableSuite[AuronParquetVectorizedSuite] override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings { override def getResourceFilePath: String = "" diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala new file mode 100644 index 000000000..aba4a3714 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetAvroCompatibilitySuite + extends ParquetAvroCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala new file mode 100644 index 000000000..a25f44a08 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetColumnIndexSuite extends ParquetColumnIndexSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala new file mode 100644 index 000000000..a049cf2b6 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetCompatibilityTest extends ParquetCompatibilityTest with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala new file mode 100644 index 000000000..ffa5355d6 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetCompressionCodecPrecedenceSuite + extends ParquetCompressionCodecPrecedenceSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala new file mode 100644 index 000000000..379d2d122 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetEncodingSuite extends ParquetEncodingSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdIOSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdIOSuite.scala new file mode 100644 index 000000000..f8f20ba91 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdIOSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFieldIdIOSuite extends ParquetFieldIdIOSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdSchemaSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdSchemaSuite.scala new file mode 100644 index 000000000..288384bb6 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdSchemaSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetFieldIdSchemaSuite + extends ParquetFieldIdSchemaSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala new file mode 100644 index 000000000..6f20c5547 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatSuite extends ParquetFileFormatSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala new file mode 100644 index 000000000..2e8dc6e83 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatV1Suite extends ParquetFileFormatV1Suite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala new file mode 100644 index 000000000..850a4d8b8 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatV2Suite extends ParquetFileFormatV2Suite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala new file mode 100644 index 000000000..96fb78238 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetIOSuite extends ParquetIOSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala new file mode 100644 index 000000000..e25558ca5 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetInteroperabilitySuite + extends ParquetInteroperabilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala new file mode 100644 index 000000000..af37a69c0 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetPartitionDiscoverySuite + extends ParquetPartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala new file mode 100644 index 000000000..83ef15c77 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetProtobufCompatibilitySuite + extends ParquetProtobufCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala new file mode 100644 index 000000000..40fa45239 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetQuerySuite extends ParquetQuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala new file mode 100644 index 000000000..948187df2 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeSuite + extends ParquetRebaseDatetimeSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala new file mode 100644 index 000000000..ebe02e058 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeV1Suite + extends ParquetRebaseDatetimeV1Suite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala new file mode 100644 index 000000000..a5b6b6f5a --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeV2Suite + extends ParquetRebaseDatetimeV2Suite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala new file mode 100644 index 000000000..6be422dde --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetSchemaInferenceSuite + extends ParquetSchemaInferenceSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala new file mode 100644 index 000000000..301c12eb5 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetSchemaPruningSuite extends ParquetSchemaPruningSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala new file mode 100644 index 000000000..ada8eb92f --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetSchemaSuite extends ParquetSchemaSuite with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala new file mode 100644 index 000000000..cc937309c --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetTest extends ParquetTest with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala new file mode 100644 index 000000000..0f9ef6a44 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetThriftCompatibilitySuite + extends ParquetThriftCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala new file mode 100644 index 000000000..8e4638841 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1FilterSuite extends ParquetV1FilterSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala new file mode 100644 index 000000000..aa28b253e --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1PartitionDiscoverySuite + extends ParquetV1PartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala new file mode 100644 index 000000000..a0d750850 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1QuerySuite extends ParquetV1QuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala new file mode 100644 index 000000000..6aecd6666 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1SchemaPruningSuite + extends ParquetV1SchemaPruningSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala new file mode 100644 index 000000000..05029c1da --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2FilterSuite extends ParquetV2FilterSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala new file mode 100644 index 000000000..677498f0b --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2PartitionDiscoverySuite + extends ParquetV2PartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala new file mode 100644 index 000000000..f8abbceca --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2QuerySuite extends ParquetV2QuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala new file mode 100644 index 000000000..30b1f47db --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2SchemaPruningSuite + extends ParquetV2SchemaPruningSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetVectorizedSuite.scala b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetVectorizedSuite.scala new file mode 100644 index 000000000..f3a18e976 --- /dev/null +++ b/auron-spark-tests/spark40/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetVectorizedSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetVectorizedSuite extends ParquetVectorizedSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala index 4bb49b98c..8f370f096 100644 --- a/auron-spark-tests/spark41/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/auron/utils/AuronSparkTestSettings.scala @@ -17,6 +17,7 @@ package org.apache.auron.utils import org.apache.spark.sql._ +import org.apache.spark.sql.execution.datasources.parquet._ class AuronSparkTestSettings extends SparkTestSettings { { @@ -25,18 +26,7 @@ class AuronSparkTestSettings extends SparkTestSettings { } enableSuite[AuronDataFrameFunctionsSuite] - // Native execution wraps SparkRuntimeException from validation failures in SparkException. - .exclude("map with arrays") - .exclude("map_concat function") - .exclude("SPARK-24734: Fix containsNull of Concat for array type") - .exclude("array_insert functions") - .exclude("transform keys function - Invalid lambda functions and exceptions") - // Native reverse only supports string inputs; array inputs fail with unsupported List type. - .exclude("reverse function - array for primitive type not containing null") - .exclude("reverse function - array for primitive type containing null") - .exclude("reverse function - array for non-primitive type") - // Native flatten can panic when child arrays have different containsNull metadata. - .exclude("flatten function") + .disable("Native execution can crash after ParquetQuery in Spark 4") enableSuite[AuronDateFunctionsSuite] // Native execution wraps Spark parsing/format validation exceptions in SparkException. @@ -50,49 +40,154 @@ class AuronSparkTestSettings extends SparkTestSettings { // Native date_trunc may produce incorrect results for historical timestamps with // non-UTC timezones due to timezone handling differences in the DataFusion engine. .exclude("SPARK-30766: date_trunc of old timestamps to hours and days") + .exclude("SPARK-30668: use legacy timestamp parser in to_timestamp") enableSuite[AuronMathFunctionsSuite] - // Native acosh uses a different floating-point formula than Spark's StrictMath.log, - // producing results that differ at the last ULP for certain edge-case inputs. - .exclude("acosh") + .disable("Native execution can crash in Spark 4") enableSuite[AuronMiscFunctionsSuite] + .exclude("reflect and java_method") enableSuite[AuronStringFunctionsSuite] + .exclude("string concat") + .exclude("string concat_ws") // Spark 4 adds the threshold argument, but native levenshtein currently supports only // two arguments. .exclude("string Levenshtein distance") // Native substr does not support BinaryType inputs. .exclude("string / binary substring function") + .exclude("UTF-8 string validate") + .exclude("RegExpReplace throws the right exception when replace fails on a particular row") enableSuite[AuronDataFrameAggregateSuite] - // See https://github.com/apache/auron/issues/1840 - .excludeByPrefix("collect functions") - // A custom version of the SPARK-19471 test has been added to AuronDataFrameAggregateSuite - // with modified plan checks for Auron's native aggregates, so we exclude the original here. - .exclude( - "SPARK-19471: AggregationIterator does not initialize the generated result projection before using it") - .exclude( - "SPARK-24788: RelationalGroupedDataset.toString with unresolved exprs should not fail") + .disable("Native execution can crash in Spark 4") enableSuite[AuronDatasetAggregatorSuite] + .disable("Native dataset aggregators fail in Spark 4") enableSuite[AuronTypedImperativeAggregateSuite] + .disable("Native execution can crash after ParquetQuery in Spark 4") enableSuite[AuronDataFrameSuite] - // Auron-specific implementations of these tests are provided above - .exclude("repartitionByRange") - .exclude("distributeBy and localSort") - .exclude("reuse exchange") - .exclude("SPARK-22520: support code generation for large CaseWhen") - .exclude("SPARK-27439: Explain result should match collected result after view change") - // These tests fail due to Auron native execution differences - .exclude("SPARK-28067: Aggregate sum should not return wrong results for decimal overflow") - .exclude("SPARK-35955: Aggregate avg should not return wrong results for decimal overflow") - .exclude("NaN is greater than all other non-NaN numeric values") - .exclude("SPARK-20897: cached self-join should not fail") - .exclude("SPARK-22271: mean overflows and returns null for some decimal variables") - .exclude("SPARK-32764: -0.0 and 0.0 should be equal") + .disable("Native execution can crash in Spark 4") + + enableSuite[AuronParquetAvroCompatibilitySuite] + .exclude("required primitives") + .exclude("optional primitives") + .exclude("non-nullable arrays") + .exclude("SPARK-10136 array of primitive array") + .exclude("map of primitive array") + .exclude("various complex types") + .exclude("SPARK-9407 Push down predicates involving Parquet ENUM columns") + enableSuite[AuronParquetColumnIndexSuite] + .exclude("reading from unaligned pages - test filters") + .exclude("test reading unaligned pages - test all types (dict encode)") + .exclude("SPARK-36123: reading from unaligned pages - test filters with nulls") + .exclude("test reading unaligned pages - test all types") + .exclude("reading unaligned pages - struct type") + enableSuite[AuronParquetCompatibilityTest] + enableSuite[AuronParquetCompressionCodecPrecedenceSuite] + enableSuite[AuronParquetEncodingSuite] + .disable("Native execution can crash in Spark 4") + enableSuite[AuronParquetFieldIdIOSuite] + .disable("Native parquet field id reads fail in Spark 4") + enableSuite[AuronParquetFieldIdSchemaSuite] + enableSuite[AuronParquetFileFormatSuite] + .exclude("Write and read back TIME values") + enableSuite[AuronParquetFileFormatV1Suite] + .exclude("Write and read back TIME values") + enableSuite[AuronParquetFileFormatV2Suite] + .exclude("Write and read back TIME values") + enableSuite[AuronParquetIOSuite] + .disable("Native execution can crash in Spark 4") + enableSuite[AuronParquetInteroperabilitySuite] + .disable("Native execution can crash in Spark 4") + enableSuite[AuronParquetPartitionDiscoverySuite] + .exclude("read partitioned table - normal case") + .exclude("Infer the TIME data type from partition values") + enableSuite[AuronParquetProtobufCompatibilitySuite] + .exclude("unannotated array of primitive type") + .exclude("unannotated array of struct") + .exclude("struct with unannotated array") + .exclude("unannotated array of struct with unannotated array") + .exclude("unannotated array of string") + enableSuite[AuronParquetQuerySuite] + .exclude("simple select queries") + .exclude("appending") + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude("Enabling/disabling ignoreCorruptFiles") + .exclude( + "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("SPARK-34212 Parquet should read decimals correctly") + .exclude("create table with TIME") + enableSuite[AuronParquetRebaseDatetimeSuite] + .exclude( + "SPARK-31159, SPARK-37705: compatibility with Spark 2.4/3.2 in reading dates/timestamps") + .exclude("SPARK-31159, SPARK-37705: rebasing timestamps in write") + .exclude("SPARK-31159: rebasing dates in write") + .exclude("SPARK-35427: datetime rebasing in the EXCEPTION mode") + enableSuite[AuronParquetRebaseDatetimeV1Suite] + .disable("Spark 4 test resources use jar paths unsupported by Hadoop Path") + enableSuite[AuronParquetRebaseDatetimeV2Suite] + .disable("Spark 4 test resources use jar paths unsupported by Hadoop Path") + enableSuite[AuronParquetSchemaInferenceSuite] + enableSuite[AuronParquetSchemaPruningSuite] + .disable("Native parquet schema pruning reads fail in Spark 4") + enableSuite[AuronParquetSchemaSuite] + .disable("Native execution can crash in Spark 4") + enableSuite[AuronParquetTest] + enableSuite[AuronParquetThriftCompatibilitySuite] + .disable("Spark 4 test resources use jar paths unsupported by Hadoop Path") + enableSuite[AuronParquetV1FilterSuite] + .disable("Native execution can crash in Spark 4") + enableSuite[AuronParquetV1PartitionDiscoverySuite] + .exclude("read partitioned table - normal case") + .exclude("Infer the TIME data type from partition values") + .exclude("read partitioned table - partition key included in Parquet file") + .exclude( + "read partitioned table - with nulls and partition keys are included in Parquet file") + .exclude( + "SPARK-18108 Parquet reader fails when data column types conflict with partition ones") + .exclude( + "SPARK-21463: MetadataLogFileIndex should respect userSpecifiedSchema for partition cols") + enableSuite[AuronParquetV1QuerySuite] + .exclude("simple select queries") + .exclude("appending") + .exclude("create table with TIME") + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude("Enabling/disabling ignoreCorruptFiles") + .exclude( + "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("SPARK-34212 Parquet should read decimals correctly") + .exclude("returning batch for wide table") + .exclude("SPARK-39833: pushed filters with count()") + .exclude("SPARK-39833: pushed filters with project without filter columns") + enableSuite[AuronParquetV1SchemaPruningSuite] + .disable("Native parquet schema pruning reads fail in Spark 4") + enableSuite[AuronParquetV2FilterSuite] + .disable("Native execution can crash in Spark 4") + enableSuite[AuronParquetV2PartitionDiscoverySuite] + .exclude("read partitioned table - normal case") + .exclude("Infer the TIME data type from partition values") + .exclude("_SUCCESS should not break partitioning discovery") + .exclude("Resolve type conflicts - decimals, dates and timestamps in partition column") + .exclude( + "SPARK-22109: Resolve type conflicts between strings and timestamps in partition column") + enableSuite[AuronParquetV2QuerySuite] + .exclude("simple select queries") + .exclude("appending") + .exclude("self-join") + .exclude("create table with TIME") + .exclude("SPARK-10634 timestamp written and read as INT64 - truncation") + .exclude( + "SPARK-26677: negated null-safe equality comparison should not filter matched row groups") + .exclude("Migration from INT96 to TIMESTAMP_MICROS timestamp type") + .exclude("returning batch for wide table") + enableSuite[AuronParquetV2SchemaPruningSuite] + .disable("Native parquet schema pruning reads fail in Spark 4") + enableSuite[AuronParquetVectorizedSuite] override def getSQLQueryTestSettings: SQLQueryTestSettings = new SQLQueryTestSettings { override def getResourceFilePath: String = "" diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala new file mode 100644 index 000000000..aba4a3714 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetAvroCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetAvroCompatibilitySuite + extends ParquetAvroCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala new file mode 100644 index 000000000..a25f44a08 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetColumnIndexSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetColumnIndexSuite extends ParquetColumnIndexSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala new file mode 100644 index 000000000..a049cf2b6 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompatibilityTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetCompatibilityTest extends ParquetCompatibilityTest with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala new file mode 100644 index 000000000..ffa5355d6 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetCompressionCodecPrecedenceSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetCompressionCodecPrecedenceSuite + extends ParquetCompressionCodecPrecedenceSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala new file mode 100644 index 000000000..379d2d122 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetEncodingSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetEncodingSuite extends ParquetEncodingSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdIOSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdIOSuite.scala new file mode 100644 index 000000000..f8f20ba91 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdIOSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFieldIdIOSuite extends ParquetFieldIdIOSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdSchemaSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdSchemaSuite.scala new file mode 100644 index 000000000..288384bb6 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFieldIdSchemaSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetFieldIdSchemaSuite + extends ParquetFieldIdSchemaSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala new file mode 100644 index 000000000..6f20c5547 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatSuite extends ParquetFileFormatSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala new file mode 100644 index 000000000..2e8dc6e83 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV1Suite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatV1Suite extends ParquetFileFormatV1Suite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala new file mode 100644 index 000000000..850a4d8b8 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetFileFormatV2Suite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetFileFormatV2Suite extends ParquetFileFormatV2Suite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala new file mode 100644 index 000000000..96fb78238 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetIOSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetIOSuite extends ParquetIOSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala new file mode 100644 index 000000000..e25558ca5 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetInteroperabilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetInteroperabilitySuite + extends ParquetInteroperabilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala new file mode 100644 index 000000000..af37a69c0 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetPartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetPartitionDiscoverySuite + extends ParquetPartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala new file mode 100644 index 000000000..83ef15c77 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetProtobufCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetProtobufCompatibilitySuite + extends ParquetProtobufCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala new file mode 100644 index 000000000..40fa45239 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetQuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetQuerySuite extends ParquetQuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala new file mode 100644 index 000000000..948187df2 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeSuite + extends ParquetRebaseDatetimeSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala new file mode 100644 index 000000000..ebe02e058 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV1Suite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeV1Suite + extends ParquetRebaseDatetimeV1Suite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala new file mode 100644 index 000000000..a5b6b6f5a --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetRebaseDatetimeV2Suite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetRebaseDatetimeV2Suite + extends ParquetRebaseDatetimeV2Suite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala new file mode 100644 index 000000000..6be422dde --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaInferenceSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetSchemaInferenceSuite + extends ParquetSchemaInferenceSuite + with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala new file mode 100644 index 000000000..301c12eb5 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaPruningSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetSchemaPruningSuite extends ParquetSchemaPruningSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala new file mode 100644 index 000000000..ada8eb92f --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetSchemaSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetSchemaSuite extends ParquetSchemaSuite with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala new file mode 100644 index 000000000..cc937309c --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetTest.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkTestsSharedSessionBase + +class AuronParquetTest extends ParquetTest with SparkTestsSharedSessionBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala new file mode 100644 index 000000000..0f9ef6a44 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetThriftCompatibilitySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetThriftCompatibilitySuite + extends ParquetThriftCompatibilitySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala new file mode 100644 index 000000000..8e4638841 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1FilterSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1FilterSuite extends ParquetV1FilterSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala new file mode 100644 index 000000000..aa28b253e --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1PartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1PartitionDiscoverySuite + extends ParquetV1PartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala new file mode 100644 index 000000000..a0d750850 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1QuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1QuerySuite extends ParquetV1QuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala new file mode 100644 index 000000000..6aecd6666 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV1SchemaPruningSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV1SchemaPruningSuite + extends ParquetV1SchemaPruningSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala new file mode 100644 index 000000000..05029c1da --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2FilterSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2FilterSuite extends ParquetV2FilterSuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala new file mode 100644 index 000000000..677498f0b --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2PartitionDiscoverySuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2PartitionDiscoverySuite + extends ParquetV2PartitionDiscoverySuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala new file mode 100644 index 000000000..f8abbceca --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2QuerySuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2QuerySuite extends ParquetV2QuerySuite with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala new file mode 100644 index 000000000..30b1f47db --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetV2SchemaPruningSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetV2SchemaPruningSuite + extends ParquetV2SchemaPruningSuite + with SparkQueryTestsBase {} diff --git a/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetVectorizedSuite.scala b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetVectorizedSuite.scala new file mode 100644 index 000000000..f3a18e976 --- /dev/null +++ b/auron-spark-tests/spark41/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/AuronParquetVectorizedSuite.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.datasources.parquet + +import org.apache.spark.sql.SparkQueryTestsBase + +class AuronParquetVectorizedSuite extends ParquetVectorizedSuite with SparkQueryTestsBase {} diff --git a/pom.xml b/pom.xml index 15fe04721..b5f4b6a76 100644 --- a/pom.xml +++ b/pom.xml @@ -52,6 +52,7 @@ 1.14.11 3.4.3 3.25.5 + 1.16.0 4.2.7.Final 8 ${javaVersion} @@ -812,6 +813,7 @@ spark-3.0 3.0.8 3.0.3 + 1.10.1 3.0 4.1.47.Final @@ -849,6 +851,7 @@ spark-3.1 3.2.9 3.1.3 + 1.10.1 3.1 4.1.51.Final @@ -886,6 +889,7 @@ spark-3.2 3.2.9 3.2.4 + 1.12.2 3.2 4.1.68.Final @@ -923,6 +927,7 @@ spark-3.3 3.2.9 3.3.4 + 1.12.2 3.3 4.1.74.Final @@ -960,6 +965,7 @@ spark-3.4 3.2.9 3.4.4 + 1.12.3 3.4 4.1.87.Final 1.10.1 @@ -998,6 +1004,7 @@ spark-3.5 3.2.9 3.5.8 + 1.13.1 3.5 4.1.96.Final 1.10.1 @@ -1036,6 +1043,7 @@ spark-4.0 3.2.9 4.0.2 + 1.15.2 4.0 4.1.111.Final 1.10.1 @@ -1094,6 +1102,7 @@ spark-4.1 3.2.9 4.1.1 + 1.16.0 4.1 4.1.118.Final