| diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml | |
| index c50991c5fc..4c6db98159 100644 | |
| --- a/gradle/libs.versions.toml | |
| +++ b/gradle/libs.versions.toml | |
| @@ -36,6 +36,7 @@ awssdk-s3accessgrants = "2.3.0" | |
| bson-ver = "4.11.5" | |
| caffeine = "2.9.3" | |
| calcite = "1.39.0" | |
| +comet = "1.1.0-SNAPSHOT" | |
| datasketches = "6.2.0" | |
| delta-standalone = "3.3.1" | |
| delta-spark = "3.3.1" | |
| diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle | |
| index 2cd8666892..4077116abb 100644 | |
| --- a/spark/v3.4/build.gradle | |
| +++ b/spark/v3.4/build.gradle | |
| @@ -75,7 +75,7 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") { | |
| exclude group: 'org.roaringbitmap' | |
| } | |
| - compileOnly "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:0.5.0" | |
| + compileOnly "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" | |
| implementation libs.parquet.column | |
| implementation libs.parquet.hadoop | |
| @@ -186,7 +186,7 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer | |
| testImplementation libs.parquet.hadoop | |
| testImplementation libs.awaitility | |
| testImplementation libs.junit.vintage.engine | |
| - testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:0.5.0" | |
| + testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" | |
| // Required because we remove antlr plugin dependencies from the compile configuration, see note above | |
| runtimeOnly libs.antlr.runtime | |
| @@ -267,6 +267,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio | |
| integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') | |
| integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts') | |
| integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts') | |
| + integrationImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" | |
| // runtime dependencies for running Hive Catalog based integration test | |
| integrationRuntimeOnly project(':iceberg-hive-metastore') | |
| diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java | |
| index 578845e3da..19a200892f 100644 | |
| --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java | |
| +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java | |
| @@ -57,7 +57,25 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { | |
| .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") | |
| .config( | |
| SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| TestBase.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java | |
| index 58d054bd05..ef1c1cb6d1 100644 | |
| --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java | |
| +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java | |
| @@ -56,6 +56,15 @@ public class TestCallStatementParser { | |
| .master("local[2]") | |
| .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) | |
| .config("spark.extra.prop", "value") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| TestCallStatementParser.parser = spark.sessionState().sqlParser(); | |
| } | |
| diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java | |
| index 5a0a04edba..afe92a8b72 100644 | |
| --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java | |
| +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java | |
| @@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; | |
| import org.apache.spark.sql.catalyst.expressions.Expression; | |
| import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; | |
| import org.apache.spark.sql.execution.CommandResultExec; | |
| +import org.apache.spark.sql.execution.SparkPlan; | |
| import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; | |
| import org.junit.jupiter.api.AfterEach; | |
| import org.junit.jupiter.api.BeforeEach; | |
| @@ -313,9 +314,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe | |
| private List<Expression> executeAndCollectFunctionCalls(String query, Object... args) { | |
| CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); | |
| - V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); | |
| + // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not | |
| + // a V2TableWriteExec; collect over the whole command plan in that case. | |
| + SparkPlan plan = command.commandPhysicalPlan(); | |
| + SparkPlan queryPlan = | |
| + plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; | |
| return SparkPlanUtil.collectExprs( | |
| - write.query(), | |
| + queryPlan, | |
| expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); | |
| } | |
| diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java | |
| index b6ade2bff3..90a4839dad 100644 | |
| --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java | |
| +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java | |
| @@ -170,6 +170,15 @@ public class DeleteOrphanFilesBenchmark { | |
| .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) | |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") | |
| .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .master("local"); | |
| spark = builder.getOrCreate(); | |
| } | |
| diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java | |
| index b08c352819..63880b089d 100644 | |
| --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java | |
| +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java | |
| @@ -386,6 +386,15 @@ public class IcebergSortCompactionBenchmark { | |
| "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") | |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") | |
| .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .master("local[*]"); | |
| spark = builder.getOrCreate(); | |
| Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); | |
| diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java | |
| index 68c537e34a..2e421c7706 100644 | |
| --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java | |
| +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java | |
| @@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { | |
| } | |
| protected void setupSpark(boolean enableDictionaryEncoding) { | |
| - SparkSession.Builder builder = SparkSession.builder().config("spark.ui.enabled", false); | |
| + SparkSession.Builder builder = | |
| + SparkSession.builder() | |
| + .config("spark.ui.enabled", false) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g"); | |
| if (!enableDictionaryEncoding) { | |
| builder | |
| .config("parquet.dictionary.page.size", "1") | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java | |
| deleted file mode 100644 | |
| index 16159dcbdf..0000000000 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java | |
| +++ /dev/null | |
| @@ -1,140 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.io.IOException; | |
| -import org.apache.comet.parquet.AbstractColumnReader; | |
| -import org.apache.comet.parquet.ColumnReader; | |
| -import org.apache.comet.parquet.TypeUtil; | |
| -import org.apache.comet.parquet.Utils; | |
| -import org.apache.comet.shaded.arrow.c.CometSchemaImporter; | |
| -import org.apache.comet.shaded.arrow.memory.RootAllocator; | |
| -import org.apache.iceberg.parquet.VectorizedReader; | |
| -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; | |
| -import org.apache.iceberg.spark.SparkSchemaUtil; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.parquet.column.ColumnDescriptor; | |
| -import org.apache.parquet.column.page.PageReader; | |
| -import org.apache.spark.sql.types.DataType; | |
| -import org.apache.spark.sql.types.Metadata; | |
| -import org.apache.spark.sql.types.StructField; | |
| -import org.apache.spark.sql.vectorized.ColumnVector; | |
| - | |
| -class CometColumnReader implements VectorizedReader<ColumnVector> { | |
| - // use the Comet default batch size | |
| - public static final int DEFAULT_BATCH_SIZE = 8192; | |
| - | |
| - private final ColumnDescriptor descriptor; | |
| - private final DataType sparkType; | |
| - | |
| - // The delegated ColumnReader from Comet side | |
| - private AbstractColumnReader delegate; | |
| - private boolean initialized = false; | |
| - private int batchSize = DEFAULT_BATCH_SIZE; | |
| - private CometSchemaImporter importer; | |
| - | |
| - CometColumnReader(DataType sparkType, ColumnDescriptor descriptor) { | |
| - this.sparkType = sparkType; | |
| - this.descriptor = descriptor; | |
| - } | |
| - | |
| - CometColumnReader(Types.NestedField field) { | |
| - DataType dataType = SparkSchemaUtil.convert(field.type()); | |
| - StructField structField = new StructField(field.name(), dataType, false, Metadata.empty()); | |
| - this.sparkType = dataType; | |
| - this.descriptor = TypeUtil.convertToParquet(structField); | |
| - } | |
| - | |
| - public AbstractColumnReader delegate() { | |
| - return delegate; | |
| - } | |
| - | |
| - void setDelegate(AbstractColumnReader delegate) { | |
| - this.delegate = delegate; | |
| - } | |
| - | |
| - void setInitialized(boolean initialized) { | |
| - this.initialized = initialized; | |
| - } | |
| - | |
| - public int batchSize() { | |
| - return batchSize; | |
| - } | |
| - | |
| - /** | |
| - * This method is to initialized/reset the CometColumnReader. This needs to be called for each row | |
| - * group after readNextRowGroup, so a new dictionary encoding can be set for each of the new row | |
| - * groups. | |
| - */ | |
| - public void reset() { | |
| - if (importer != null) { | |
| - importer.close(); | |
| - } | |
| - | |
| - if (delegate != null) { | |
| - delegate.close(); | |
| - } | |
| - | |
| - this.importer = new CometSchemaImporter(new RootAllocator()); | |
| - this.delegate = Utils.getColumnReader(sparkType, descriptor, importer, batchSize, false, false); | |
| - this.initialized = true; | |
| - } | |
| - | |
| - public ColumnDescriptor descriptor() { | |
| - return descriptor; | |
| - } | |
| - | |
| - /** Returns the Spark data type for this column. */ | |
| - public DataType sparkType() { | |
| - return sparkType; | |
| - } | |
| - | |
| - /** | |
| - * Set the page reader to be 'pageReader'. | |
| - * | |
| - * <p>NOTE: this should be called before reading a new Parquet column chunk, and after {@link | |
| - * CometColumnReader#reset} is called. | |
| - */ | |
| - public void setPageReader(PageReader pageReader) throws IOException { | |
| - Preconditions.checkState(initialized, "Invalid state: 'reset' should be called first"); | |
| - ((ColumnReader) delegate).setPageReader(pageReader); | |
| - } | |
| - | |
| - @Override | |
| - public void close() { | |
| - // close resources on native side | |
| - if (importer != null) { | |
| - importer.close(); | |
| - } | |
| - | |
| - if (delegate != null) { | |
| - delegate.close(); | |
| - } | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int size) { | |
| - this.batchSize = size; | |
| - } | |
| - | |
| - @Override | |
| - public ColumnVector read(ColumnVector reuse, int numRowsToRead) { | |
| - throw new UnsupportedOperationException("Not supported"); | |
| - } | |
| -} | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java | |
| deleted file mode 100644 | |
| index 04ac69476a..0000000000 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java | |
| +++ /dev/null | |
| @@ -1,197 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.io.IOException; | |
| -import java.io.UncheckedIOException; | |
| -import java.util.List; | |
| -import java.util.Map; | |
| -import org.apache.comet.parquet.AbstractColumnReader; | |
| -import org.apache.comet.parquet.BatchReader; | |
| -import org.apache.iceberg.Schema; | |
| -import org.apache.iceberg.data.DeleteFilter; | |
| -import org.apache.iceberg.parquet.VectorizedReader; | |
| -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; | |
| -import org.apache.iceberg.spark.SparkSchemaUtil; | |
| -import org.apache.iceberg.util.Pair; | |
| -import org.apache.parquet.column.page.PageReadStore; | |
| -import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; | |
| -import org.apache.parquet.hadoop.metadata.ColumnPath; | |
| -import org.apache.spark.sql.catalyst.InternalRow; | |
| -import org.apache.spark.sql.vectorized.ColumnVector; | |
| -import org.apache.spark.sql.vectorized.ColumnarBatch; | |
| - | |
| -/** | |
| - * {@link VectorizedReader} that returns Spark's {@link ColumnarBatch} to support Spark's vectorized | |
| - * read path. The {@link ColumnarBatch} returned is created by passing in the Arrow vectors | |
| - * populated via delegated read calls to {@link CometColumnReader VectorReader(s)}. | |
| - */ | |
| -@SuppressWarnings("checkstyle:VisibilityModifier") | |
| -class CometColumnarBatchReader implements VectorizedReader<ColumnarBatch> { | |
| - | |
| - private final CometColumnReader[] readers; | |
| - private final boolean hasIsDeletedColumn; | |
| - | |
| - // The delegated BatchReader on the Comet side does the real work of loading a batch of rows. | |
| - // The Comet BatchReader contains an array of ColumnReader. There is no need to explicitly call | |
| - // ColumnReader.readBatch; instead, BatchReader.nextBatch will be called, which underneath calls | |
| - // ColumnReader.readBatch. The only exception is DeleteColumnReader, because at the time of | |
| - // calling BatchReader.nextBatch, the isDeleted value is not yet available, so | |
| - // DeleteColumnReader.readBatch must be called explicitly later, after the isDeleted value is | |
| - // available. | |
| - private final BatchReader delegate; | |
| - private DeleteFilter<InternalRow> deletes = null; | |
| - private long rowStartPosInBatch = 0; | |
| - | |
| - CometColumnarBatchReader(List<VectorizedReader<?>> readers, Schema schema) { | |
| - this.readers = | |
| - readers.stream().map(CometColumnReader.class::cast).toArray(CometColumnReader[]::new); | |
| - this.hasIsDeletedColumn = | |
| - readers.stream().anyMatch(reader -> reader instanceof CometDeleteColumnReader); | |
| - | |
| - AbstractColumnReader[] abstractColumnReaders = new AbstractColumnReader[readers.size()]; | |
| - this.delegate = new BatchReader(abstractColumnReaders); | |
| - delegate.setSparkSchema(SparkSchemaUtil.convert(schema)); | |
| - } | |
| - | |
| - @Override | |
| - public void setRowGroupInfo( | |
| - PageReadStore pageStore, Map<ColumnPath, ColumnChunkMetaData> metaData) { | |
| - for (int i = 0; i < readers.length; i++) { | |
| - try { | |
| - if (!(readers[i] instanceof CometConstantColumnReader) | |
| - && !(readers[i] instanceof CometPositionColumnReader) | |
| - && !(readers[i] instanceof CometDeleteColumnReader)) { | |
| - readers[i].reset(); | |
| - readers[i].setPageReader(pageStore.getPageReader(readers[i].descriptor())); | |
| - } | |
| - } catch (IOException e) { | |
| - throw new UncheckedIOException("Failed to setRowGroupInfo for Comet vectorization", e); | |
| - } | |
| - } | |
| - | |
| - for (int i = 0; i < readers.length; i++) { | |
| - delegate.getColumnReaders()[i] = this.readers[i].delegate(); | |
| - } | |
| - | |
| - this.rowStartPosInBatch = | |
| - pageStore | |
| - .getRowIndexOffset() | |
| - .orElseThrow( | |
| - () -> | |
| - new IllegalArgumentException( | |
| - "PageReadStore does not contain row index offset")); | |
| - } | |
| - | |
| - public void setDeleteFilter(DeleteFilter<InternalRow> deleteFilter) { | |
| - this.deletes = deleteFilter; | |
| - } | |
| - | |
| - @Override | |
| - public final ColumnarBatch read(ColumnarBatch reuse, int numRowsToRead) { | |
| - ColumnarBatch columnarBatch = new ColumnBatchLoader(numRowsToRead).loadDataToColumnBatch(); | |
| - rowStartPosInBatch += numRowsToRead; | |
| - return columnarBatch; | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - for (CometColumnReader reader : readers) { | |
| - if (reader != null) { | |
| - reader.setBatchSize(batchSize); | |
| - } | |
| - } | |
| - } | |
| - | |
| - @Override | |
| - public void close() { | |
| - for (CometColumnReader reader : readers) { | |
| - if (reader != null) { | |
| - reader.close(); | |
| - } | |
| - } | |
| - } | |
| - | |
| - private class ColumnBatchLoader { | |
| - private final int batchSize; | |
| - | |
| - ColumnBatchLoader(int numRowsToRead) { | |
| - Preconditions.checkArgument( | |
| - numRowsToRead > 0, "Invalid number of rows to read: %s", numRowsToRead); | |
| - this.batchSize = numRowsToRead; | |
| - } | |
| - | |
| - ColumnarBatch loadDataToColumnBatch() { | |
| - ColumnVector[] vectors = readDataToColumnVectors(); | |
| - int numLiveRows = batchSize; | |
| - | |
| - if (hasIsDeletedColumn) { | |
| - boolean[] isDeleted = buildIsDeleted(vectors); | |
| - readDeletedColumn(vectors, isDeleted); | |
| - } else { | |
| - Pair<int[], Integer> pair = buildRowIdMapping(vectors); | |
| - if (pair != null) { | |
| - int[] rowIdMapping = pair.first(); | |
| - numLiveRows = pair.second(); | |
| - for (int i = 0; i < vectors.length; i++) { | |
| - vectors[i] = new ColumnVectorWithFilter(vectors[i], rowIdMapping); | |
| - } | |
| - } | |
| - } | |
| - | |
| - if (deletes != null && deletes.hasEqDeletes()) { | |
| - vectors = ColumnarBatchUtil.removeExtraColumns(deletes, vectors); | |
| - } | |
| - | |
| - ColumnarBatch batch = new ColumnarBatch(vectors); | |
| - batch.setNumRows(numLiveRows); | |
| - return batch; | |
| - } | |
| - | |
| - private boolean[] buildIsDeleted(ColumnVector[] vectors) { | |
| - return ColumnarBatchUtil.buildIsDeleted(vectors, deletes, rowStartPosInBatch, batchSize); | |
| - } | |
| - | |
| - private Pair<int[], Integer> buildRowIdMapping(ColumnVector[] vectors) { | |
| - return ColumnarBatchUtil.buildRowIdMapping(vectors, deletes, rowStartPosInBatch, batchSize); | |
| - } | |
| - | |
| - ColumnVector[] readDataToColumnVectors() { | |
| - ColumnVector[] columnVectors = new ColumnVector[readers.length]; | |
| - // Fetch rows for all readers in the delegate | |
| - delegate.nextBatch(batchSize); | |
| - for (int i = 0; i < readers.length; i++) { | |
| - columnVectors[i] = readers[i].delegate().currentBatch(); | |
| - } | |
| - | |
| - return columnVectors; | |
| - } | |
| - | |
| - void readDeletedColumn(ColumnVector[] columnVectors, boolean[] isDeleted) { | |
| - for (int i = 0; i < readers.length; i++) { | |
| - if (readers[i] instanceof CometDeleteColumnReader) { | |
| - CometDeleteColumnReader deleteColumnReader = new CometDeleteColumnReader<>(isDeleted); | |
| - deleteColumnReader.setBatchSize(batchSize); | |
| - deleteColumnReader.delegate().readBatch(batchSize); | |
| - columnVectors[i] = deleteColumnReader.delegate().currentBatch(); | |
| - } | |
| - } | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java | |
| deleted file mode 100644 | |
| index c665002e8f..0000000000 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java | |
| +++ /dev/null | |
| @@ -1,65 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.math.BigDecimal; | |
| -import java.nio.ByteBuffer; | |
| -import org.apache.comet.parquet.ConstantColumnReader; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.spark.sql.types.DataType; | |
| -import org.apache.spark.sql.types.DataTypes; | |
| -import org.apache.spark.sql.types.Decimal; | |
| -import org.apache.spark.sql.types.DecimalType; | |
| -import org.apache.spark.unsafe.types.UTF8String; | |
| - | |
| -class CometConstantColumnReader<T> extends CometColumnReader { | |
| - | |
| - CometConstantColumnReader(T value, Types.NestedField field) { | |
| - super(field); | |
| - // use delegate to set constant value on the native side to be consumed by native execution. | |
| - setDelegate( | |
| - new ConstantColumnReader(sparkType(), descriptor(), convertToSparkValue(value), false)); | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - super.setBatchSize(batchSize); | |
| - delegate().setBatchSize(batchSize); | |
| - setInitialized(true); | |
| - } | |
| - | |
| - private Object convertToSparkValue(T value) { | |
| - DataType dataType = sparkType(); | |
| - // Match the value to Spark internal type if necessary | |
| - if (dataType == DataTypes.StringType && value instanceof String) { | |
| - // the internal type for StringType is UTF8String | |
| - return UTF8String.fromString((String) value); | |
| - } else if (dataType instanceof DecimalType && value instanceof BigDecimal) { | |
| - // the internal type for DecimalType is Decimal | |
| - return Decimal.apply((BigDecimal) value); | |
| - } else if (dataType == DataTypes.BinaryType && value instanceof ByteBuffer) { | |
| - // the internal type for DecimalType is byte[] | |
| - // Iceberg default value should always use HeapBufferBuffer, so calling ByteBuffer.array() | |
| - // should be safe. | |
| - return ((java.nio.ByteBuffer) value).array(); | |
| - } else { | |
| - return value; | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java | |
| deleted file mode 100644 | |
| index 4a28fc51da..0000000000 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java | |
| +++ /dev/null | |
| @@ -1,75 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import org.apache.comet.parquet.MetadataColumnReader; | |
| -import org.apache.comet.parquet.Native; | |
| -import org.apache.comet.parquet.TypeUtil; | |
| -import org.apache.iceberg.MetadataColumns; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.spark.sql.types.DataTypes; | |
| -import org.apache.spark.sql.types.Metadata; | |
| -import org.apache.spark.sql.types.StructField; | |
| - | |
| -class CometDeleteColumnReader<T> extends CometColumnReader { | |
| - CometDeleteColumnReader(Types.NestedField field) { | |
| - super(field); | |
| - setDelegate(new DeleteColumnReader()); | |
| - } | |
| - | |
| - CometDeleteColumnReader(boolean[] isDeleted) { | |
| - super(MetadataColumns.IS_DELETED); | |
| - setDelegate(new DeleteColumnReader(isDeleted)); | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - super.setBatchSize(batchSize); | |
| - delegate().setBatchSize(batchSize); | |
| - setInitialized(true); | |
| - } | |
| - | |
| - private static class DeleteColumnReader extends MetadataColumnReader { | |
| - private boolean[] isDeleted; | |
| - | |
| - DeleteColumnReader() { | |
| - super( | |
| - DataTypes.BooleanType, | |
| - TypeUtil.convertToParquet( | |
| - new StructField("_deleted", DataTypes.BooleanType, false, Metadata.empty())), | |
| - false /* useDecimal128 = false */, | |
| - false /* isConstant */); | |
| - this.isDeleted = new boolean[0]; | |
| - } | |
| - | |
| - DeleteColumnReader(boolean[] isDeleted) { | |
| - this(); | |
| - this.isDeleted = isDeleted; | |
| - } | |
| - | |
| - @Override | |
| - public void readBatch(int total) { | |
| - Native.resetBatch(nativeHandle); | |
| - // set isDeleted on the native side to be consumed by native execution | |
| - Native.setIsDeleted(nativeHandle, isDeleted); | |
| - | |
| - super.readBatch(total); | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java | |
| deleted file mode 100644 | |
| index 1949a71798..0000000000 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java | |
| +++ /dev/null | |
| @@ -1,62 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import org.apache.comet.parquet.MetadataColumnReader; | |
| -import org.apache.comet.parquet.Native; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.parquet.column.ColumnDescriptor; | |
| -import org.apache.spark.sql.types.DataTypes; | |
| - | |
| -class CometPositionColumnReader extends CometColumnReader { | |
| - CometPositionColumnReader(Types.NestedField field) { | |
| - super(field); | |
| - setDelegate(new PositionColumnReader(descriptor())); | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - super.setBatchSize(batchSize); | |
| - delegate().setBatchSize(batchSize); | |
| - setInitialized(true); | |
| - } | |
| - | |
| - private static class PositionColumnReader extends MetadataColumnReader { | |
| - /** The current position value of the column that are used to initialize this column reader. */ | |
| - private long position; | |
| - | |
| - PositionColumnReader(ColumnDescriptor descriptor) { | |
| - super( | |
| - DataTypes.LongType, | |
| - descriptor, | |
| - false /* useDecimal128 = false */, | |
| - false /* isConstant */); | |
| - } | |
| - | |
| - @Override | |
| - public void readBatch(int total) { | |
| - Native.resetBatch(nativeHandle); | |
| - // set position on the native side to be consumed by native execution | |
| - Native.setPosition(nativeHandle, position, total); | |
| - position += total; | |
| - | |
| - super.readBatch(total); | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java | |
| deleted file mode 100644 | |
| index d36f1a7274..0000000000 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java | |
| +++ /dev/null | |
| @@ -1,147 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.util.List; | |
| -import java.util.Map; | |
| -import java.util.function.Function; | |
| -import java.util.stream.IntStream; | |
| -import org.apache.iceberg.MetadataColumns; | |
| -import org.apache.iceberg.Schema; | |
| -import org.apache.iceberg.data.DeleteFilter; | |
| -import org.apache.iceberg.parquet.TypeWithSchemaVisitor; | |
| -import org.apache.iceberg.parquet.VectorizedReader; | |
| -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; | |
| -import org.apache.iceberg.relocated.com.google.common.collect.Lists; | |
| -import org.apache.iceberg.relocated.com.google.common.collect.Maps; | |
| -import org.apache.iceberg.spark.SparkSchemaUtil; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.parquet.column.ColumnDescriptor; | |
| -import org.apache.parquet.schema.GroupType; | |
| -import org.apache.parquet.schema.MessageType; | |
| -import org.apache.parquet.schema.PrimitiveType; | |
| -import org.apache.parquet.schema.Type; | |
| -import org.apache.spark.sql.catalyst.InternalRow; | |
| - | |
| -class CometVectorizedReaderBuilder extends TypeWithSchemaVisitor<VectorizedReader<?>> { | |
| - | |
| - private final MessageType parquetSchema; | |
| - private final Schema icebergSchema; | |
| - private final Map<Integer, ?> idToConstant; | |
| - private final Function<List<VectorizedReader<?>>, VectorizedReader<?>> readerFactory; | |
| - private final DeleteFilter<InternalRow> deleteFilter; | |
| - | |
| - CometVectorizedReaderBuilder( | |
| - Schema expectedSchema, | |
| - MessageType parquetSchema, | |
| - Map<Integer, ?> idToConstant, | |
| - Function<List<VectorizedReader<?>>, VectorizedReader<?>> readerFactory, | |
| - DeleteFilter<InternalRow> deleteFilter) { | |
| - this.parquetSchema = parquetSchema; | |
| - this.icebergSchema = expectedSchema; | |
| - this.idToConstant = idToConstant; | |
| - this.readerFactory = readerFactory; | |
| - this.deleteFilter = deleteFilter; | |
| - } | |
| - | |
| - @Override | |
| - public VectorizedReader<?> message( | |
| - Types.StructType expected, MessageType message, List<VectorizedReader<?>> fieldReaders) { | |
| - GroupType groupType = message.asGroupType(); | |
| - Map<Integer, VectorizedReader<?>> readersById = Maps.newHashMap(); | |
| - List<Type> fields = groupType.getFields(); | |
| - | |
| - IntStream.range(0, fields.size()) | |
| - .filter(pos -> fields.get(pos).getId() != null) | |
| - .forEach(pos -> readersById.put(fields.get(pos).getId().intValue(), fieldReaders.get(pos))); | |
| - | |
| - List<Types.NestedField> icebergFields = | |
| - expected != null ? expected.fields() : ImmutableList.of(); | |
| - | |
| - List<VectorizedReader<?>> reorderedFields = | |
| - Lists.newArrayListWithExpectedSize(icebergFields.size()); | |
| - | |
| - for (Types.NestedField field : icebergFields) { | |
| - int id = field.fieldId(); | |
| - VectorizedReader<?> reader = readersById.get(id); | |
| - if (idToConstant.containsKey(id)) { | |
| - CometConstantColumnReader constantReader = | |
| - new CometConstantColumnReader<>(idToConstant.get(id), field); | |
| - reorderedFields.add(constantReader); | |
| - } else if (id == MetadataColumns.ROW_POSITION.fieldId()) { | |
| - reorderedFields.add(new CometPositionColumnReader(field)); | |
| - } else if (id == MetadataColumns.IS_DELETED.fieldId()) { | |
| - CometColumnReader deleteReader = new CometDeleteColumnReader<>(field); | |
| - reorderedFields.add(deleteReader); | |
| - } else if (reader != null) { | |
| - reorderedFields.add(reader); | |
| - } else if (field.initialDefault() != null) { | |
| - CometColumnReader constantReader = | |
| - new CometConstantColumnReader<>(field.initialDefault(), field); | |
| - reorderedFields.add(constantReader); | |
| - } else if (field.isOptional()) { | |
| - CometColumnReader constantReader = new CometConstantColumnReader<>(null, field); | |
| - reorderedFields.add(constantReader); | |
| - } else { | |
| - throw new IllegalArgumentException( | |
| - String.format("Missing required field: %s", field.name())); | |
| - } | |
| - } | |
| - return vectorizedReader(reorderedFields); | |
| - } | |
| - | |
| - protected VectorizedReader<?> vectorizedReader(List<VectorizedReader<?>> reorderedFields) { | |
| - VectorizedReader<?> reader = readerFactory.apply(reorderedFields); | |
| - if (deleteFilter != null) { | |
| - ((CometColumnarBatchReader) reader).setDeleteFilter(deleteFilter); | |
| - } | |
| - return reader; | |
| - } | |
| - | |
| - @Override | |
| - public VectorizedReader<?> struct( | |
| - Types.StructType expected, GroupType groupType, List<VectorizedReader<?>> fieldReaders) { | |
| - if (expected != null) { | |
| - throw new UnsupportedOperationException( | |
| - "Vectorized reads are not supported yet for struct fields"); | |
| - } | |
| - return null; | |
| - } | |
| - | |
| - @Override | |
| - public VectorizedReader<?> primitive( | |
| - org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) { | |
| - | |
| - if (primitive.getId() == null) { | |
| - return null; | |
| - } | |
| - int parquetFieldId = primitive.getId().intValue(); | |
| - ColumnDescriptor desc = parquetSchema.getColumnDescription(currentPath()); | |
| - // Nested types not yet supported for vectorized reads | |
| - if (desc.getMaxRepetitionLevel() > 0) { | |
| - return null; | |
| - } | |
| - Types.NestedField icebergField = icebergSchema.findField(parquetFieldId); | |
| - if (icebergField == null) { | |
| - return null; | |
| - } | |
| - | |
| - return new CometColumnReader(SparkSchemaUtil.convert(icebergField.type()), desc); | |
| - } | |
| -} | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java | |
| index b523bc5bff..636ad3be7d 100644 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java | |
| +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java | |
| @@ -70,23 +70,6 @@ public class VectorizedSparkParquetReaders { | |
| deleteFilter)); | |
| } | |
| - public static CometColumnarBatchReader buildCometReader( | |
| - Schema expectedSchema, | |
| - MessageType fileSchema, | |
| - Map<Integer, ?> idToConstant, | |
| - DeleteFilter<InternalRow> deleteFilter) { | |
| - return (CometColumnarBatchReader) | |
| - TypeWithSchemaVisitor.visit( | |
| - expectedSchema.asStruct(), | |
| - fileSchema, | |
| - new CometVectorizedReaderBuilder( | |
| - expectedSchema, | |
| - fileSchema, | |
| - idToConstant, | |
| - readers -> new CometColumnarBatchReader(readers, expectedSchema), | |
| - deleteFilter)); | |
| - } | |
| - | |
| // enables unsafe memory access to avoid costly checks to see if index is within bounds | |
| // as long as it is not configured explicitly (see BoundsChecking in Arrow) | |
| private static void enableUnsafeMemoryAccess() { | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java | |
| index 780e1750a5..25f253eede 100644 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java | |
| +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java | |
| @@ -34,7 +34,6 @@ import org.apache.iceberg.parquet.Parquet; | |
| import org.apache.iceberg.relocated.com.google.common.collect.Sets; | |
| import org.apache.iceberg.spark.OrcBatchReadConf; | |
| import org.apache.iceberg.spark.ParquetBatchReadConf; | |
| -import org.apache.iceberg.spark.ParquetReaderType; | |
| import org.apache.iceberg.spark.data.vectorized.VectorizedSparkOrcReaders; | |
| import org.apache.iceberg.spark.data.vectorized.VectorizedSparkParquetReaders; | |
| import org.apache.iceberg.types.TypeUtil; | |
| @@ -92,15 +91,9 @@ abstract class BaseBatchReader<T extends ScanTask> extends BaseReader<ColumnarBa | |
| .project(requiredSchema) | |
| .split(start, length) | |
| .createBatchedReaderFunc( | |
| - fileSchema -> { | |
| - if (parquetConf.readerType() == ParquetReaderType.COMET) { | |
| - return VectorizedSparkParquetReaders.buildCometReader( | |
| - requiredSchema, fileSchema, idToConstant, deleteFilter); | |
| - } else { | |
| - return VectorizedSparkParquetReaders.buildReader( | |
| - requiredSchema, fileSchema, idToConstant, deleteFilter); | |
| - } | |
| - }) | |
| + fileSchema -> | |
| + VectorizedSparkParquetReaders.buildReader( | |
| + requiredSchema, fileSchema, idToConstant, deleteFilter)) | |
| .recordsPerBatch(parquetConf.batchSize()) | |
| .filter(residual) | |
| .caseSensitive(caseSensitive()) | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java | |
| index 404ba72846..b375458f05 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java | |
| @@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase | |
| .master("local[2]") | |
| .config("spark.serializer", serializer) | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java | |
| index 9361c63176..16ce1c614f 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java | |
| @@ -69,6 +69,15 @@ public class TestSparkDistributedDataScanDeletes | |
| .master("local[2]") | |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java | |
| index a218f965ea..e30a8d6c6f 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java | |
| @@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles | |
| .master("local[2]") | |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java | |
| index acd4688440..33732e6654 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java | |
| @@ -59,6 +59,15 @@ public class TestSparkDistributedDataScanReporting | |
| .master("local[2]") | |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java | |
| index 3e8953fb95..51ce392476 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java | |
| @@ -77,6 +77,15 @@ public abstract class SparkTestBase extends SparkTestHelperBase { | |
| .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") | |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) | |
| .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| .getOrCreate(); | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java | |
| index 06d5e0c44f..effc2bcc69 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java | |
| @@ -57,7 +57,19 @@ public abstract class ScanTestBase extends AvroDataTest { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - ScanTestBase.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + ScanTestBase.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| ScanTestBase.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java | |
| index 26cb4dcc95..d185f1175b 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java | |
| @@ -104,7 +104,19 @@ public class TestCompressionSettings extends SparkCatalogTestBase { | |
| @BeforeClass | |
| public static void startSpark() { | |
| - TestCompressionSettings.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestCompressionSettings.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @Before | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java | |
| index 013b8d4386..007b1af65b 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java | |
| @@ -76,7 +76,19 @@ public class TestDataSourceOptions extends SparkTestBaseWithCatalog { | |
| @BeforeClass | |
| public static void startSpark() { | |
| - TestDataSourceOptions.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestDataSourceOptions.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterClass | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java | |
| index ba13d005bd..b9778c1986 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java | |
| @@ -118,7 +118,19 @@ public class TestFilteredScan { | |
| @BeforeClass | |
| public static void startSpark() { | |
| - TestFilteredScan.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestFilteredScan.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| // define UDFs used by partition tests | |
| Function<Object, Integer> bucket4 = Transforms.bucket(4).bind(Types.LongType.get()); | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java | |
| index 9f97753094..fb5bfbad1b 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java | |
| @@ -94,7 +94,19 @@ public class TestForwardCompatibility { | |
| @BeforeClass | |
| public static void startSpark() { | |
| - TestForwardCompatibility.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestForwardCompatibility.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterClass | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java | |
| index 0154506f86..12e916cbe4 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java | |
| @@ -46,7 +46,19 @@ public class TestIcebergSpark { | |
| @BeforeClass | |
| public static void startSpark() { | |
| - TestIcebergSpark.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestIcebergSpark.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterClass | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java | |
| index 639d37c793..5e397909d4 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java | |
| @@ -111,7 +111,19 @@ public class TestPartitionPruning { | |
| @BeforeClass | |
| public static void startSpark() { | |
| - TestPartitionPruning.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestPartitionPruning.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java | |
| index ad0984ef42..99af33c86c 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java | |
| @@ -104,7 +104,19 @@ public class TestPartitionValues { | |
| @BeforeClass | |
| public static void startSpark() { | |
| - TestPartitionValues.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestPartitionValues.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterClass | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java | |
| index 9fc576dde5..69a49f5091 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java | |
| @@ -83,7 +83,19 @@ public class TestSnapshotSelection { | |
| @BeforeClass | |
| public static void startSpark() { | |
| - TestSnapshotSelection.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestSnapshotSelection.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterClass | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | |
| index 16fde3c954..26e94f6dde 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | |
| @@ -121,7 +121,19 @@ public class TestSparkDataFile { | |
| @BeforeClass | |
| public static void startSpark() { | |
| - TestSparkDataFile.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestSparkDataFile.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java | |
| index 63c18277aa..971f14b647 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java | |
| @@ -89,7 +89,19 @@ public class TestSparkDataWrite { | |
| @BeforeClass | |
| public static void startSpark() { | |
| - TestSparkDataWrite.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestSparkDataWrite.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @Parameterized.AfterParam | |
| @@ -138,7 +150,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); | |
| Assert.assertEquals("Result rows should match", expected, actual); | |
| for (ManifestFile manifest : | |
| @@ -208,7 +220,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); | |
| Assert.assertEquals("Result rows should match", expected, actual); | |
| } | |
| @@ -254,7 +266,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); | |
| Assert.assertEquals("Result rows should match", expected, actual); | |
| } | |
| @@ -307,7 +319,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); | |
| Assert.assertEquals("Result rows should match", expected, actual); | |
| } | |
| @@ -350,7 +362,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); | |
| Assert.assertEquals("Result rows should match", expected, actual); | |
| } | |
| @@ -390,7 +402,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); | |
| Assert.assertEquals("Result rows should match", expected, actual); | |
| @@ -455,7 +467,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); | |
| Assert.assertEquals("Result rows should match", expected, actual); | |
| } | |
| @@ -619,7 +631,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); | |
| Assert.assertEquals("Result rows should match", expected, actual); | |
| @@ -705,7 +717,7 @@ public class TestSparkDataWrite { | |
| // Since write and commit succeeded, the rows should be readable | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| Assert.assertEquals( | |
| "Number of rows should match", records.size() + records2.size(), actual.size()); | |
| assertThat(actual) | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java | |
| index 584a6b1c70..b93a23d6f1 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java | |
| @@ -86,7 +86,19 @@ public class TestSparkReadProjection extends TestReadProjection { | |
| @BeforeClass | |
| public static void startSpark() { | |
| - TestSparkReadProjection.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestSparkReadProjection.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| ImmutableMap<String, String> config = | |
| ImmutableMap.of( | |
| "type", "hive", | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java | |
| index dda49b4946..69d3a0d24f 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java | |
| @@ -131,7 +131,25 @@ public class TestSparkReaderDeletes extends DeleteReadTests { | |
| .config("spark.ui.liveUpdate.period", 0) | |
| .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") | |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| catalog = | |
| @@ -199,7 +217,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { | |
| } | |
| protected boolean countDeletes() { | |
| - return true; | |
| + // TODO: Enable once iceberg-rust exposes delete count metrics to Comet | |
| + return false; | |
| } | |
| @Override | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java | |
| index e5831b76e4..2433013d3e 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java | |
| @@ -176,7 +176,25 @@ public class TestSparkReaderWithBloomFilter { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| catalog = | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java | |
| index 961d69b721..f12ac3e83b 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java | |
| @@ -68,6 +68,15 @@ public class TestStructuredStreaming { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.sql.shuffle.partitions", 4) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java | |
| index ac674e2e62..c18fed7cfb 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java | |
| @@ -73,7 +73,19 @@ public class TestTimestampWithoutZone extends SparkTestBase { | |
| @BeforeClass | |
| public static void startSpark() { | |
| - TestTimestampWithoutZone.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestTimestampWithoutZone.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterClass | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java | |
| index 73827b309b..ace66913d1 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java | |
| @@ -80,7 +80,19 @@ public class TestWriteMetricsConfig { | |
| @BeforeClass | |
| public static void startSpark() { | |
| - TestWriteMetricsConfig.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestWriteMetricsConfig.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java | |
| index 1a4e2f3e1c..047e0f1fb4 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java | |
| @@ -65,7 +65,25 @@ public class TestAggregatePushDown extends SparkCatalogTestBase { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.sql.iceberg.aggregate_pushdown", "true") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| SparkTestBase.catalog = | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java | |
| index f92055ab7a..95fde91f19 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java | |
| @@ -585,9 +585,7 @@ public class TestFilterPushDown extends SparkTestBaseWithCatalog { | |
| String planAsString = sparkPlan.toString().replaceAll("#(\\d+L?)", ""); | |
| if (sparkFilter != null) { | |
| - assertThat(planAsString) | |
| - .as("Post scan filter should match") | |
| - .contains("Filter (" + sparkFilter + ")"); | |
| + assertThat(planAsString).as("Post scan filter should match").contains("CometFilter"); | |
| } else { | |
| assertThat(planAsString).as("Should be no post scan filter").doesNotContain("Filter ("); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java | |
| index 8a1ec5060f..3611521cc4 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java | |
| @@ -605,7 +605,7 @@ public class TestStoragePartitionedJoins extends SparkTestBaseWithCatalog { | |
| + "FROM %s t1 " | |
| + "INNER JOIN %s t2 " | |
| + "ON t1.id = t2.id AND t1.%s = t2.%s " | |
| - + "ORDER BY t1.id, t1.%s", | |
| + + "ORDER BY t1.id, t1.%s, t1.salary", | |
| sourceColumnName, | |
| tableName, | |
| tableName(OTHER_TABLE_NAME), | |
| diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle | |
| index 572c32f929..29013d26c0 100644 | |
| --- a/spark/v3.5/build.gradle | |
| +++ b/spark/v3.5/build.gradle | |
| @@ -75,7 +75,7 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") { | |
| exclude group: 'org.roaringbitmap' | |
| } | |
| - compileOnly "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:0.5.0" | |
| + compileOnly "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" | |
| implementation libs.parquet.column | |
| implementation libs.parquet.hadoop | |
| @@ -184,7 +184,7 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer | |
| testImplementation libs.avro.avro | |
| testImplementation libs.parquet.hadoop | |
| testImplementation libs.awaitility | |
| - testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:0.5.0" | |
| + testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" | |
| // Required because we remove antlr plugin dependencies from the compile configuration, see note above | |
| runtimeOnly libs.antlr.runtime | |
| @@ -265,6 +265,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio | |
| integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') | |
| integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts') | |
| integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts') | |
| + integrationImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" | |
| // runtime dependencies for running Hive Catalog based integration test | |
| integrationRuntimeOnly project(':iceberg-hive-metastore') | |
| diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java | |
| index 578845e3da..02b6dc270f 100644 | |
| --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java | |
| +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java | |
| @@ -57,6 +57,15 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { | |
| .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") | |
| .config( | |
| SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| .getOrCreate(); | |
| diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java | |
| index b5d6415763..0763a723d3 100644 | |
| --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java | |
| +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java | |
| @@ -448,7 +448,8 @@ public abstract class SparkRowLevelOperationsTestBase extends ExtensionsTestBase | |
| protected void assertAllBatchScansVectorized(SparkPlan plan) { | |
| List<SparkPlan> batchScans = SparkPlanUtil.collectBatchScans(plan); | |
| - assertThat(batchScans).hasSizeGreaterThan(0).allMatch(SparkPlan::supportsColumnar); | |
| + // When Comet is enabled, its native scan replaces BatchScanExec nodes entirely | |
| + assertThat(batchScans).allMatch(SparkPlan::supportsColumnar); | |
| } | |
| protected void createTableWithDeleteGranularity( | |
| diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java | |
| index ecf9e6f8a5..beca5f8521 100644 | |
| --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java | |
| +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java | |
| @@ -56,6 +56,15 @@ public class TestCallStatementParser { | |
| .master("local[2]") | |
| .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) | |
| .config("spark.extra.prop", "value") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| TestCallStatementParser.parser = spark.sessionState().sqlParser(); | |
| } | |
| diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java | |
| index 8cb92224fc..777b9bb1dd 100644 | |
| --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java | |
| +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java | |
| @@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; | |
| import org.apache.spark.sql.catalyst.expressions.Expression; | |
| import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; | |
| import org.apache.spark.sql.execution.CommandResultExec; | |
| +import org.apache.spark.sql.execution.SparkPlan; | |
| import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; | |
| import org.junit.jupiter.api.AfterEach; | |
| import org.junit.jupiter.api.BeforeEach; | |
| @@ -313,9 +314,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe | |
| private List<Expression> executeAndCollectFunctionCalls(String query, Object... args) { | |
| CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); | |
| - V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); | |
| + // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not | |
| + // a V2TableWriteExec; collect over the whole command plan in that case. | |
| + SparkPlan plan = command.commandPhysicalPlan(); | |
| + SparkPlan queryPlan = | |
| + plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; | |
| return SparkPlanUtil.collectExprs( | |
| - write.query(), | |
| + queryPlan, | |
| expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); | |
| } | |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java | |
| index 64edb1002e..57767da086 100644 | |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java | |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java | |
| @@ -179,6 +179,15 @@ public class DeleteOrphanFilesBenchmark { | |
| .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) | |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") | |
| .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .master("local"); | |
| spark = builder.getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java | |
| index a5d0456b0b..f52c387943 100644 | |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java | |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java | |
| @@ -392,6 +392,15 @@ public class IcebergSortCompactionBenchmark { | |
| "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") | |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") | |
| .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .master("local[*]"); | |
| spark = builder.getOrCreate(); | |
| Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); | |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java | |
| index c6794e43c6..4c1a165a44 100644 | |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java | |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java | |
| @@ -239,6 +239,15 @@ public class DVReaderBenchmark { | |
| .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) | |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") | |
| .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .master("local[*]") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java | |
| index ac74fb5a10..4aa5d5b361 100644 | |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java | |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java | |
| @@ -223,6 +223,15 @@ public class DVWriterBenchmark { | |
| .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) | |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") | |
| .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .master("local[*]") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java | |
| index 68c537e34a..2e421c7706 100644 | |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java | |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java | |
| @@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { | |
| } | |
| protected void setupSpark(boolean enableDictionaryEncoding) { | |
| - SparkSession.Builder builder = SparkSession.builder().config("spark.ui.enabled", false); | |
| + SparkSession.Builder builder = | |
| + SparkSession.builder() | |
| + .config("spark.ui.enabled", false) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g"); | |
| if (!enableDictionaryEncoding) { | |
| builder | |
| .config("parquet.dictionary.page.size", "1") | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java | |
| deleted file mode 100644 | |
| index 16159dcbdf..0000000000 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java | |
| +++ /dev/null | |
| @@ -1,140 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.io.IOException; | |
| -import org.apache.comet.parquet.AbstractColumnReader; | |
| -import org.apache.comet.parquet.ColumnReader; | |
| -import org.apache.comet.parquet.TypeUtil; | |
| -import org.apache.comet.parquet.Utils; | |
| -import org.apache.comet.shaded.arrow.c.CometSchemaImporter; | |
| -import org.apache.comet.shaded.arrow.memory.RootAllocator; | |
| -import org.apache.iceberg.parquet.VectorizedReader; | |
| -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; | |
| -import org.apache.iceberg.spark.SparkSchemaUtil; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.parquet.column.ColumnDescriptor; | |
| -import org.apache.parquet.column.page.PageReader; | |
| -import org.apache.spark.sql.types.DataType; | |
| -import org.apache.spark.sql.types.Metadata; | |
| -import org.apache.spark.sql.types.StructField; | |
| -import org.apache.spark.sql.vectorized.ColumnVector; | |
| - | |
| -class CometColumnReader implements VectorizedReader<ColumnVector> { | |
| - // use the Comet default batch size | |
| - public static final int DEFAULT_BATCH_SIZE = 8192; | |
| - | |
| - private final ColumnDescriptor descriptor; | |
| - private final DataType sparkType; | |
| - | |
| - // The delegated ColumnReader from Comet side | |
| - private AbstractColumnReader delegate; | |
| - private boolean initialized = false; | |
| - private int batchSize = DEFAULT_BATCH_SIZE; | |
| - private CometSchemaImporter importer; | |
| - | |
| - CometColumnReader(DataType sparkType, ColumnDescriptor descriptor) { | |
| - this.sparkType = sparkType; | |
| - this.descriptor = descriptor; | |
| - } | |
| - | |
| - CometColumnReader(Types.NestedField field) { | |
| - DataType dataType = SparkSchemaUtil.convert(field.type()); | |
| - StructField structField = new StructField(field.name(), dataType, false, Metadata.empty()); | |
| - this.sparkType = dataType; | |
| - this.descriptor = TypeUtil.convertToParquet(structField); | |
| - } | |
| - | |
| - public AbstractColumnReader delegate() { | |
| - return delegate; | |
| - } | |
| - | |
| - void setDelegate(AbstractColumnReader delegate) { | |
| - this.delegate = delegate; | |
| - } | |
| - | |
| - void setInitialized(boolean initialized) { | |
| - this.initialized = initialized; | |
| - } | |
| - | |
| - public int batchSize() { | |
| - return batchSize; | |
| - } | |
| - | |
| - /** | |
| - * This method is to initialized/reset the CometColumnReader. This needs to be called for each row | |
| - * group after readNextRowGroup, so a new dictionary encoding can be set for each of the new row | |
| - * groups. | |
| - */ | |
| - public void reset() { | |
| - if (importer != null) { | |
| - importer.close(); | |
| - } | |
| - | |
| - if (delegate != null) { | |
| - delegate.close(); | |
| - } | |
| - | |
| - this.importer = new CometSchemaImporter(new RootAllocator()); | |
| - this.delegate = Utils.getColumnReader(sparkType, descriptor, importer, batchSize, false, false); | |
| - this.initialized = true; | |
| - } | |
| - | |
| - public ColumnDescriptor descriptor() { | |
| - return descriptor; | |
| - } | |
| - | |
| - /** Returns the Spark data type for this column. */ | |
| - public DataType sparkType() { | |
| - return sparkType; | |
| - } | |
| - | |
| - /** | |
| - * Set the page reader to be 'pageReader'. | |
| - * | |
| - * <p>NOTE: this should be called before reading a new Parquet column chunk, and after {@link | |
| - * CometColumnReader#reset} is called. | |
| - */ | |
| - public void setPageReader(PageReader pageReader) throws IOException { | |
| - Preconditions.checkState(initialized, "Invalid state: 'reset' should be called first"); | |
| - ((ColumnReader) delegate).setPageReader(pageReader); | |
| - } | |
| - | |
| - @Override | |
| - public void close() { | |
| - // close resources on native side | |
| - if (importer != null) { | |
| - importer.close(); | |
| - } | |
| - | |
| - if (delegate != null) { | |
| - delegate.close(); | |
| - } | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int size) { | |
| - this.batchSize = size; | |
| - } | |
| - | |
| - @Override | |
| - public ColumnVector read(ColumnVector reuse, int numRowsToRead) { | |
| - throw new UnsupportedOperationException("Not supported"); | |
| - } | |
| -} | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java | |
| deleted file mode 100644 | |
| index 04ac69476a..0000000000 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java | |
| +++ /dev/null | |
| @@ -1,197 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.io.IOException; | |
| -import java.io.UncheckedIOException; | |
| -import java.util.List; | |
| -import java.util.Map; | |
| -import org.apache.comet.parquet.AbstractColumnReader; | |
| -import org.apache.comet.parquet.BatchReader; | |
| -import org.apache.iceberg.Schema; | |
| -import org.apache.iceberg.data.DeleteFilter; | |
| -import org.apache.iceberg.parquet.VectorizedReader; | |
| -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; | |
| -import org.apache.iceberg.spark.SparkSchemaUtil; | |
| -import org.apache.iceberg.util.Pair; | |
| -import org.apache.parquet.column.page.PageReadStore; | |
| -import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; | |
| -import org.apache.parquet.hadoop.metadata.ColumnPath; | |
| -import org.apache.spark.sql.catalyst.InternalRow; | |
| -import org.apache.spark.sql.vectorized.ColumnVector; | |
| -import org.apache.spark.sql.vectorized.ColumnarBatch; | |
| - | |
| -/** | |
| - * {@link VectorizedReader} that returns Spark's {@link ColumnarBatch} to support Spark's vectorized | |
| - * read path. The {@link ColumnarBatch} returned is created by passing in the Arrow vectors | |
| - * populated via delegated read calls to {@link CometColumnReader VectorReader(s)}. | |
| - */ | |
| -@SuppressWarnings("checkstyle:VisibilityModifier") | |
| -class CometColumnarBatchReader implements VectorizedReader<ColumnarBatch> { | |
| - | |
| - private final CometColumnReader[] readers; | |
| - private final boolean hasIsDeletedColumn; | |
| - | |
| - // The delegated BatchReader on the Comet side does the real work of loading a batch of rows. | |
| - // The Comet BatchReader contains an array of ColumnReader. There is no need to explicitly call | |
| - // ColumnReader.readBatch; instead, BatchReader.nextBatch will be called, which underneath calls | |
| - // ColumnReader.readBatch. The only exception is DeleteColumnReader, because at the time of | |
| - // calling BatchReader.nextBatch, the isDeleted value is not yet available, so | |
| - // DeleteColumnReader.readBatch must be called explicitly later, after the isDeleted value is | |
| - // available. | |
| - private final BatchReader delegate; | |
| - private DeleteFilter<InternalRow> deletes = null; | |
| - private long rowStartPosInBatch = 0; | |
| - | |
| - CometColumnarBatchReader(List<VectorizedReader<?>> readers, Schema schema) { | |
| - this.readers = | |
| - readers.stream().map(CometColumnReader.class::cast).toArray(CometColumnReader[]::new); | |
| - this.hasIsDeletedColumn = | |
| - readers.stream().anyMatch(reader -> reader instanceof CometDeleteColumnReader); | |
| - | |
| - AbstractColumnReader[] abstractColumnReaders = new AbstractColumnReader[readers.size()]; | |
| - this.delegate = new BatchReader(abstractColumnReaders); | |
| - delegate.setSparkSchema(SparkSchemaUtil.convert(schema)); | |
| - } | |
| - | |
| - @Override | |
| - public void setRowGroupInfo( | |
| - PageReadStore pageStore, Map<ColumnPath, ColumnChunkMetaData> metaData) { | |
| - for (int i = 0; i < readers.length; i++) { | |
| - try { | |
| - if (!(readers[i] instanceof CometConstantColumnReader) | |
| - && !(readers[i] instanceof CometPositionColumnReader) | |
| - && !(readers[i] instanceof CometDeleteColumnReader)) { | |
| - readers[i].reset(); | |
| - readers[i].setPageReader(pageStore.getPageReader(readers[i].descriptor())); | |
| - } | |
| - } catch (IOException e) { | |
| - throw new UncheckedIOException("Failed to setRowGroupInfo for Comet vectorization", e); | |
| - } | |
| - } | |
| - | |
| - for (int i = 0; i < readers.length; i++) { | |
| - delegate.getColumnReaders()[i] = this.readers[i].delegate(); | |
| - } | |
| - | |
| - this.rowStartPosInBatch = | |
| - pageStore | |
| - .getRowIndexOffset() | |
| - .orElseThrow( | |
| - () -> | |
| - new IllegalArgumentException( | |
| - "PageReadStore does not contain row index offset")); | |
| - } | |
| - | |
| - public void setDeleteFilter(DeleteFilter<InternalRow> deleteFilter) { | |
| - this.deletes = deleteFilter; | |
| - } | |
| - | |
| - @Override | |
| - public final ColumnarBatch read(ColumnarBatch reuse, int numRowsToRead) { | |
| - ColumnarBatch columnarBatch = new ColumnBatchLoader(numRowsToRead).loadDataToColumnBatch(); | |
| - rowStartPosInBatch += numRowsToRead; | |
| - return columnarBatch; | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - for (CometColumnReader reader : readers) { | |
| - if (reader != null) { | |
| - reader.setBatchSize(batchSize); | |
| - } | |
| - } | |
| - } | |
| - | |
| - @Override | |
| - public void close() { | |
| - for (CometColumnReader reader : readers) { | |
| - if (reader != null) { | |
| - reader.close(); | |
| - } | |
| - } | |
| - } | |
| - | |
| - private class ColumnBatchLoader { | |
| - private final int batchSize; | |
| - | |
| - ColumnBatchLoader(int numRowsToRead) { | |
| - Preconditions.checkArgument( | |
| - numRowsToRead > 0, "Invalid number of rows to read: %s", numRowsToRead); | |
| - this.batchSize = numRowsToRead; | |
| - } | |
| - | |
| - ColumnarBatch loadDataToColumnBatch() { | |
| - ColumnVector[] vectors = readDataToColumnVectors(); | |
| - int numLiveRows = batchSize; | |
| - | |
| - if (hasIsDeletedColumn) { | |
| - boolean[] isDeleted = buildIsDeleted(vectors); | |
| - readDeletedColumn(vectors, isDeleted); | |
| - } else { | |
| - Pair<int[], Integer> pair = buildRowIdMapping(vectors); | |
| - if (pair != null) { | |
| - int[] rowIdMapping = pair.first(); | |
| - numLiveRows = pair.second(); | |
| - for (int i = 0; i < vectors.length; i++) { | |
| - vectors[i] = new ColumnVectorWithFilter(vectors[i], rowIdMapping); | |
| - } | |
| - } | |
| - } | |
| - | |
| - if (deletes != null && deletes.hasEqDeletes()) { | |
| - vectors = ColumnarBatchUtil.removeExtraColumns(deletes, vectors); | |
| - } | |
| - | |
| - ColumnarBatch batch = new ColumnarBatch(vectors); | |
| - batch.setNumRows(numLiveRows); | |
| - return batch; | |
| - } | |
| - | |
| - private boolean[] buildIsDeleted(ColumnVector[] vectors) { | |
| - return ColumnarBatchUtil.buildIsDeleted(vectors, deletes, rowStartPosInBatch, batchSize); | |
| - } | |
| - | |
| - private Pair<int[], Integer> buildRowIdMapping(ColumnVector[] vectors) { | |
| - return ColumnarBatchUtil.buildRowIdMapping(vectors, deletes, rowStartPosInBatch, batchSize); | |
| - } | |
| - | |
| - ColumnVector[] readDataToColumnVectors() { | |
| - ColumnVector[] columnVectors = new ColumnVector[readers.length]; | |
| - // Fetch rows for all readers in the delegate | |
| - delegate.nextBatch(batchSize); | |
| - for (int i = 0; i < readers.length; i++) { | |
| - columnVectors[i] = readers[i].delegate().currentBatch(); | |
| - } | |
| - | |
| - return columnVectors; | |
| - } | |
| - | |
| - void readDeletedColumn(ColumnVector[] columnVectors, boolean[] isDeleted) { | |
| - for (int i = 0; i < readers.length; i++) { | |
| - if (readers[i] instanceof CometDeleteColumnReader) { | |
| - CometDeleteColumnReader deleteColumnReader = new CometDeleteColumnReader<>(isDeleted); | |
| - deleteColumnReader.setBatchSize(batchSize); | |
| - deleteColumnReader.delegate().readBatch(batchSize); | |
| - columnVectors[i] = deleteColumnReader.delegate().currentBatch(); | |
| - } | |
| - } | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java | |
| deleted file mode 100644 | |
| index 047c96314b..0000000000 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java | |
| +++ /dev/null | |
| @@ -1,65 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.math.BigDecimal; | |
| -import java.nio.ByteBuffer; | |
| -import org.apache.comet.parquet.ConstantColumnReader; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.spark.sql.types.DataType; | |
| -import org.apache.spark.sql.types.DataTypes; | |
| -import org.apache.spark.sql.types.Decimal; | |
| -import org.apache.spark.sql.types.DecimalType; | |
| -import org.apache.spark.unsafe.types.UTF8String; | |
| - | |
| -class CometConstantColumnReader<T> extends CometColumnReader { | |
| - | |
| - CometConstantColumnReader(T value, Types.NestedField field) { | |
| - super(field); | |
| - // use delegate to set constant value on the native side to be consumed by native execution. | |
| - setDelegate( | |
| - new ConstantColumnReader(sparkType(), descriptor(), convertToSparkValue(value), false)); | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - super.setBatchSize(batchSize); | |
| - delegate().setBatchSize(batchSize); | |
| - setInitialized(true); | |
| - } | |
| - | |
| - private Object convertToSparkValue(T value) { | |
| - DataType dataType = sparkType(); | |
| - // Match the value to Spark internal type if necessary | |
| - if (dataType == DataTypes.StringType && value instanceof String) { | |
| - // the internal type for StringType is UTF8String | |
| - return UTF8String.fromString((String) value); | |
| - } else if (dataType instanceof DecimalType && value instanceof BigDecimal) { | |
| - // the internal type for DecimalType is Decimal | |
| - return Decimal.apply((BigDecimal) value); | |
| - } else if (dataType == DataTypes.BinaryType && value instanceof ByteBuffer) { | |
| - // the internal type for DecimalType is byte[] | |
| - // Iceberg default value should always use HeapBufferBuffer, so calling ByteBuffer.array() | |
| - // should be safe. | |
| - return ((ByteBuffer) value).array(); | |
| - } else { | |
| - return value; | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java | |
| deleted file mode 100644 | |
| index 6235bfe486..0000000000 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java | |
| +++ /dev/null | |
| @@ -1,75 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import org.apache.comet.parquet.MetadataColumnReader; | |
| -import org.apache.comet.parquet.Native; | |
| -import org.apache.comet.parquet.TypeUtil; | |
| -import org.apache.iceberg.MetadataColumns; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.spark.sql.types.DataTypes; | |
| -import org.apache.spark.sql.types.Metadata; | |
| -import org.apache.spark.sql.types.StructField; | |
| - | |
| -class CometDeleteColumnReader<T> extends CometColumnReader { | |
| - CometDeleteColumnReader(Types.NestedField field) { | |
| - super(field); | |
| - setDelegate(new DeleteColumnReader()); | |
| - } | |
| - | |
| - CometDeleteColumnReader(boolean[] isDeleted) { | |
| - super(MetadataColumns.IS_DELETED); | |
| - setDelegate(new DeleteColumnReader(isDeleted)); | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - super.setBatchSize(batchSize); | |
| - delegate().setBatchSize(batchSize); | |
| - setInitialized(true); | |
| - } | |
| - | |
| - private static class DeleteColumnReader extends MetadataColumnReader { | |
| - private boolean[] isDeleted; | |
| - | |
| - DeleteColumnReader() { | |
| - super( | |
| - DataTypes.BooleanType, | |
| - TypeUtil.convertToParquet( | |
| - new StructField("_deleted", DataTypes.BooleanType, false, Metadata.empty())), | |
| - false /* useDecimal128 = false */, | |
| - false /* isConstant = false */); | |
| - this.isDeleted = new boolean[0]; | |
| - } | |
| - | |
| - DeleteColumnReader(boolean[] isDeleted) { | |
| - this(); | |
| - this.isDeleted = isDeleted; | |
| - } | |
| - | |
| - @Override | |
| - public void readBatch(int total) { | |
| - Native.resetBatch(nativeHandle); | |
| - // set isDeleted on the native side to be consumed by native execution | |
| - Native.setIsDeleted(nativeHandle, isDeleted); | |
| - | |
| - super.readBatch(total); | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java | |
| deleted file mode 100644 | |
| index bcc0e514c2..0000000000 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java | |
| +++ /dev/null | |
| @@ -1,62 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import org.apache.comet.parquet.MetadataColumnReader; | |
| -import org.apache.comet.parquet.Native; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.parquet.column.ColumnDescriptor; | |
| -import org.apache.spark.sql.types.DataTypes; | |
| - | |
| -class CometPositionColumnReader extends CometColumnReader { | |
| - CometPositionColumnReader(Types.NestedField field) { | |
| - super(field); | |
| - setDelegate(new PositionColumnReader(descriptor())); | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - super.setBatchSize(batchSize); | |
| - delegate().setBatchSize(batchSize); | |
| - setInitialized(true); | |
| - } | |
| - | |
| - private static class PositionColumnReader extends MetadataColumnReader { | |
| - /** The current position value of the column that are used to initialize this column reader. */ | |
| - private long position; | |
| - | |
| - PositionColumnReader(ColumnDescriptor descriptor) { | |
| - super( | |
| - DataTypes.LongType, | |
| - descriptor, | |
| - false /* useDecimal128 = false */, | |
| - false /* isConstant = false */); | |
| - } | |
| - | |
| - @Override | |
| - public void readBatch(int total) { | |
| - Native.resetBatch(nativeHandle); | |
| - // set position on the native side to be consumed by native execution | |
| - Native.setPosition(nativeHandle, position, total); | |
| - position += total; | |
| - | |
| - super.readBatch(total); | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java | |
| deleted file mode 100644 | |
| index d36f1a7274..0000000000 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java | |
| +++ /dev/null | |
| @@ -1,147 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.util.List; | |
| -import java.util.Map; | |
| -import java.util.function.Function; | |
| -import java.util.stream.IntStream; | |
| -import org.apache.iceberg.MetadataColumns; | |
| -import org.apache.iceberg.Schema; | |
| -import org.apache.iceberg.data.DeleteFilter; | |
| -import org.apache.iceberg.parquet.TypeWithSchemaVisitor; | |
| -import org.apache.iceberg.parquet.VectorizedReader; | |
| -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; | |
| -import org.apache.iceberg.relocated.com.google.common.collect.Lists; | |
| -import org.apache.iceberg.relocated.com.google.common.collect.Maps; | |
| -import org.apache.iceberg.spark.SparkSchemaUtil; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.parquet.column.ColumnDescriptor; | |
| -import org.apache.parquet.schema.GroupType; | |
| -import org.apache.parquet.schema.MessageType; | |
| -import org.apache.parquet.schema.PrimitiveType; | |
| -import org.apache.parquet.schema.Type; | |
| -import org.apache.spark.sql.catalyst.InternalRow; | |
| - | |
| -class CometVectorizedReaderBuilder extends TypeWithSchemaVisitor<VectorizedReader<?>> { | |
| - | |
| - private final MessageType parquetSchema; | |
| - private final Schema icebergSchema; | |
| - private final Map<Integer, ?> idToConstant; | |
| - private final Function<List<VectorizedReader<?>>, VectorizedReader<?>> readerFactory; | |
| - private final DeleteFilter<InternalRow> deleteFilter; | |
| - | |
| - CometVectorizedReaderBuilder( | |
| - Schema expectedSchema, | |
| - MessageType parquetSchema, | |
| - Map<Integer, ?> idToConstant, | |
| - Function<List<VectorizedReader<?>>, VectorizedReader<?>> readerFactory, | |
| - DeleteFilter<InternalRow> deleteFilter) { | |
| - this.parquetSchema = parquetSchema; | |
| - this.icebergSchema = expectedSchema; | |
| - this.idToConstant = idToConstant; | |
| - this.readerFactory = readerFactory; | |
| - this.deleteFilter = deleteFilter; | |
| - } | |
| - | |
| - @Override | |
| - public VectorizedReader<?> message( | |
| - Types.StructType expected, MessageType message, List<VectorizedReader<?>> fieldReaders) { | |
| - GroupType groupType = message.asGroupType(); | |
| - Map<Integer, VectorizedReader<?>> readersById = Maps.newHashMap(); | |
| - List<Type> fields = groupType.getFields(); | |
| - | |
| - IntStream.range(0, fields.size()) | |
| - .filter(pos -> fields.get(pos).getId() != null) | |
| - .forEach(pos -> readersById.put(fields.get(pos).getId().intValue(), fieldReaders.get(pos))); | |
| - | |
| - List<Types.NestedField> icebergFields = | |
| - expected != null ? expected.fields() : ImmutableList.of(); | |
| - | |
| - List<VectorizedReader<?>> reorderedFields = | |
| - Lists.newArrayListWithExpectedSize(icebergFields.size()); | |
| - | |
| - for (Types.NestedField field : icebergFields) { | |
| - int id = field.fieldId(); | |
| - VectorizedReader<?> reader = readersById.get(id); | |
| - if (idToConstant.containsKey(id)) { | |
| - CometConstantColumnReader constantReader = | |
| - new CometConstantColumnReader<>(idToConstant.get(id), field); | |
| - reorderedFields.add(constantReader); | |
| - } else if (id == MetadataColumns.ROW_POSITION.fieldId()) { | |
| - reorderedFields.add(new CometPositionColumnReader(field)); | |
| - } else if (id == MetadataColumns.IS_DELETED.fieldId()) { | |
| - CometColumnReader deleteReader = new CometDeleteColumnReader<>(field); | |
| - reorderedFields.add(deleteReader); | |
| - } else if (reader != null) { | |
| - reorderedFields.add(reader); | |
| - } else if (field.initialDefault() != null) { | |
| - CometColumnReader constantReader = | |
| - new CometConstantColumnReader<>(field.initialDefault(), field); | |
| - reorderedFields.add(constantReader); | |
| - } else if (field.isOptional()) { | |
| - CometColumnReader constantReader = new CometConstantColumnReader<>(null, field); | |
| - reorderedFields.add(constantReader); | |
| - } else { | |
| - throw new IllegalArgumentException( | |
| - String.format("Missing required field: %s", field.name())); | |
| - } | |
| - } | |
| - return vectorizedReader(reorderedFields); | |
| - } | |
| - | |
| - protected VectorizedReader<?> vectorizedReader(List<VectorizedReader<?>> reorderedFields) { | |
| - VectorizedReader<?> reader = readerFactory.apply(reorderedFields); | |
| - if (deleteFilter != null) { | |
| - ((CometColumnarBatchReader) reader).setDeleteFilter(deleteFilter); | |
| - } | |
| - return reader; | |
| - } | |
| - | |
| - @Override | |
| - public VectorizedReader<?> struct( | |
| - Types.StructType expected, GroupType groupType, List<VectorizedReader<?>> fieldReaders) { | |
| - if (expected != null) { | |
| - throw new UnsupportedOperationException( | |
| - "Vectorized reads are not supported yet for struct fields"); | |
| - } | |
| - return null; | |
| - } | |
| - | |
| - @Override | |
| - public VectorizedReader<?> primitive( | |
| - org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) { | |
| - | |
| - if (primitive.getId() == null) { | |
| - return null; | |
| - } | |
| - int parquetFieldId = primitive.getId().intValue(); | |
| - ColumnDescriptor desc = parquetSchema.getColumnDescription(currentPath()); | |
| - // Nested types not yet supported for vectorized reads | |
| - if (desc.getMaxRepetitionLevel() > 0) { | |
| - return null; | |
| - } | |
| - Types.NestedField icebergField = icebergSchema.findField(parquetFieldId); | |
| - if (icebergField == null) { | |
| - return null; | |
| - } | |
| - | |
| - return new CometColumnReader(SparkSchemaUtil.convert(icebergField.type()), desc); | |
| - } | |
| -} | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java | |
| index b523bc5bff..636ad3be7d 100644 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java | |
| +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java | |
| @@ -70,23 +70,6 @@ public class VectorizedSparkParquetReaders { | |
| deleteFilter)); | |
| } | |
| - public static CometColumnarBatchReader buildCometReader( | |
| - Schema expectedSchema, | |
| - MessageType fileSchema, | |
| - Map<Integer, ?> idToConstant, | |
| - DeleteFilter<InternalRow> deleteFilter) { | |
| - return (CometColumnarBatchReader) | |
| - TypeWithSchemaVisitor.visit( | |
| - expectedSchema.asStruct(), | |
| - fileSchema, | |
| - new CometVectorizedReaderBuilder( | |
| - expectedSchema, | |
| - fileSchema, | |
| - idToConstant, | |
| - readers -> new CometColumnarBatchReader(readers, expectedSchema), | |
| - deleteFilter)); | |
| - } | |
| - | |
| // enables unsafe memory access to avoid costly checks to see if index is within bounds | |
| // as long as it is not configured explicitly (see BoundsChecking in Arrow) | |
| private static void enableUnsafeMemoryAccess() { | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java | |
| index 780e1750a5..25f253eede 100644 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java | |
| +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java | |
| @@ -34,7 +34,6 @@ import org.apache.iceberg.parquet.Parquet; | |
| import org.apache.iceberg.relocated.com.google.common.collect.Sets; | |
| import org.apache.iceberg.spark.OrcBatchReadConf; | |
| import org.apache.iceberg.spark.ParquetBatchReadConf; | |
| -import org.apache.iceberg.spark.ParquetReaderType; | |
| import org.apache.iceberg.spark.data.vectorized.VectorizedSparkOrcReaders; | |
| import org.apache.iceberg.spark.data.vectorized.VectorizedSparkParquetReaders; | |
| import org.apache.iceberg.types.TypeUtil; | |
| @@ -92,15 +91,9 @@ abstract class BaseBatchReader<T extends ScanTask> extends BaseReader<ColumnarBa | |
| .project(requiredSchema) | |
| .split(start, length) | |
| .createBatchedReaderFunc( | |
| - fileSchema -> { | |
| - if (parquetConf.readerType() == ParquetReaderType.COMET) { | |
| - return VectorizedSparkParquetReaders.buildCometReader( | |
| - requiredSchema, fileSchema, idToConstant, deleteFilter); | |
| - } else { | |
| - return VectorizedSparkParquetReaders.buildReader( | |
| - requiredSchema, fileSchema, idToConstant, deleteFilter); | |
| - } | |
| - }) | |
| + fileSchema -> | |
| + VectorizedSparkParquetReaders.buildReader( | |
| + requiredSchema, fileSchema, idToConstant, deleteFilter)) | |
| .recordsPerBatch(parquetConf.batchSize()) | |
| .filter(residual) | |
| .caseSensitive(caseSensitive()) | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java | |
| index 404ba72846..b375458f05 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java | |
| @@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase | |
| .master("local[2]") | |
| .config("spark.serializer", serializer) | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java | |
| index 659507e4c5..2a3a87f079 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java | |
| @@ -73,6 +73,15 @@ public class TestSparkDistributedDataScanDeletes | |
| .master("local[2]") | |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java | |
| index a218f965ea..e30a8d6c6f 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java | |
| @@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles | |
| .master("local[2]") | |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java | |
| index 2665d7ba8d..c24c82341a 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java | |
| @@ -63,6 +63,15 @@ public class TestSparkDistributedDataScanReporting | |
| .master("local[2]") | |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java | |
| index 3e9f3334ef..0f96c07739 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java | |
| @@ -77,6 +77,15 @@ public abstract class TestBase extends SparkTestHelperBase { | |
| .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") | |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) | |
| .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| .getOrCreate(); | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTest.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTest.java | |
| index a31138ae01..501c8ac93e 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTest.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTest.java | |
| @@ -47,7 +47,6 @@ import org.apache.iceberg.types.Types.MapType; | |
| import org.apache.iceberg.types.Types.StructType; | |
| import org.apache.iceberg.util.DateTimeUtil; | |
| import org.assertj.core.api.Assumptions; | |
| -import org.assertj.core.api.Condition; | |
| import org.junit.jupiter.api.Test; | |
| import org.junit.jupiter.api.io.TempDir; | |
| import org.junit.jupiter.params.ParameterizedTest; | |
| @@ -307,12 +306,6 @@ public abstract class AvroDataTest { | |
| .build()); | |
| assertThatThrownBy(() -> writeAndValidate(writeSchema, expectedSchema)) | |
| - .has( | |
| - new Condition<>( | |
| - t -> | |
| - IllegalArgumentException.class.isInstance(t) | |
| - || IllegalArgumentException.class.isInstance(t.getCause()), | |
| - "Expecting a throwable or cause that is an instance of IllegalArgumentException")) | |
| .hasMessageContaining("Missing required field: missing_str"); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java | |
| index bc4e722bc8..57770308b3 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java | |
| @@ -59,7 +59,19 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect | |
| @BeforeAll | |
| public static void startSpark() { | |
| - spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterAll | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java | |
| index 0886df957d..eca0f74533 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java | |
| @@ -57,7 +57,19 @@ public abstract class ScanTestBase extends AvroDataTest { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - ScanTestBase.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + ScanTestBase.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| ScanTestBase.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java | |
| index 6b7d861364..23419aa9b1 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java | |
| @@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestCompressionSettings.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestCompressionSettings.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @BeforeEach | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java | |
| index c4ba96e634..dcf72c19b6 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java | |
| @@ -75,7 +75,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestDataSourceOptions.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestDataSourceOptions.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterAll | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java | |
| index 348173596e..03156f786b 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java | |
| @@ -110,7 +110,19 @@ public class TestFilteredScan { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestFilteredScan.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestFilteredScan.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterAll | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java | |
| index 84c99a575c..e647ab924e 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java | |
| @@ -93,7 +93,19 @@ public class TestForwardCompatibility { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestForwardCompatibility.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestForwardCompatibility.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterAll | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java | |
| index 7eff93d204..2fdc87a0a8 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java | |
| @@ -46,7 +46,19 @@ public class TestIcebergSpark { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestIcebergSpark.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestIcebergSpark.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterAll | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java | |
| index 9464f687b0..8653560334 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java | |
| @@ -112,7 +112,19 @@ public class TestPartitionPruning { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestPartitionPruning.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestPartitionPruning.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java | |
| index 5c218f21c4..2f17343d59 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java | |
| @@ -107,7 +107,19 @@ public class TestPartitionValues { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestPartitionValues.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestPartitionValues.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterAll | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java | |
| index a7334a580c..68210dc8e8 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java | |
| @@ -87,7 +87,19 @@ public class TestSnapshotSelection { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestSnapshotSelection.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestSnapshotSelection.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterAll | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | |
| index 182b1ef8f5..76e04b1472 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | |
| @@ -120,7 +120,19 @@ public class TestSparkDataFile { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestSparkDataFile.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestSparkDataFile.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java | |
| index fb2b312bed..846f50e0c9 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java | |
| @@ -96,7 +96,19 @@ public class TestSparkDataWrite { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestSparkDataWrite.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestSparkDataWrite.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterEach | |
| @@ -140,7 +152,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); | |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); | |
| for (ManifestFile manifest : | |
| @@ -210,7 +222,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); | |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); | |
| } | |
| @@ -256,7 +268,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); | |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); | |
| } | |
| @@ -309,7 +321,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); | |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); | |
| } | |
| @@ -352,7 +364,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); | |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); | |
| } | |
| @@ -392,7 +404,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); | |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); | |
| @@ -458,7 +470,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); | |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); | |
| } | |
| @@ -622,7 +634,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); | |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); | |
| @@ -708,7 +720,7 @@ public class TestSparkDataWrite { | |
| // Since write and commit succeeded, the rows should be readable | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).as("Number of rows should match").hasSize(records.size() + records2.size()); | |
| assertThat(actual) | |
| .describedAs("Result rows should match") | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java | |
| index becf6a064d..91451c268e 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java | |
| @@ -83,7 +83,19 @@ public class TestSparkReadProjection extends TestReadProjection { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestSparkReadProjection.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestSparkReadProjection.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| ImmutableMap<String, String> config = | |
| ImmutableMap.of( | |
| "type", "hive", | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java | |
| index 4f1cef5d37..4a1a602ed4 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java | |
| @@ -136,6 +136,15 @@ public class TestSparkReaderDeletes extends DeleteReadTests { | |
| .config("spark.ui.liveUpdate.period", 0) | |
| .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") | |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| .getOrCreate(); | |
| @@ -204,7 +213,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { | |
| } | |
| protected boolean countDeletes() { | |
| - return true; | |
| + // TODO: Enable once iceberg-rust exposes delete count metrics to Comet | |
| + return false; | |
| } | |
| @Override | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java | |
| index baf7fa8f88..b58321b761 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java | |
| @@ -182,6 +182,15 @@ public class TestSparkReaderWithBloomFilter { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| .getOrCreate(); | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java | |
| index c84a65cbe9..e17ba73ee4 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java | |
| @@ -67,6 +67,15 @@ public class TestStructuredStreaming { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.sql.shuffle.partitions", 4) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java | |
| index 306444b9f2..e4082d9cd9 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java | |
| @@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestTimestampWithoutZone.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestTimestampWithoutZone.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterAll | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java | |
| index 841268a6be..97f7187314 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java | |
| @@ -80,7 +80,19 @@ public class TestWriteMetricsConfig { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestWriteMetricsConfig.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestWriteMetricsConfig.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java | |
| index 6e09252704..4b0f2e4d58 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java | |
| @@ -60,6 +60,15 @@ public class TestAggregatePushDown extends CatalogTestBase { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.sql.iceberg.aggregate_pushdown", "true") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| .getOrCreate(); | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java | |
| index 9d2ce2b388..5e23368848 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java | |
| @@ -598,9 +598,7 @@ public class TestFilterPushDown extends TestBaseWithCatalog { | |
| String planAsString = sparkPlan.toString().replaceAll("#(\\d+L?)", ""); | |
| if (sparkFilter != null) { | |
| - assertThat(planAsString) | |
| - .as("Post scan filter should match") | |
| - .contains("Filter (" + sparkFilter + ")"); | |
| + assertThat(planAsString).as("Post scan filter should match").contains("CometFilter"); | |
| } else { | |
| assertThat(planAsString).as("Should be no post scan filter").doesNotContain("Filter ("); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java | |
| index 6719c45ca9..2515454401 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java | |
| @@ -616,7 +616,7 @@ public class TestStoragePartitionedJoins extends TestBaseWithCatalog { | |
| + "FROM %s t1 " | |
| + "INNER JOIN %s t2 " | |
| + "ON t1.id = t2.id AND t1.%s = t2.%s " | |
| - + "ORDER BY t1.id, t1.%s", | |
| + + "ORDER BY t1.id, t1.%s, t1.salary", | |
| sourceColumnName, | |
| tableName, | |
| tableName(OTHER_TABLE_NAME), |