| diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml | |
| index eeabe54f5f..72873ccbc9 100644 | |
| --- a/gradle/libs.versions.toml | |
| +++ b/gradle/libs.versions.toml | |
| @@ -37,7 +37,7 @@ awssdk-s3accessgrants = "2.3.0" | |
| bson-ver = "4.11.5" | |
| caffeine = "2.9.3" | |
| calcite = "1.40.0" | |
| -comet = "0.8.1" | |
| +comet = "1.1.0-SNAPSHOT" | |
| datasketches = "6.2.0" | |
| delta-standalone = "3.3.2" | |
| delta-spark = "3.3.2" | |
| diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle | |
| index 714be0831d..dede65f213 100644 | |
| --- a/spark/v3.4/build.gradle | |
| +++ b/spark/v3.4/build.gradle | |
| @@ -264,6 +264,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio | |
| integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') | |
| integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts') | |
| integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts') | |
| + integrationImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" | |
| // runtime dependencies for running Hive Catalog based integration test | |
| integrationRuntimeOnly project(':iceberg-hive-metastore') | |
| diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java | |
| index 4c1a509591..186a264a88 100644 | |
| --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java | |
| +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java | |
| @@ -59,7 +59,25 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { | |
| .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") | |
| .config( | |
| SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| TestBase.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java | |
| index 58d054bd05..ef1c1cb6d1 100644 | |
| --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java | |
| +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java | |
| @@ -56,6 +56,15 @@ public class TestCallStatementParser { | |
| .master("local[2]") | |
| .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) | |
| .config("spark.extra.prop", "value") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| TestCallStatementParser.parser = spark.sessionState().sqlParser(); | |
| } | |
| diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java | |
| index 5a0a04edba..afe92a8b72 100644 | |
| --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java | |
| +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java | |
| @@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; | |
| import org.apache.spark.sql.catalyst.expressions.Expression; | |
| import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; | |
| import org.apache.spark.sql.execution.CommandResultExec; | |
| +import org.apache.spark.sql.execution.SparkPlan; | |
| import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; | |
| import org.junit.jupiter.api.AfterEach; | |
| import org.junit.jupiter.api.BeforeEach; | |
| @@ -313,9 +314,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe | |
| private List<Expression> executeAndCollectFunctionCalls(String query, Object... args) { | |
| CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); | |
| - V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); | |
| + // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not | |
| + // a V2TableWriteExec; collect over the whole command plan in that case. | |
| + SparkPlan plan = command.commandPhysicalPlan(); | |
| + SparkPlan queryPlan = | |
| + plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; | |
| return SparkPlanUtil.collectExprs( | |
| - write.query(), | |
| + queryPlan, | |
| expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); | |
| } | |
| diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java | |
| index b6ade2bff3..90a4839dad 100644 | |
| --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java | |
| +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java | |
| @@ -170,6 +170,15 @@ public class DeleteOrphanFilesBenchmark { | |
| .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) | |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") | |
| .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .master("local"); | |
| spark = builder.getOrCreate(); | |
| } | |
| diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java | |
| index 7cd41bd7e7..0b6d490529 100644 | |
| --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java | |
| +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java | |
| @@ -386,6 +386,15 @@ public class IcebergSortCompactionBenchmark { | |
| "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") | |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") | |
| .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .master("local[*]"); | |
| spark = builder.getOrCreate(); | |
| Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); | |
| diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java | |
| index 68c537e34a..2e421c7706 100644 | |
| --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java | |
| +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java | |
| @@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { | |
| } | |
| protected void setupSpark(boolean enableDictionaryEncoding) { | |
| - SparkSession.Builder builder = SparkSession.builder().config("spark.ui.enabled", false); | |
| + SparkSession.Builder builder = | |
| + SparkSession.builder() | |
| + .config("spark.ui.enabled", false) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g"); | |
| if (!enableDictionaryEncoding) { | |
| builder | |
| .config("parquet.dictionary.page.size", "1") | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java | |
| deleted file mode 100644 | |
| index 81b7d83a70..0000000000 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java | |
| +++ /dev/null | |
| @@ -1,140 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.io.IOException; | |
| -import org.apache.comet.CometSchemaImporter; | |
| -import org.apache.comet.parquet.AbstractColumnReader; | |
| -import org.apache.comet.parquet.ColumnReader; | |
| -import org.apache.comet.parquet.TypeUtil; | |
| -import org.apache.comet.parquet.Utils; | |
| -import org.apache.comet.shaded.arrow.memory.RootAllocator; | |
| -import org.apache.iceberg.parquet.VectorizedReader; | |
| -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; | |
| -import org.apache.iceberg.spark.SparkSchemaUtil; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.parquet.column.ColumnDescriptor; | |
| -import org.apache.parquet.column.page.PageReader; | |
| -import org.apache.spark.sql.types.DataType; | |
| -import org.apache.spark.sql.types.Metadata; | |
| -import org.apache.spark.sql.types.StructField; | |
| -import org.apache.spark.sql.vectorized.ColumnVector; | |
| - | |
| -class CometColumnReader implements VectorizedReader<ColumnVector> { | |
| - // use the Comet default batch size | |
| - public static final int DEFAULT_BATCH_SIZE = 8192; | |
| - | |
| - private final ColumnDescriptor descriptor; | |
| - private final DataType sparkType; | |
| - | |
| - // The delegated ColumnReader from Comet side | |
| - private AbstractColumnReader delegate; | |
| - private boolean initialized = false; | |
| - private int batchSize = DEFAULT_BATCH_SIZE; | |
| - private CometSchemaImporter importer; | |
| - | |
| - CometColumnReader(DataType sparkType, ColumnDescriptor descriptor) { | |
| - this.sparkType = sparkType; | |
| - this.descriptor = descriptor; | |
| - } | |
| - | |
| - CometColumnReader(Types.NestedField field) { | |
| - DataType dataType = SparkSchemaUtil.convert(field.type()); | |
| - StructField structField = new StructField(field.name(), dataType, false, Metadata.empty()); | |
| - this.sparkType = dataType; | |
| - this.descriptor = TypeUtil.convertToParquet(structField); | |
| - } | |
| - | |
| - public AbstractColumnReader delegate() { | |
| - return delegate; | |
| - } | |
| - | |
| - void setDelegate(AbstractColumnReader delegate) { | |
| - this.delegate = delegate; | |
| - } | |
| - | |
| - void setInitialized(boolean initialized) { | |
| - this.initialized = initialized; | |
| - } | |
| - | |
| - public int batchSize() { | |
| - return batchSize; | |
| - } | |
| - | |
| - /** | |
| - * This method is to initialized/reset the CometColumnReader. This needs to be called for each row | |
| - * group after readNextRowGroup, so a new dictionary encoding can be set for each of the new row | |
| - * groups. | |
| - */ | |
| - public void reset() { | |
| - if (importer != null) { | |
| - importer.close(); | |
| - } | |
| - | |
| - if (delegate != null) { | |
| - delegate.close(); | |
| - } | |
| - | |
| - this.importer = new CometSchemaImporter(new RootAllocator()); | |
| - this.delegate = Utils.getColumnReader(sparkType, descriptor, importer, batchSize, false, false); | |
| - this.initialized = true; | |
| - } | |
| - | |
| - public ColumnDescriptor descriptor() { | |
| - return descriptor; | |
| - } | |
| - | |
| - /** Returns the Spark data type for this column. */ | |
| - public DataType sparkType() { | |
| - return sparkType; | |
| - } | |
| - | |
| - /** | |
| - * Set the page reader to be 'pageReader'. | |
| - * | |
| - * <p>NOTE: this should be called before reading a new Parquet column chunk, and after {@link | |
| - * CometColumnReader#reset} is called. | |
| - */ | |
| - public void setPageReader(PageReader pageReader) throws IOException { | |
| - Preconditions.checkState(initialized, "Invalid state: 'reset' should be called first"); | |
| - ((ColumnReader) delegate).setPageReader(pageReader); | |
| - } | |
| - | |
| - @Override | |
| - public void close() { | |
| - // close resources on native side | |
| - if (importer != null) { | |
| - importer.close(); | |
| - } | |
| - | |
| - if (delegate != null) { | |
| - delegate.close(); | |
| - } | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int size) { | |
| - this.batchSize = size; | |
| - } | |
| - | |
| - @Override | |
| - public ColumnVector read(ColumnVector reuse, int numRowsToRead) { | |
| - throw new UnsupportedOperationException("Not supported"); | |
| - } | |
| -} | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java | |
| deleted file mode 100644 | |
| index 04ac69476a..0000000000 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java | |
| +++ /dev/null | |
| @@ -1,197 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.io.IOException; | |
| -import java.io.UncheckedIOException; | |
| -import java.util.List; | |
| -import java.util.Map; | |
| -import org.apache.comet.parquet.AbstractColumnReader; | |
| -import org.apache.comet.parquet.BatchReader; | |
| -import org.apache.iceberg.Schema; | |
| -import org.apache.iceberg.data.DeleteFilter; | |
| -import org.apache.iceberg.parquet.VectorizedReader; | |
| -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; | |
| -import org.apache.iceberg.spark.SparkSchemaUtil; | |
| -import org.apache.iceberg.util.Pair; | |
| -import org.apache.parquet.column.page.PageReadStore; | |
| -import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; | |
| -import org.apache.parquet.hadoop.metadata.ColumnPath; | |
| -import org.apache.spark.sql.catalyst.InternalRow; | |
| -import org.apache.spark.sql.vectorized.ColumnVector; | |
| -import org.apache.spark.sql.vectorized.ColumnarBatch; | |
| - | |
| -/** | |
| - * {@link VectorizedReader} that returns Spark's {@link ColumnarBatch} to support Spark's vectorized | |
| - * read path. The {@link ColumnarBatch} returned is created by passing in the Arrow vectors | |
| - * populated via delegated read calls to {@link CometColumnReader VectorReader(s)}. | |
| - */ | |
| -@SuppressWarnings("checkstyle:VisibilityModifier") | |
| -class CometColumnarBatchReader implements VectorizedReader<ColumnarBatch> { | |
| - | |
| - private final CometColumnReader[] readers; | |
| - private final boolean hasIsDeletedColumn; | |
| - | |
| - // The delegated BatchReader on the Comet side does the real work of loading a batch of rows. | |
| - // The Comet BatchReader contains an array of ColumnReader. There is no need to explicitly call | |
| - // ColumnReader.readBatch; instead, BatchReader.nextBatch will be called, which underneath calls | |
| - // ColumnReader.readBatch. The only exception is DeleteColumnReader, because at the time of | |
| - // calling BatchReader.nextBatch, the isDeleted value is not yet available, so | |
| - // DeleteColumnReader.readBatch must be called explicitly later, after the isDeleted value is | |
| - // available. | |
| - private final BatchReader delegate; | |
| - private DeleteFilter<InternalRow> deletes = null; | |
| - private long rowStartPosInBatch = 0; | |
| - | |
| - CometColumnarBatchReader(List<VectorizedReader<?>> readers, Schema schema) { | |
| - this.readers = | |
| - readers.stream().map(CometColumnReader.class::cast).toArray(CometColumnReader[]::new); | |
| - this.hasIsDeletedColumn = | |
| - readers.stream().anyMatch(reader -> reader instanceof CometDeleteColumnReader); | |
| - | |
| - AbstractColumnReader[] abstractColumnReaders = new AbstractColumnReader[readers.size()]; | |
| - this.delegate = new BatchReader(abstractColumnReaders); | |
| - delegate.setSparkSchema(SparkSchemaUtil.convert(schema)); | |
| - } | |
| - | |
| - @Override | |
| - public void setRowGroupInfo( | |
| - PageReadStore pageStore, Map<ColumnPath, ColumnChunkMetaData> metaData) { | |
| - for (int i = 0; i < readers.length; i++) { | |
| - try { | |
| - if (!(readers[i] instanceof CometConstantColumnReader) | |
| - && !(readers[i] instanceof CometPositionColumnReader) | |
| - && !(readers[i] instanceof CometDeleteColumnReader)) { | |
| - readers[i].reset(); | |
| - readers[i].setPageReader(pageStore.getPageReader(readers[i].descriptor())); | |
| - } | |
| - } catch (IOException e) { | |
| - throw new UncheckedIOException("Failed to setRowGroupInfo for Comet vectorization", e); | |
| - } | |
| - } | |
| - | |
| - for (int i = 0; i < readers.length; i++) { | |
| - delegate.getColumnReaders()[i] = this.readers[i].delegate(); | |
| - } | |
| - | |
| - this.rowStartPosInBatch = | |
| - pageStore | |
| - .getRowIndexOffset() | |
| - .orElseThrow( | |
| - () -> | |
| - new IllegalArgumentException( | |
| - "PageReadStore does not contain row index offset")); | |
| - } | |
| - | |
| - public void setDeleteFilter(DeleteFilter<InternalRow> deleteFilter) { | |
| - this.deletes = deleteFilter; | |
| - } | |
| - | |
| - @Override | |
| - public final ColumnarBatch read(ColumnarBatch reuse, int numRowsToRead) { | |
| - ColumnarBatch columnarBatch = new ColumnBatchLoader(numRowsToRead).loadDataToColumnBatch(); | |
| - rowStartPosInBatch += numRowsToRead; | |
| - return columnarBatch; | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - for (CometColumnReader reader : readers) { | |
| - if (reader != null) { | |
| - reader.setBatchSize(batchSize); | |
| - } | |
| - } | |
| - } | |
| - | |
| - @Override | |
| - public void close() { | |
| - for (CometColumnReader reader : readers) { | |
| - if (reader != null) { | |
| - reader.close(); | |
| - } | |
| - } | |
| - } | |
| - | |
| - private class ColumnBatchLoader { | |
| - private final int batchSize; | |
| - | |
| - ColumnBatchLoader(int numRowsToRead) { | |
| - Preconditions.checkArgument( | |
| - numRowsToRead > 0, "Invalid number of rows to read: %s", numRowsToRead); | |
| - this.batchSize = numRowsToRead; | |
| - } | |
| - | |
| - ColumnarBatch loadDataToColumnBatch() { | |
| - ColumnVector[] vectors = readDataToColumnVectors(); | |
| - int numLiveRows = batchSize; | |
| - | |
| - if (hasIsDeletedColumn) { | |
| - boolean[] isDeleted = buildIsDeleted(vectors); | |
| - readDeletedColumn(vectors, isDeleted); | |
| - } else { | |
| - Pair<int[], Integer> pair = buildRowIdMapping(vectors); | |
| - if (pair != null) { | |
| - int[] rowIdMapping = pair.first(); | |
| - numLiveRows = pair.second(); | |
| - for (int i = 0; i < vectors.length; i++) { | |
| - vectors[i] = new ColumnVectorWithFilter(vectors[i], rowIdMapping); | |
| - } | |
| - } | |
| - } | |
| - | |
| - if (deletes != null && deletes.hasEqDeletes()) { | |
| - vectors = ColumnarBatchUtil.removeExtraColumns(deletes, vectors); | |
| - } | |
| - | |
| - ColumnarBatch batch = new ColumnarBatch(vectors); | |
| - batch.setNumRows(numLiveRows); | |
| - return batch; | |
| - } | |
| - | |
| - private boolean[] buildIsDeleted(ColumnVector[] vectors) { | |
| - return ColumnarBatchUtil.buildIsDeleted(vectors, deletes, rowStartPosInBatch, batchSize); | |
| - } | |
| - | |
| - private Pair<int[], Integer> buildRowIdMapping(ColumnVector[] vectors) { | |
| - return ColumnarBatchUtil.buildRowIdMapping(vectors, deletes, rowStartPosInBatch, batchSize); | |
| - } | |
| - | |
| - ColumnVector[] readDataToColumnVectors() { | |
| - ColumnVector[] columnVectors = new ColumnVector[readers.length]; | |
| - // Fetch rows for all readers in the delegate | |
| - delegate.nextBatch(batchSize); | |
| - for (int i = 0; i < readers.length; i++) { | |
| - columnVectors[i] = readers[i].delegate().currentBatch(); | |
| - } | |
| - | |
| - return columnVectors; | |
| - } | |
| - | |
| - void readDeletedColumn(ColumnVector[] columnVectors, boolean[] isDeleted) { | |
| - for (int i = 0; i < readers.length; i++) { | |
| - if (readers[i] instanceof CometDeleteColumnReader) { | |
| - CometDeleteColumnReader deleteColumnReader = new CometDeleteColumnReader<>(isDeleted); | |
| - deleteColumnReader.setBatchSize(batchSize); | |
| - deleteColumnReader.delegate().readBatch(batchSize); | |
| - columnVectors[i] = deleteColumnReader.delegate().currentBatch(); | |
| - } | |
| - } | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java | |
| deleted file mode 100644 | |
| index c665002e8f..0000000000 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java | |
| +++ /dev/null | |
| @@ -1,65 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.math.BigDecimal; | |
| -import java.nio.ByteBuffer; | |
| -import org.apache.comet.parquet.ConstantColumnReader; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.spark.sql.types.DataType; | |
| -import org.apache.spark.sql.types.DataTypes; | |
| -import org.apache.spark.sql.types.Decimal; | |
| -import org.apache.spark.sql.types.DecimalType; | |
| -import org.apache.spark.unsafe.types.UTF8String; | |
| - | |
| -class CometConstantColumnReader<T> extends CometColumnReader { | |
| - | |
| - CometConstantColumnReader(T value, Types.NestedField field) { | |
| - super(field); | |
| - // use delegate to set constant value on the native side to be consumed by native execution. | |
| - setDelegate( | |
| - new ConstantColumnReader(sparkType(), descriptor(), convertToSparkValue(value), false)); | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - super.setBatchSize(batchSize); | |
| - delegate().setBatchSize(batchSize); | |
| - setInitialized(true); | |
| - } | |
| - | |
| - private Object convertToSparkValue(T value) { | |
| - DataType dataType = sparkType(); | |
| - // Match the value to Spark internal type if necessary | |
| - if (dataType == DataTypes.StringType && value instanceof String) { | |
| - // the internal type for StringType is UTF8String | |
| - return UTF8String.fromString((String) value); | |
| - } else if (dataType instanceof DecimalType && value instanceof BigDecimal) { | |
| - // the internal type for DecimalType is Decimal | |
| - return Decimal.apply((BigDecimal) value); | |
| - } else if (dataType == DataTypes.BinaryType && value instanceof ByteBuffer) { | |
| - // the internal type for DecimalType is byte[] | |
| - // Iceberg default value should always use HeapBufferBuffer, so calling ByteBuffer.array() | |
| - // should be safe. | |
| - return ((java.nio.ByteBuffer) value).array(); | |
| - } else { | |
| - return value; | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java | |
| deleted file mode 100644 | |
| index 4a28fc51da..0000000000 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java | |
| +++ /dev/null | |
| @@ -1,75 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import org.apache.comet.parquet.MetadataColumnReader; | |
| -import org.apache.comet.parquet.Native; | |
| -import org.apache.comet.parquet.TypeUtil; | |
| -import org.apache.iceberg.MetadataColumns; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.spark.sql.types.DataTypes; | |
| -import org.apache.spark.sql.types.Metadata; | |
| -import org.apache.spark.sql.types.StructField; | |
| - | |
| -class CometDeleteColumnReader<T> extends CometColumnReader { | |
| - CometDeleteColumnReader(Types.NestedField field) { | |
| - super(field); | |
| - setDelegate(new DeleteColumnReader()); | |
| - } | |
| - | |
| - CometDeleteColumnReader(boolean[] isDeleted) { | |
| - super(MetadataColumns.IS_DELETED); | |
| - setDelegate(new DeleteColumnReader(isDeleted)); | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - super.setBatchSize(batchSize); | |
| - delegate().setBatchSize(batchSize); | |
| - setInitialized(true); | |
| - } | |
| - | |
| - private static class DeleteColumnReader extends MetadataColumnReader { | |
| - private boolean[] isDeleted; | |
| - | |
| - DeleteColumnReader() { | |
| - super( | |
| - DataTypes.BooleanType, | |
| - TypeUtil.convertToParquet( | |
| - new StructField("_deleted", DataTypes.BooleanType, false, Metadata.empty())), | |
| - false /* useDecimal128 = false */, | |
| - false /* isConstant */); | |
| - this.isDeleted = new boolean[0]; | |
| - } | |
| - | |
| - DeleteColumnReader(boolean[] isDeleted) { | |
| - this(); | |
| - this.isDeleted = isDeleted; | |
| - } | |
| - | |
| - @Override | |
| - public void readBatch(int total) { | |
| - Native.resetBatch(nativeHandle); | |
| - // set isDeleted on the native side to be consumed by native execution | |
| - Native.setIsDeleted(nativeHandle, isDeleted); | |
| - | |
| - super.readBatch(total); | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java | |
| deleted file mode 100644 | |
| index 1949a71798..0000000000 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java | |
| +++ /dev/null | |
| @@ -1,62 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import org.apache.comet.parquet.MetadataColumnReader; | |
| -import org.apache.comet.parquet.Native; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.parquet.column.ColumnDescriptor; | |
| -import org.apache.spark.sql.types.DataTypes; | |
| - | |
| -class CometPositionColumnReader extends CometColumnReader { | |
| - CometPositionColumnReader(Types.NestedField field) { | |
| - super(field); | |
| - setDelegate(new PositionColumnReader(descriptor())); | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - super.setBatchSize(batchSize); | |
| - delegate().setBatchSize(batchSize); | |
| - setInitialized(true); | |
| - } | |
| - | |
| - private static class PositionColumnReader extends MetadataColumnReader { | |
| - /** The current position value of the column that are used to initialize this column reader. */ | |
| - private long position; | |
| - | |
| - PositionColumnReader(ColumnDescriptor descriptor) { | |
| - super( | |
| - DataTypes.LongType, | |
| - descriptor, | |
| - false /* useDecimal128 = false */, | |
| - false /* isConstant */); | |
| - } | |
| - | |
| - @Override | |
| - public void readBatch(int total) { | |
| - Native.resetBatch(nativeHandle); | |
| - // set position on the native side to be consumed by native execution | |
| - Native.setPosition(nativeHandle, position, total); | |
| - position += total; | |
| - | |
| - super.readBatch(total); | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java | |
| deleted file mode 100644 | |
| index d36f1a7274..0000000000 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java | |
| +++ /dev/null | |
| @@ -1,147 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.util.List; | |
| -import java.util.Map; | |
| -import java.util.function.Function; | |
| -import java.util.stream.IntStream; | |
| -import org.apache.iceberg.MetadataColumns; | |
| -import org.apache.iceberg.Schema; | |
| -import org.apache.iceberg.data.DeleteFilter; | |
| -import org.apache.iceberg.parquet.TypeWithSchemaVisitor; | |
| -import org.apache.iceberg.parquet.VectorizedReader; | |
| -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; | |
| -import org.apache.iceberg.relocated.com.google.common.collect.Lists; | |
| -import org.apache.iceberg.relocated.com.google.common.collect.Maps; | |
| -import org.apache.iceberg.spark.SparkSchemaUtil; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.parquet.column.ColumnDescriptor; | |
| -import org.apache.parquet.schema.GroupType; | |
| -import org.apache.parquet.schema.MessageType; | |
| -import org.apache.parquet.schema.PrimitiveType; | |
| -import org.apache.parquet.schema.Type; | |
| -import org.apache.spark.sql.catalyst.InternalRow; | |
| - | |
| -class CometVectorizedReaderBuilder extends TypeWithSchemaVisitor<VectorizedReader<?>> { | |
| - | |
| - private final MessageType parquetSchema; | |
| - private final Schema icebergSchema; | |
| - private final Map<Integer, ?> idToConstant; | |
| - private final Function<List<VectorizedReader<?>>, VectorizedReader<?>> readerFactory; | |
| - private final DeleteFilter<InternalRow> deleteFilter; | |
| - | |
| - CometVectorizedReaderBuilder( | |
| - Schema expectedSchema, | |
| - MessageType parquetSchema, | |
| - Map<Integer, ?> idToConstant, | |
| - Function<List<VectorizedReader<?>>, VectorizedReader<?>> readerFactory, | |
| - DeleteFilter<InternalRow> deleteFilter) { | |
| - this.parquetSchema = parquetSchema; | |
| - this.icebergSchema = expectedSchema; | |
| - this.idToConstant = idToConstant; | |
| - this.readerFactory = readerFactory; | |
| - this.deleteFilter = deleteFilter; | |
| - } | |
| - | |
| - @Override | |
| - public VectorizedReader<?> message( | |
| - Types.StructType expected, MessageType message, List<VectorizedReader<?>> fieldReaders) { | |
| - GroupType groupType = message.asGroupType(); | |
| - Map<Integer, VectorizedReader<?>> readersById = Maps.newHashMap(); | |
| - List<Type> fields = groupType.getFields(); | |
| - | |
| - IntStream.range(0, fields.size()) | |
| - .filter(pos -> fields.get(pos).getId() != null) | |
| - .forEach(pos -> readersById.put(fields.get(pos).getId().intValue(), fieldReaders.get(pos))); | |
| - | |
| - List<Types.NestedField> icebergFields = | |
| - expected != null ? expected.fields() : ImmutableList.of(); | |
| - | |
| - List<VectorizedReader<?>> reorderedFields = | |
| - Lists.newArrayListWithExpectedSize(icebergFields.size()); | |
| - | |
| - for (Types.NestedField field : icebergFields) { | |
| - int id = field.fieldId(); | |
| - VectorizedReader<?> reader = readersById.get(id); | |
| - if (idToConstant.containsKey(id)) { | |
| - CometConstantColumnReader constantReader = | |
| - new CometConstantColumnReader<>(idToConstant.get(id), field); | |
| - reorderedFields.add(constantReader); | |
| - } else if (id == MetadataColumns.ROW_POSITION.fieldId()) { | |
| - reorderedFields.add(new CometPositionColumnReader(field)); | |
| - } else if (id == MetadataColumns.IS_DELETED.fieldId()) { | |
| - CometColumnReader deleteReader = new CometDeleteColumnReader<>(field); | |
| - reorderedFields.add(deleteReader); | |
| - } else if (reader != null) { | |
| - reorderedFields.add(reader); | |
| - } else if (field.initialDefault() != null) { | |
| - CometColumnReader constantReader = | |
| - new CometConstantColumnReader<>(field.initialDefault(), field); | |
| - reorderedFields.add(constantReader); | |
| - } else if (field.isOptional()) { | |
| - CometColumnReader constantReader = new CometConstantColumnReader<>(null, field); | |
| - reorderedFields.add(constantReader); | |
| - } else { | |
| - throw new IllegalArgumentException( | |
| - String.format("Missing required field: %s", field.name())); | |
| - } | |
| - } | |
| - return vectorizedReader(reorderedFields); | |
| - } | |
| - | |
| - protected VectorizedReader<?> vectorizedReader(List<VectorizedReader<?>> reorderedFields) { | |
| - VectorizedReader<?> reader = readerFactory.apply(reorderedFields); | |
| - if (deleteFilter != null) { | |
| - ((CometColumnarBatchReader) reader).setDeleteFilter(deleteFilter); | |
| - } | |
| - return reader; | |
| - } | |
| - | |
| - @Override | |
| - public VectorizedReader<?> struct( | |
| - Types.StructType expected, GroupType groupType, List<VectorizedReader<?>> fieldReaders) { | |
| - if (expected != null) { | |
| - throw new UnsupportedOperationException( | |
| - "Vectorized reads are not supported yet for struct fields"); | |
| - } | |
| - return null; | |
| - } | |
| - | |
| - @Override | |
| - public VectorizedReader<?> primitive( | |
| - org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) { | |
| - | |
| - if (primitive.getId() == null) { | |
| - return null; | |
| - } | |
| - int parquetFieldId = primitive.getId().intValue(); | |
| - ColumnDescriptor desc = parquetSchema.getColumnDescription(currentPath()); | |
| - // Nested types not yet supported for vectorized reads | |
| - if (desc.getMaxRepetitionLevel() > 0) { | |
| - return null; | |
| - } | |
| - Types.NestedField icebergField = icebergSchema.findField(parquetFieldId); | |
| - if (icebergField == null) { | |
| - return null; | |
| - } | |
| - | |
| - return new CometColumnReader(SparkSchemaUtil.convert(icebergField.type()), desc); | |
| - } | |
| -} | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java | |
| index d95baa724b..865cf5085f 100644 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java | |
| +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java | |
| @@ -83,23 +83,6 @@ public class VectorizedSparkParquetReaders { | |
| expectedSchema, fileSchema, idToConstant, deleteFilter, ArrowAllocation.rootAllocator()); | |
| } | |
| - public static CometColumnarBatchReader buildCometReader( | |
| - Schema expectedSchema, | |
| - MessageType fileSchema, | |
| - Map<Integer, ?> idToConstant, | |
| - DeleteFilter<InternalRow> deleteFilter) { | |
| - return (CometColumnarBatchReader) | |
| - TypeWithSchemaVisitor.visit( | |
| - expectedSchema.asStruct(), | |
| - fileSchema, | |
| - new CometVectorizedReaderBuilder( | |
| - expectedSchema, | |
| - fileSchema, | |
| - idToConstant, | |
| - readers -> new CometColumnarBatchReader(readers, expectedSchema), | |
| - deleteFilter)); | |
| - } | |
| - | |
| // enables unsafe memory access to avoid costly checks to see if index is within bounds | |
| // as long as it is not configured explicitly (see BoundsChecking in Arrow) | |
| private static void enableUnsafeMemoryAccess() { | |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java | |
| index a0f45e7610..78e7845911 100644 | |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java | |
| +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java | |
| @@ -34,7 +34,6 @@ import org.apache.iceberg.parquet.Parquet; | |
| import org.apache.iceberg.relocated.com.google.common.collect.Sets; | |
| import org.apache.iceberg.spark.OrcBatchReadConf; | |
| import org.apache.iceberg.spark.ParquetBatchReadConf; | |
| -import org.apache.iceberg.spark.ParquetReaderType; | |
| import org.apache.iceberg.spark.data.vectorized.VectorizedSparkOrcReaders; | |
| import org.apache.iceberg.spark.data.vectorized.VectorizedSparkParquetReaders; | |
| import org.apache.iceberg.types.TypeUtil; | |
| @@ -94,15 +93,9 @@ abstract class BaseBatchReader<T extends ScanTask> extends BaseReader<ColumnarBa | |
| .project(requiredSchema) | |
| .split(start, length) | |
| .createBatchedReaderFunc( | |
| - fileSchema -> { | |
| - if (parquetConf.readerType() == ParquetReaderType.COMET) { | |
| - return VectorizedSparkParquetReaders.buildCometReader( | |
| - requiredSchema, fileSchema, idToConstant, deleteFilter); | |
| - } else { | |
| - return VectorizedSparkParquetReaders.buildReader( | |
| - requiredSchema, fileSchema, idToConstant, deleteFilter); | |
| - } | |
| - }) | |
| + fileSchema -> | |
| + VectorizedSparkParquetReaders.buildReader( | |
| + requiredSchema, fileSchema, idToConstant, deleteFilter)) | |
| .recordsPerBatch(parquetConf.batchSize()) | |
| .filter(residual) | |
| .caseSensitive(caseSensitive()) | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java | |
| index 404ba72846..b375458f05 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java | |
| @@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase | |
| .master("local[2]") | |
| .config("spark.serializer", serializer) | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java | |
| index 9361c63176..16ce1c614f 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java | |
| @@ -69,6 +69,15 @@ public class TestSparkDistributedDataScanDeletes | |
| .master("local[2]") | |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java | |
| index a218f965ea..e30a8d6c6f 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java | |
| @@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles | |
| .master("local[2]") | |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java | |
| index acd4688440..33732e6654 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java | |
| @@ -59,6 +59,15 @@ public class TestSparkDistributedDataScanReporting | |
| .master("local[2]") | |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java | |
| index 3c32b46936..2a39c4605c 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/TestBase.java | |
| @@ -79,6 +79,15 @@ public abstract class TestBase extends SparkTestHelperBase { | |
| .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") | |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) | |
| .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| .getOrCreate(); | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java | |
| index 6647a1b483..aeb16f85ac 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java | |
| @@ -62,6 +62,15 @@ public abstract class ScanTestBase extends AvroDataTestBase { | |
| SparkSession.builder() | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| ScanTestBase.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java | |
| index 24a14bb64d..9329cd99b3 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java | |
| @@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestCompressionSettings.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestCompressionSettings.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @BeforeEach | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java | |
| index 99d3f38ee7..f1a74280dd 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java | |
| @@ -74,7 +74,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestDataSourceOptions.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestDataSourceOptions.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterAll | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java | |
| index a70aeb1ba6..fae2359d01 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java | |
| @@ -123,6 +123,15 @@ public class TestFilteredScan { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| // define UDFs used by partition tests | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java | |
| index c03f7b94ec..30032fa107 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java | |
| @@ -99,6 +99,15 @@ public class TestForwardCompatibility { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java | |
| index f4f57157e4..418364e3da 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java | |
| @@ -51,6 +51,15 @@ public class TestIcebergSpark { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java | |
| index e1402396fa..3092902873 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java | |
| @@ -118,6 +118,15 @@ public class TestPartitionPruning { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java | |
| index b0ad930487..6764bf645a 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java | |
| @@ -112,6 +112,15 @@ public class TestPartitionValues { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java | |
| index 11865db7fc..c7c1e6a02f 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java | |
| @@ -91,6 +91,15 @@ public class TestSnapshotSelection { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | |
| index 833a0b6b5e..eb29e23aac 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | |
| @@ -125,6 +125,15 @@ public class TestSparkDataFile { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java | |
| index 8884654fe0..fde607663e 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java | |
| @@ -100,6 +100,15 @@ public class TestSparkDataWrite { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| @@ -144,7 +153,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); | |
| for (ManifestFile manifest : | |
| SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) { | |
| @@ -213,7 +222,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); | |
| } | |
| @@ -258,7 +267,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); | |
| } | |
| @@ -310,7 +319,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); | |
| } | |
| @@ -352,7 +361,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); | |
| } | |
| @@ -391,7 +400,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); | |
| List<DataFile> files = Lists.newArrayList(); | |
| @@ -459,7 +468,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); | |
| } | |
| @@ -707,7 +716,7 @@ public class TestSparkDataWrite { | |
| // Since write and commit succeeded, the rows should be readable | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual) | |
| .hasSize(records.size() + records2.size()) | |
| .containsExactlyInAnyOrder( | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java | |
| index 596d05d30b..acbacf65e0 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java | |
| @@ -88,6 +88,15 @@ public class TestSparkReadProjection extends TestReadProjection { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| ImmutableMap<String, String> config = | |
| ImmutableMap.of( | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java | |
| index 16fa726032..ce27c5cd4d 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java | |
| @@ -132,7 +132,25 @@ public class TestSparkReaderDeletes extends DeleteReadTests { | |
| .config("spark.ui.liveUpdate.period", 0) | |
| .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") | |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| catalog = | |
| @@ -200,7 +218,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { | |
| } | |
| protected boolean countDeletes() { | |
| - return true; | |
| + // TODO: Enable once iceberg-rust exposes delete count metrics to Comet | |
| + return false; | |
| } | |
| @Override | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java | |
| index baf7fa8f88..fb677d7114 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java | |
| @@ -182,7 +182,25 @@ public class TestSparkReaderWithBloomFilter { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| catalog = | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java | |
| index dc4fc7e187..2ab6dd12c2 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java | |
| @@ -69,6 +69,15 @@ public class TestStructuredStreaming { | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| .config("spark.sql.shuffle.partitions", 4) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java | |
| index 8b1e3fbfc7..c519a2be0f 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java | |
| @@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestTimestampWithoutZone.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestTimestampWithoutZone.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterAll | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java | |
| index c3fac70dd3..d3ffd034d2 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java | |
| @@ -84,6 +84,15 @@ public class TestWriteMetricsConfig { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| } | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java | |
| index 5ce56b4fec..3b614e3fc9 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java | |
| @@ -63,7 +63,25 @@ public class TestAggregatePushDown extends CatalogTestBase { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.sql.iceberg.aggregate_pushdown", "true") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| TestBase.catalog = | |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java | |
| index 9d2ce2b388..5e23368848 100644 | |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java | |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java | |
| @@ -598,9 +598,7 @@ public class TestFilterPushDown extends TestBaseWithCatalog { | |
| String planAsString = sparkPlan.toString().replaceAll("#(\\d+L?)", ""); | |
| if (sparkFilter != null) { | |
| - assertThat(planAsString) | |
| - .as("Post scan filter should match") | |
| - .contains("Filter (" + sparkFilter + ")"); | |
| + assertThat(planAsString).as("Post scan filter should match").contains("CometFilter"); | |
| } else { | |
| assertThat(planAsString).as("Should be no post scan filter").doesNotContain("Filter ("); | |
| } | |
| diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle | |
| index 69700d8436..49ea338a45 100644 | |
| --- a/spark/v3.5/build.gradle | |
| +++ b/spark/v3.5/build.gradle | |
| @@ -264,6 +264,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio | |
| integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') | |
| integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts') | |
| integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts') | |
| + integrationImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" | |
| // runtime dependencies for running Hive Catalog based integration test | |
| integrationRuntimeOnly project(':iceberg-hive-metastore') | |
| diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java | |
| index 4c1a509591..3b4e44cc13 100644 | |
| --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java | |
| +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java | |
| @@ -59,6 +59,15 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { | |
| .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") | |
| .config( | |
| SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| .getOrCreate(); | |
| diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java | |
| index 893f9931cf..17e5fef217 100644 | |
| --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java | |
| +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java | |
| @@ -460,7 +460,8 @@ public abstract class SparkRowLevelOperationsTestBase extends ExtensionsTestBase | |
| protected void assertAllBatchScansVectorized(SparkPlan plan) { | |
| List<SparkPlan> batchScans = SparkPlanUtil.collectBatchScans(plan); | |
| - assertThat(batchScans).hasSizeGreaterThan(0).allMatch(SparkPlan::supportsColumnar); | |
| + // When Comet is enabled, its native scan replaces BatchScanExec nodes entirely | |
| + assertThat(batchScans).allMatch(SparkPlan::supportsColumnar); | |
| } | |
| protected void createTableWithDeleteGranularity( | |
| diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java | |
| index ecf9e6f8a5..beca5f8521 100644 | |
| --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java | |
| +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java | |
| @@ -56,6 +56,15 @@ public class TestCallStatementParser { | |
| .master("local[2]") | |
| .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) | |
| .config("spark.extra.prop", "value") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| TestCallStatementParser.parser = spark.sessionState().sqlParser(); | |
| } | |
| diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java | |
| index 8cb92224fc..777b9bb1dd 100644 | |
| --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java | |
| +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSystemFunctionPushDownInRowLevelOperations.java | |
| @@ -38,6 +38,7 @@ import org.apache.spark.sql.catalyst.expressions.ApplyFunctionExpression; | |
| import org.apache.spark.sql.catalyst.expressions.Expression; | |
| import org.apache.spark.sql.catalyst.expressions.objects.StaticInvoke; | |
| import org.apache.spark.sql.execution.CommandResultExec; | |
| +import org.apache.spark.sql.execution.SparkPlan; | |
| import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec; | |
| import org.junit.jupiter.api.AfterEach; | |
| import org.junit.jupiter.api.BeforeEach; | |
| @@ -313,9 +314,13 @@ public class TestSystemFunctionPushDownInRowLevelOperations extends ExtensionsTe | |
| private List<Expression> executeAndCollectFunctionCalls(String query, Object... args) { | |
| CommandResultExec command = (CommandResultExec) executeAndKeepPlan(query, args); | |
| - V2TableWriteExec write = (V2TableWriteExec) command.commandPhysicalPlan(); | |
| + // Comet's split-operator Iceberg write plans the command as IcebergCommitExec, which is not | |
| + // a V2TableWriteExec; collect over the whole command plan in that case. | |
| + SparkPlan plan = command.commandPhysicalPlan(); | |
| + SparkPlan queryPlan = | |
| + plan instanceof V2TableWriteExec ? ((V2TableWriteExec) plan).query() : plan; | |
| return SparkPlanUtil.collectExprs( | |
| - write.query(), | |
| + queryPlan, | |
| expr -> expr instanceof StaticInvoke || expr instanceof ApplyFunctionExpression); | |
| } | |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java | |
| index 64edb1002e..57767da086 100644 | |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java | |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java | |
| @@ -179,6 +179,15 @@ public class DeleteOrphanFilesBenchmark { | |
| .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) | |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") | |
| .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .master("local"); | |
| spark = builder.getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java | |
| index 77b79384a6..8708ffa72a 100644 | |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java | |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java | |
| @@ -392,6 +392,15 @@ public class IcebergSortCompactionBenchmark { | |
| "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") | |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") | |
| .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .master("local[*]"); | |
| spark = builder.getOrCreate(); | |
| Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); | |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java | |
| index c6794e43c6..4c1a165a44 100644 | |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java | |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java | |
| @@ -239,6 +239,15 @@ public class DVReaderBenchmark { | |
| .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) | |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") | |
| .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .master("local[*]") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java | |
| index ac74fb5a10..4aa5d5b361 100644 | |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java | |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java | |
| @@ -223,6 +223,15 @@ public class DVWriterBenchmark { | |
| .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) | |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") | |
| .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .master("local[*]") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java | |
| index 68c537e34a..2e421c7706 100644 | |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java | |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java | |
| @@ -94,7 +94,18 @@ public abstract class IcebergSourceBenchmark { | |
| } | |
| protected void setupSpark(boolean enableDictionaryEncoding) { | |
| - SparkSession.Builder builder = SparkSession.builder().config("spark.ui.enabled", false); | |
| + SparkSession.Builder builder = | |
| + SparkSession.builder() | |
| + .config("spark.ui.enabled", false) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g"); | |
| if (!enableDictionaryEncoding) { | |
| builder | |
| .config("parquet.dictionary.page.size", "1") | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java | |
| deleted file mode 100644 | |
| index 81b7d83a70..0000000000 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java | |
| +++ /dev/null | |
| @@ -1,140 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.io.IOException; | |
| -import org.apache.comet.CometSchemaImporter; | |
| -import org.apache.comet.parquet.AbstractColumnReader; | |
| -import org.apache.comet.parquet.ColumnReader; | |
| -import org.apache.comet.parquet.TypeUtil; | |
| -import org.apache.comet.parquet.Utils; | |
| -import org.apache.comet.shaded.arrow.memory.RootAllocator; | |
| -import org.apache.iceberg.parquet.VectorizedReader; | |
| -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; | |
| -import org.apache.iceberg.spark.SparkSchemaUtil; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.parquet.column.ColumnDescriptor; | |
| -import org.apache.parquet.column.page.PageReader; | |
| -import org.apache.spark.sql.types.DataType; | |
| -import org.apache.spark.sql.types.Metadata; | |
| -import org.apache.spark.sql.types.StructField; | |
| -import org.apache.spark.sql.vectorized.ColumnVector; | |
| - | |
| -class CometColumnReader implements VectorizedReader<ColumnVector> { | |
| - // use the Comet default batch size | |
| - public static final int DEFAULT_BATCH_SIZE = 8192; | |
| - | |
| - private final ColumnDescriptor descriptor; | |
| - private final DataType sparkType; | |
| - | |
| - // The delegated ColumnReader from Comet side | |
| - private AbstractColumnReader delegate; | |
| - private boolean initialized = false; | |
| - private int batchSize = DEFAULT_BATCH_SIZE; | |
| - private CometSchemaImporter importer; | |
| - | |
| - CometColumnReader(DataType sparkType, ColumnDescriptor descriptor) { | |
| - this.sparkType = sparkType; | |
| - this.descriptor = descriptor; | |
| - } | |
| - | |
| - CometColumnReader(Types.NestedField field) { | |
| - DataType dataType = SparkSchemaUtil.convert(field.type()); | |
| - StructField structField = new StructField(field.name(), dataType, false, Metadata.empty()); | |
| - this.sparkType = dataType; | |
| - this.descriptor = TypeUtil.convertToParquet(structField); | |
| - } | |
| - | |
| - public AbstractColumnReader delegate() { | |
| - return delegate; | |
| - } | |
| - | |
| - void setDelegate(AbstractColumnReader delegate) { | |
| - this.delegate = delegate; | |
| - } | |
| - | |
| - void setInitialized(boolean initialized) { | |
| - this.initialized = initialized; | |
| - } | |
| - | |
| - public int batchSize() { | |
| - return batchSize; | |
| - } | |
| - | |
| - /** | |
| - * This method is to initialized/reset the CometColumnReader. This needs to be called for each row | |
| - * group after readNextRowGroup, so a new dictionary encoding can be set for each of the new row | |
| - * groups. | |
| - */ | |
| - public void reset() { | |
| - if (importer != null) { | |
| - importer.close(); | |
| - } | |
| - | |
| - if (delegate != null) { | |
| - delegate.close(); | |
| - } | |
| - | |
| - this.importer = new CometSchemaImporter(new RootAllocator()); | |
| - this.delegate = Utils.getColumnReader(sparkType, descriptor, importer, batchSize, false, false); | |
| - this.initialized = true; | |
| - } | |
| - | |
| - public ColumnDescriptor descriptor() { | |
| - return descriptor; | |
| - } | |
| - | |
| - /** Returns the Spark data type for this column. */ | |
| - public DataType sparkType() { | |
| - return sparkType; | |
| - } | |
| - | |
| - /** | |
| - * Set the page reader to be 'pageReader'. | |
| - * | |
| - * <p>NOTE: this should be called before reading a new Parquet column chunk, and after {@link | |
| - * CometColumnReader#reset} is called. | |
| - */ | |
| - public void setPageReader(PageReader pageReader) throws IOException { | |
| - Preconditions.checkState(initialized, "Invalid state: 'reset' should be called first"); | |
| - ((ColumnReader) delegate).setPageReader(pageReader); | |
| - } | |
| - | |
| - @Override | |
| - public void close() { | |
| - // close resources on native side | |
| - if (importer != null) { | |
| - importer.close(); | |
| - } | |
| - | |
| - if (delegate != null) { | |
| - delegate.close(); | |
| - } | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int size) { | |
| - this.batchSize = size; | |
| - } | |
| - | |
| - @Override | |
| - public ColumnVector read(ColumnVector reuse, int numRowsToRead) { | |
| - throw new UnsupportedOperationException("Not supported"); | |
| - } | |
| -} | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java | |
| deleted file mode 100644 | |
| index 04ac69476a..0000000000 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java | |
| +++ /dev/null | |
| @@ -1,197 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.io.IOException; | |
| -import java.io.UncheckedIOException; | |
| -import java.util.List; | |
| -import java.util.Map; | |
| -import org.apache.comet.parquet.AbstractColumnReader; | |
| -import org.apache.comet.parquet.BatchReader; | |
| -import org.apache.iceberg.Schema; | |
| -import org.apache.iceberg.data.DeleteFilter; | |
| -import org.apache.iceberg.parquet.VectorizedReader; | |
| -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; | |
| -import org.apache.iceberg.spark.SparkSchemaUtil; | |
| -import org.apache.iceberg.util.Pair; | |
| -import org.apache.parquet.column.page.PageReadStore; | |
| -import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; | |
| -import org.apache.parquet.hadoop.metadata.ColumnPath; | |
| -import org.apache.spark.sql.catalyst.InternalRow; | |
| -import org.apache.spark.sql.vectorized.ColumnVector; | |
| -import org.apache.spark.sql.vectorized.ColumnarBatch; | |
| - | |
| -/** | |
| - * {@link VectorizedReader} that returns Spark's {@link ColumnarBatch} to support Spark's vectorized | |
| - * read path. The {@link ColumnarBatch} returned is created by passing in the Arrow vectors | |
| - * populated via delegated read calls to {@link CometColumnReader VectorReader(s)}. | |
| - */ | |
| -@SuppressWarnings("checkstyle:VisibilityModifier") | |
| -class CometColumnarBatchReader implements VectorizedReader<ColumnarBatch> { | |
| - | |
| - private final CometColumnReader[] readers; | |
| - private final boolean hasIsDeletedColumn; | |
| - | |
| - // The delegated BatchReader on the Comet side does the real work of loading a batch of rows. | |
| - // The Comet BatchReader contains an array of ColumnReader. There is no need to explicitly call | |
| - // ColumnReader.readBatch; instead, BatchReader.nextBatch will be called, which underneath calls | |
| - // ColumnReader.readBatch. The only exception is DeleteColumnReader, because at the time of | |
| - // calling BatchReader.nextBatch, the isDeleted value is not yet available, so | |
| - // DeleteColumnReader.readBatch must be called explicitly later, after the isDeleted value is | |
| - // available. | |
| - private final BatchReader delegate; | |
| - private DeleteFilter<InternalRow> deletes = null; | |
| - private long rowStartPosInBatch = 0; | |
| - | |
| - CometColumnarBatchReader(List<VectorizedReader<?>> readers, Schema schema) { | |
| - this.readers = | |
| - readers.stream().map(CometColumnReader.class::cast).toArray(CometColumnReader[]::new); | |
| - this.hasIsDeletedColumn = | |
| - readers.stream().anyMatch(reader -> reader instanceof CometDeleteColumnReader); | |
| - | |
| - AbstractColumnReader[] abstractColumnReaders = new AbstractColumnReader[readers.size()]; | |
| - this.delegate = new BatchReader(abstractColumnReaders); | |
| - delegate.setSparkSchema(SparkSchemaUtil.convert(schema)); | |
| - } | |
| - | |
| - @Override | |
| - public void setRowGroupInfo( | |
| - PageReadStore pageStore, Map<ColumnPath, ColumnChunkMetaData> metaData) { | |
| - for (int i = 0; i < readers.length; i++) { | |
| - try { | |
| - if (!(readers[i] instanceof CometConstantColumnReader) | |
| - && !(readers[i] instanceof CometPositionColumnReader) | |
| - && !(readers[i] instanceof CometDeleteColumnReader)) { | |
| - readers[i].reset(); | |
| - readers[i].setPageReader(pageStore.getPageReader(readers[i].descriptor())); | |
| - } | |
| - } catch (IOException e) { | |
| - throw new UncheckedIOException("Failed to setRowGroupInfo for Comet vectorization", e); | |
| - } | |
| - } | |
| - | |
| - for (int i = 0; i < readers.length; i++) { | |
| - delegate.getColumnReaders()[i] = this.readers[i].delegate(); | |
| - } | |
| - | |
| - this.rowStartPosInBatch = | |
| - pageStore | |
| - .getRowIndexOffset() | |
| - .orElseThrow( | |
| - () -> | |
| - new IllegalArgumentException( | |
| - "PageReadStore does not contain row index offset")); | |
| - } | |
| - | |
| - public void setDeleteFilter(DeleteFilter<InternalRow> deleteFilter) { | |
| - this.deletes = deleteFilter; | |
| - } | |
| - | |
| - @Override | |
| - public final ColumnarBatch read(ColumnarBatch reuse, int numRowsToRead) { | |
| - ColumnarBatch columnarBatch = new ColumnBatchLoader(numRowsToRead).loadDataToColumnBatch(); | |
| - rowStartPosInBatch += numRowsToRead; | |
| - return columnarBatch; | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - for (CometColumnReader reader : readers) { | |
| - if (reader != null) { | |
| - reader.setBatchSize(batchSize); | |
| - } | |
| - } | |
| - } | |
| - | |
| - @Override | |
| - public void close() { | |
| - for (CometColumnReader reader : readers) { | |
| - if (reader != null) { | |
| - reader.close(); | |
| - } | |
| - } | |
| - } | |
| - | |
| - private class ColumnBatchLoader { | |
| - private final int batchSize; | |
| - | |
| - ColumnBatchLoader(int numRowsToRead) { | |
| - Preconditions.checkArgument( | |
| - numRowsToRead > 0, "Invalid number of rows to read: %s", numRowsToRead); | |
| - this.batchSize = numRowsToRead; | |
| - } | |
| - | |
| - ColumnarBatch loadDataToColumnBatch() { | |
| - ColumnVector[] vectors = readDataToColumnVectors(); | |
| - int numLiveRows = batchSize; | |
| - | |
| - if (hasIsDeletedColumn) { | |
| - boolean[] isDeleted = buildIsDeleted(vectors); | |
| - readDeletedColumn(vectors, isDeleted); | |
| - } else { | |
| - Pair<int[], Integer> pair = buildRowIdMapping(vectors); | |
| - if (pair != null) { | |
| - int[] rowIdMapping = pair.first(); | |
| - numLiveRows = pair.second(); | |
| - for (int i = 0; i < vectors.length; i++) { | |
| - vectors[i] = new ColumnVectorWithFilter(vectors[i], rowIdMapping); | |
| - } | |
| - } | |
| - } | |
| - | |
| - if (deletes != null && deletes.hasEqDeletes()) { | |
| - vectors = ColumnarBatchUtil.removeExtraColumns(deletes, vectors); | |
| - } | |
| - | |
| - ColumnarBatch batch = new ColumnarBatch(vectors); | |
| - batch.setNumRows(numLiveRows); | |
| - return batch; | |
| - } | |
| - | |
| - private boolean[] buildIsDeleted(ColumnVector[] vectors) { | |
| - return ColumnarBatchUtil.buildIsDeleted(vectors, deletes, rowStartPosInBatch, batchSize); | |
| - } | |
| - | |
| - private Pair<int[], Integer> buildRowIdMapping(ColumnVector[] vectors) { | |
| - return ColumnarBatchUtil.buildRowIdMapping(vectors, deletes, rowStartPosInBatch, batchSize); | |
| - } | |
| - | |
| - ColumnVector[] readDataToColumnVectors() { | |
| - ColumnVector[] columnVectors = new ColumnVector[readers.length]; | |
| - // Fetch rows for all readers in the delegate | |
| - delegate.nextBatch(batchSize); | |
| - for (int i = 0; i < readers.length; i++) { | |
| - columnVectors[i] = readers[i].delegate().currentBatch(); | |
| - } | |
| - | |
| - return columnVectors; | |
| - } | |
| - | |
| - void readDeletedColumn(ColumnVector[] columnVectors, boolean[] isDeleted) { | |
| - for (int i = 0; i < readers.length; i++) { | |
| - if (readers[i] instanceof CometDeleteColumnReader) { | |
| - CometDeleteColumnReader deleteColumnReader = new CometDeleteColumnReader<>(isDeleted); | |
| - deleteColumnReader.setBatchSize(batchSize); | |
| - deleteColumnReader.delegate().readBatch(batchSize); | |
| - columnVectors[i] = deleteColumnReader.delegate().currentBatch(); | |
| - } | |
| - } | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java | |
| deleted file mode 100644 | |
| index 047c96314b..0000000000 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java | |
| +++ /dev/null | |
| @@ -1,65 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.math.BigDecimal; | |
| -import java.nio.ByteBuffer; | |
| -import org.apache.comet.parquet.ConstantColumnReader; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.spark.sql.types.DataType; | |
| -import org.apache.spark.sql.types.DataTypes; | |
| -import org.apache.spark.sql.types.Decimal; | |
| -import org.apache.spark.sql.types.DecimalType; | |
| -import org.apache.spark.unsafe.types.UTF8String; | |
| - | |
| -class CometConstantColumnReader<T> extends CometColumnReader { | |
| - | |
| - CometConstantColumnReader(T value, Types.NestedField field) { | |
| - super(field); | |
| - // use delegate to set constant value on the native side to be consumed by native execution. | |
| - setDelegate( | |
| - new ConstantColumnReader(sparkType(), descriptor(), convertToSparkValue(value), false)); | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - super.setBatchSize(batchSize); | |
| - delegate().setBatchSize(batchSize); | |
| - setInitialized(true); | |
| - } | |
| - | |
| - private Object convertToSparkValue(T value) { | |
| - DataType dataType = sparkType(); | |
| - // Match the value to Spark internal type if necessary | |
| - if (dataType == DataTypes.StringType && value instanceof String) { | |
| - // the internal type for StringType is UTF8String | |
| - return UTF8String.fromString((String) value); | |
| - } else if (dataType instanceof DecimalType && value instanceof BigDecimal) { | |
| - // the internal type for DecimalType is Decimal | |
| - return Decimal.apply((BigDecimal) value); | |
| - } else if (dataType == DataTypes.BinaryType && value instanceof ByteBuffer) { | |
| - // the internal type for DecimalType is byte[] | |
| - // Iceberg default value should always use HeapBufferBuffer, so calling ByteBuffer.array() | |
| - // should be safe. | |
| - return ((ByteBuffer) value).array(); | |
| - } else { | |
| - return value; | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java | |
| deleted file mode 100644 | |
| index 6235bfe486..0000000000 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java | |
| +++ /dev/null | |
| @@ -1,75 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import org.apache.comet.parquet.MetadataColumnReader; | |
| -import org.apache.comet.parquet.Native; | |
| -import org.apache.comet.parquet.TypeUtil; | |
| -import org.apache.iceberg.MetadataColumns; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.spark.sql.types.DataTypes; | |
| -import org.apache.spark.sql.types.Metadata; | |
| -import org.apache.spark.sql.types.StructField; | |
| - | |
| -class CometDeleteColumnReader<T> extends CometColumnReader { | |
| - CometDeleteColumnReader(Types.NestedField field) { | |
| - super(field); | |
| - setDelegate(new DeleteColumnReader()); | |
| - } | |
| - | |
| - CometDeleteColumnReader(boolean[] isDeleted) { | |
| - super(MetadataColumns.IS_DELETED); | |
| - setDelegate(new DeleteColumnReader(isDeleted)); | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - super.setBatchSize(batchSize); | |
| - delegate().setBatchSize(batchSize); | |
| - setInitialized(true); | |
| - } | |
| - | |
| - private static class DeleteColumnReader extends MetadataColumnReader { | |
| - private boolean[] isDeleted; | |
| - | |
| - DeleteColumnReader() { | |
| - super( | |
| - DataTypes.BooleanType, | |
| - TypeUtil.convertToParquet( | |
| - new StructField("_deleted", DataTypes.BooleanType, false, Metadata.empty())), | |
| - false /* useDecimal128 = false */, | |
| - false /* isConstant = false */); | |
| - this.isDeleted = new boolean[0]; | |
| - } | |
| - | |
| - DeleteColumnReader(boolean[] isDeleted) { | |
| - this(); | |
| - this.isDeleted = isDeleted; | |
| - } | |
| - | |
| - @Override | |
| - public void readBatch(int total) { | |
| - Native.resetBatch(nativeHandle); | |
| - // set isDeleted on the native side to be consumed by native execution | |
| - Native.setIsDeleted(nativeHandle, isDeleted); | |
| - | |
| - super.readBatch(total); | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java | |
| deleted file mode 100644 | |
| index bcc0e514c2..0000000000 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java | |
| +++ /dev/null | |
| @@ -1,62 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import org.apache.comet.parquet.MetadataColumnReader; | |
| -import org.apache.comet.parquet.Native; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.parquet.column.ColumnDescriptor; | |
| -import org.apache.spark.sql.types.DataTypes; | |
| - | |
| -class CometPositionColumnReader extends CometColumnReader { | |
| - CometPositionColumnReader(Types.NestedField field) { | |
| - super(field); | |
| - setDelegate(new PositionColumnReader(descriptor())); | |
| - } | |
| - | |
| - @Override | |
| - public void setBatchSize(int batchSize) { | |
| - super.setBatchSize(batchSize); | |
| - delegate().setBatchSize(batchSize); | |
| - setInitialized(true); | |
| - } | |
| - | |
| - private static class PositionColumnReader extends MetadataColumnReader { | |
| - /** The current position value of the column that are used to initialize this column reader. */ | |
| - private long position; | |
| - | |
| - PositionColumnReader(ColumnDescriptor descriptor) { | |
| - super( | |
| - DataTypes.LongType, | |
| - descriptor, | |
| - false /* useDecimal128 = false */, | |
| - false /* isConstant = false */); | |
| - } | |
| - | |
| - @Override | |
| - public void readBatch(int total) { | |
| - Native.resetBatch(nativeHandle); | |
| - // set position on the native side to be consumed by native execution | |
| - Native.setPosition(nativeHandle, position, total); | |
| - position += total; | |
| - | |
| - super.readBatch(total); | |
| - } | |
| - } | |
| -} | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java | |
| deleted file mode 100644 | |
| index d36f1a7274..0000000000 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java | |
| +++ /dev/null | |
| @@ -1,147 +0,0 @@ | |
| -/* | |
| - * Licensed to the Apache Software Foundation (ASF) under one | |
| - * or more contributor license agreements. See the NOTICE file | |
| - * distributed with this work for additional information | |
| - * regarding copyright ownership. The ASF licenses this file | |
| - * to you under the Apache License, Version 2.0 (the | |
| - * "License"); you may not use this file except in compliance | |
| - * with the License. You may obtain a copy of the License at | |
| - * | |
| - * http://www.apache.org/licenses/LICENSE-2.0 | |
| - * | |
| - * Unless required by applicable law or agreed to in writing, | |
| - * software distributed under the License is distributed on an | |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| - * KIND, either express or implied. See the License for the | |
| - * specific language governing permissions and limitations | |
| - * under the License. | |
| - */ | |
| -package org.apache.iceberg.spark.data.vectorized; | |
| - | |
| -import java.util.List; | |
| -import java.util.Map; | |
| -import java.util.function.Function; | |
| -import java.util.stream.IntStream; | |
| -import org.apache.iceberg.MetadataColumns; | |
| -import org.apache.iceberg.Schema; | |
| -import org.apache.iceberg.data.DeleteFilter; | |
| -import org.apache.iceberg.parquet.TypeWithSchemaVisitor; | |
| -import org.apache.iceberg.parquet.VectorizedReader; | |
| -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; | |
| -import org.apache.iceberg.relocated.com.google.common.collect.Lists; | |
| -import org.apache.iceberg.relocated.com.google.common.collect.Maps; | |
| -import org.apache.iceberg.spark.SparkSchemaUtil; | |
| -import org.apache.iceberg.types.Types; | |
| -import org.apache.parquet.column.ColumnDescriptor; | |
| -import org.apache.parquet.schema.GroupType; | |
| -import org.apache.parquet.schema.MessageType; | |
| -import org.apache.parquet.schema.PrimitiveType; | |
| -import org.apache.parquet.schema.Type; | |
| -import org.apache.spark.sql.catalyst.InternalRow; | |
| - | |
| -class CometVectorizedReaderBuilder extends TypeWithSchemaVisitor<VectorizedReader<?>> { | |
| - | |
| - private final MessageType parquetSchema; | |
| - private final Schema icebergSchema; | |
| - private final Map<Integer, ?> idToConstant; | |
| - private final Function<List<VectorizedReader<?>>, VectorizedReader<?>> readerFactory; | |
| - private final DeleteFilter<InternalRow> deleteFilter; | |
| - | |
| - CometVectorizedReaderBuilder( | |
| - Schema expectedSchema, | |
| - MessageType parquetSchema, | |
| - Map<Integer, ?> idToConstant, | |
| - Function<List<VectorizedReader<?>>, VectorizedReader<?>> readerFactory, | |
| - DeleteFilter<InternalRow> deleteFilter) { | |
| - this.parquetSchema = parquetSchema; | |
| - this.icebergSchema = expectedSchema; | |
| - this.idToConstant = idToConstant; | |
| - this.readerFactory = readerFactory; | |
| - this.deleteFilter = deleteFilter; | |
| - } | |
| - | |
| - @Override | |
| - public VectorizedReader<?> message( | |
| - Types.StructType expected, MessageType message, List<VectorizedReader<?>> fieldReaders) { | |
| - GroupType groupType = message.asGroupType(); | |
| - Map<Integer, VectorizedReader<?>> readersById = Maps.newHashMap(); | |
| - List<Type> fields = groupType.getFields(); | |
| - | |
| - IntStream.range(0, fields.size()) | |
| - .filter(pos -> fields.get(pos).getId() != null) | |
| - .forEach(pos -> readersById.put(fields.get(pos).getId().intValue(), fieldReaders.get(pos))); | |
| - | |
| - List<Types.NestedField> icebergFields = | |
| - expected != null ? expected.fields() : ImmutableList.of(); | |
| - | |
| - List<VectorizedReader<?>> reorderedFields = | |
| - Lists.newArrayListWithExpectedSize(icebergFields.size()); | |
| - | |
| - for (Types.NestedField field : icebergFields) { | |
| - int id = field.fieldId(); | |
| - VectorizedReader<?> reader = readersById.get(id); | |
| - if (idToConstant.containsKey(id)) { | |
| - CometConstantColumnReader constantReader = | |
| - new CometConstantColumnReader<>(idToConstant.get(id), field); | |
| - reorderedFields.add(constantReader); | |
| - } else if (id == MetadataColumns.ROW_POSITION.fieldId()) { | |
| - reorderedFields.add(new CometPositionColumnReader(field)); | |
| - } else if (id == MetadataColumns.IS_DELETED.fieldId()) { | |
| - CometColumnReader deleteReader = new CometDeleteColumnReader<>(field); | |
| - reorderedFields.add(deleteReader); | |
| - } else if (reader != null) { | |
| - reorderedFields.add(reader); | |
| - } else if (field.initialDefault() != null) { | |
| - CometColumnReader constantReader = | |
| - new CometConstantColumnReader<>(field.initialDefault(), field); | |
| - reorderedFields.add(constantReader); | |
| - } else if (field.isOptional()) { | |
| - CometColumnReader constantReader = new CometConstantColumnReader<>(null, field); | |
| - reorderedFields.add(constantReader); | |
| - } else { | |
| - throw new IllegalArgumentException( | |
| - String.format("Missing required field: %s", field.name())); | |
| - } | |
| - } | |
| - return vectorizedReader(reorderedFields); | |
| - } | |
| - | |
| - protected VectorizedReader<?> vectorizedReader(List<VectorizedReader<?>> reorderedFields) { | |
| - VectorizedReader<?> reader = readerFactory.apply(reorderedFields); | |
| - if (deleteFilter != null) { | |
| - ((CometColumnarBatchReader) reader).setDeleteFilter(deleteFilter); | |
| - } | |
| - return reader; | |
| - } | |
| - | |
| - @Override | |
| - public VectorizedReader<?> struct( | |
| - Types.StructType expected, GroupType groupType, List<VectorizedReader<?>> fieldReaders) { | |
| - if (expected != null) { | |
| - throw new UnsupportedOperationException( | |
| - "Vectorized reads are not supported yet for struct fields"); | |
| - } | |
| - return null; | |
| - } | |
| - | |
| - @Override | |
| - public VectorizedReader<?> primitive( | |
| - org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) { | |
| - | |
| - if (primitive.getId() == null) { | |
| - return null; | |
| - } | |
| - int parquetFieldId = primitive.getId().intValue(); | |
| - ColumnDescriptor desc = parquetSchema.getColumnDescription(currentPath()); | |
| - // Nested types not yet supported for vectorized reads | |
| - if (desc.getMaxRepetitionLevel() > 0) { | |
| - return null; | |
| - } | |
| - Types.NestedField icebergField = icebergSchema.findField(parquetFieldId); | |
| - if (icebergField == null) { | |
| - return null; | |
| - } | |
| - | |
| - return new CometColumnReader(SparkSchemaUtil.convert(icebergField.type()), desc); | |
| - } | |
| -} | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java | |
| index d95baa724b..865cf5085f 100644 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java | |
| +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java | |
| @@ -83,23 +83,6 @@ public class VectorizedSparkParquetReaders { | |
| expectedSchema, fileSchema, idToConstant, deleteFilter, ArrowAllocation.rootAllocator()); | |
| } | |
| - public static CometColumnarBatchReader buildCometReader( | |
| - Schema expectedSchema, | |
| - MessageType fileSchema, | |
| - Map<Integer, ?> idToConstant, | |
| - DeleteFilter<InternalRow> deleteFilter) { | |
| - return (CometColumnarBatchReader) | |
| - TypeWithSchemaVisitor.visit( | |
| - expectedSchema.asStruct(), | |
| - fileSchema, | |
| - new CometVectorizedReaderBuilder( | |
| - expectedSchema, | |
| - fileSchema, | |
| - idToConstant, | |
| - readers -> new CometColumnarBatchReader(readers, expectedSchema), | |
| - deleteFilter)); | |
| - } | |
| - | |
| // enables unsafe memory access to avoid costly checks to see if index is within bounds | |
| // as long as it is not configured explicitly (see BoundsChecking in Arrow) | |
| private static void enableUnsafeMemoryAccess() { | |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java | |
| index a0f45e7610..78e7845911 100644 | |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java | |
| +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java | |
| @@ -34,7 +34,6 @@ import org.apache.iceberg.parquet.Parquet; | |
| import org.apache.iceberg.relocated.com.google.common.collect.Sets; | |
| import org.apache.iceberg.spark.OrcBatchReadConf; | |
| import org.apache.iceberg.spark.ParquetBatchReadConf; | |
| -import org.apache.iceberg.spark.ParquetReaderType; | |
| import org.apache.iceberg.spark.data.vectorized.VectorizedSparkOrcReaders; | |
| import org.apache.iceberg.spark.data.vectorized.VectorizedSparkParquetReaders; | |
| import org.apache.iceberg.types.TypeUtil; | |
| @@ -94,15 +93,9 @@ abstract class BaseBatchReader<T extends ScanTask> extends BaseReader<ColumnarBa | |
| .project(requiredSchema) | |
| .split(start, length) | |
| .createBatchedReaderFunc( | |
| - fileSchema -> { | |
| - if (parquetConf.readerType() == ParquetReaderType.COMET) { | |
| - return VectorizedSparkParquetReaders.buildCometReader( | |
| - requiredSchema, fileSchema, idToConstant, deleteFilter); | |
| - } else { | |
| - return VectorizedSparkParquetReaders.buildReader( | |
| - requiredSchema, fileSchema, idToConstant, deleteFilter); | |
| - } | |
| - }) | |
| + fileSchema -> | |
| + VectorizedSparkParquetReaders.buildReader( | |
| + requiredSchema, fileSchema, idToConstant, deleteFilter)) | |
| .recordsPerBatch(parquetConf.batchSize()) | |
| .filter(residual) | |
| .caseSensitive(caseSensitive()) | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java | |
| index 404ba72846..b375458f05 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java | |
| @@ -90,6 +90,15 @@ public abstract class SparkDistributedDataScanTestBase | |
| .master("local[2]") | |
| .config("spark.serializer", serializer) | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java | |
| index 659507e4c5..2a3a87f079 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java | |
| @@ -73,6 +73,15 @@ public class TestSparkDistributedDataScanDeletes | |
| .master("local[2]") | |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java | |
| index a218f965ea..e30a8d6c6f 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java | |
| @@ -62,6 +62,15 @@ public class TestSparkDistributedDataScanFilterFiles | |
| .master("local[2]") | |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java | |
| index 2665d7ba8d..c24c82341a 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java | |
| @@ -63,6 +63,15 @@ public class TestSparkDistributedDataScanReporting | |
| .master("local[2]") | |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") | |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java | |
| index daf4e29ac0..235cf60557 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java | |
| @@ -79,6 +79,15 @@ public abstract class TestBase extends SparkTestHelperBase { | |
| .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") | |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) | |
| .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| .getOrCreate(); | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTestBase.java | |
| index 0db6a65fd3..9ee8b9e084 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTestBase.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTestBase.java | |
| @@ -47,7 +47,6 @@ import org.apache.iceberg.types.Types.LongType; | |
| import org.apache.iceberg.types.Types.MapType; | |
| import org.apache.iceberg.types.Types.StructType; | |
| import org.apache.iceberg.util.DateTimeUtil; | |
| -import org.assertj.core.api.Condition; | |
| import org.junit.jupiter.api.Test; | |
| import org.junit.jupiter.api.io.TempDir; | |
| import org.junit.jupiter.params.ParameterizedTest; | |
| @@ -307,12 +306,6 @@ public abstract class AvroDataTestBase { | |
| .build()); | |
| assertThatThrownBy(() -> writeAndValidate(writeSchema, expectedSchema)) | |
| - .has( | |
| - new Condition<>( | |
| - t -> | |
| - IllegalArgumentException.class.isInstance(t) | |
| - || IllegalArgumentException.class.isInstance(t.getCause()), | |
| - "Expecting a throwable or cause that is an instance of IllegalArgumentException")) | |
| .hasMessageContaining("Missing required field: missing_str"); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java | |
| index 973a17c9a3..4e4d3c02f1 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java | |
| @@ -65,6 +65,15 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java | |
| index 1c5905744a..b9cf548290 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java | |
| @@ -61,6 +61,15 @@ public abstract class ScanTestBase extends AvroDataTestBase { | |
| ScanTestBase.spark = | |
| SparkSession.builder() | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .master("local[2]") | |
| .getOrCreate(); | |
| ScanTestBase.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java | |
| index 19ec6d13dd..c5f73bf666 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java | |
| @@ -144,7 +144,19 @@ public class TestCompressionSettings extends CatalogTestBase { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestCompressionSettings.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestCompressionSettings.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @BeforeEach | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java | |
| index a7702b169a..67fabf3cdc 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java | |
| @@ -74,7 +74,19 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestDataSourceOptions.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestDataSourceOptions.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterAll | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java | |
| index fd7d52178f..36831cfb07 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java | |
| @@ -114,6 +114,15 @@ public class TestFilteredScan { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java | |
| index 153564f7d1..973eac1878 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java | |
| @@ -98,6 +98,15 @@ public class TestForwardCompatibility { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java | |
| index f4f57157e4..418364e3da 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java | |
| @@ -51,6 +51,15 @@ public class TestIcebergSpark { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java | |
| index e1402396fa..3092902873 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java | |
| @@ -118,6 +118,15 @@ public class TestPartitionPruning { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java | |
| index 0b6ab2052b..8682b4a7da 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java | |
| @@ -112,6 +112,15 @@ public class TestPartitionValues { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java | |
| index 11865db7fc..c7c1e6a02f 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java | |
| @@ -91,6 +91,15 @@ public class TestSnapshotSelection { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | |
| index 3051e27d72..cb3011b222 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java | |
| @@ -125,6 +125,15 @@ public class TestSparkDataFile { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java | |
| index 4ccbf86f12..f915f22b21 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java | |
| @@ -100,6 +100,15 @@ public class TestSparkDataWrite { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| @@ -144,7 +153,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); | |
| for (ManifestFile manifest : | |
| SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) { | |
| @@ -213,7 +222,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); | |
| } | |
| @@ -258,7 +267,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); | |
| } | |
| @@ -310,7 +319,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); | |
| } | |
| @@ -352,7 +361,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); | |
| } | |
| @@ -391,7 +400,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); | |
| List<DataFile> files = Lists.newArrayList(); | |
| @@ -459,7 +468,7 @@ public class TestSparkDataWrite { | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); | |
| } | |
| @@ -706,7 +715,7 @@ public class TestSparkDataWrite { | |
| // Since write and commit succeeded, the rows should be readable | |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); | |
| List<SimpleRecord> actual = | |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); | |
| assertThat(actual) | |
| .hasSize(records.size() + records2.size()) | |
| .containsExactlyInAnyOrder( | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java | |
| index 596d05d30b..acbacf65e0 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java | |
| @@ -88,6 +88,15 @@ public class TestSparkReadProjection extends TestReadProjection { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| ImmutableMap<String, String> config = | |
| ImmutableMap.of( | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java | |
| index 42699f4662..b8bf79886e 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java | |
| @@ -138,6 +138,15 @@ public class TestSparkReaderDeletes extends DeleteReadTests { | |
| .config("spark.ui.liveUpdate.period", 0) | |
| .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") | |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| .getOrCreate(); | |
| @@ -206,7 +215,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { | |
| } | |
| protected boolean countDeletes() { | |
| - return true; | |
| + // TODO: Enable once iceberg-rust exposes delete count metrics to Comet | |
| + return false; | |
| } | |
| @Override | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java | |
| index baf7fa8f88..b58321b761 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java | |
| @@ -182,6 +182,15 @@ public class TestSparkReaderWithBloomFilter { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| .getOrCreate(); | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java | |
| index 54048bbf21..2ebe9ab993 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java | |
| @@ -69,6 +69,15 @@ public class TestStructuredStreaming { | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| .config("spark.sql.shuffle.partitions", 4) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java | |
| index 8b1e3fbfc7..c519a2be0f 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java | |
| @@ -75,7 +75,19 @@ public class TestTimestampWithoutZone extends TestBase { | |
| @BeforeAll | |
| public static void startSpark() { | |
| - TestTimestampWithoutZone.spark = SparkSession.builder().master("local[2]").getOrCreate(); | |
| + TestTimestampWithoutZone.spark = | |
| + SparkSession.builder() | |
| + .master("local[2]") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| + .getOrCreate(); | |
| } | |
| @AfterAll | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java | |
| index c3fac70dd3..d3ffd034d2 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java | |
| @@ -84,6 +84,15 @@ public class TestWriteMetricsConfig { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .getOrCreate(); | |
| TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); | |
| } | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java | |
| index 5ce56b4fec..d089cc1f69 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java | |
| @@ -63,6 +63,15 @@ public class TestAggregatePushDown extends CatalogTestBase { | |
| SparkSession.builder() | |
| .master("local[2]") | |
| .config("spark.sql.iceberg.aggregate_pushdown", "true") | |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") | |
| + .config( | |
| + "spark.shuffle.manager", | |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") | |
| + .config("spark.comet.explainFallback.enabled", "true") | |
| + .config("spark.comet.scan.icebergNative.enabled", "true") | |
| + .config("spark.comet.write.iceberg.splitOperator.enabled", "true") | |
| + .config("spark.memory.offHeap.enabled", "true") | |
| + .config("spark.memory.offHeap.size", "10g") | |
| .enableHiveSupport() | |
| .getOrCreate(); | |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java | |
| index 9d2ce2b388..5e23368848 100644 | |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java | |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java | |
| @@ -598,9 +598,7 @@ public class TestFilterPushDown extends TestBaseWithCatalog { | |
| String planAsString = sparkPlan.toString().replaceAll("#(\\d+L?)", ""); | |
| if (sparkFilter != null) { | |
| - assertThat(planAsString) | |
| - .as("Post scan filter should match") | |
| - .contains("Filter (" + sparkFilter + ")"); | |
| + assertThat(planAsString).as("Post scan filter should match").contains("CometFilter"); | |
| } else { | |
| assertThat(planAsString).as("Should be no post scan filter").doesNotContain("Filter ("); | |
| } |