| diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml |
| index 04ffa8f4ed..5a925fd594 100644 |
| --- a/gradle/libs.versions.toml |
| +++ b/gradle/libs.versions.toml |
| @@ -34,6 +34,7 @@ azuresdk-bom = "1.2.31" |
| awssdk-s3accessgrants = "2.3.0" |
| caffeine = "2.9.3" |
| calcite = "1.10.0" |
| +comet = "1.0.0-SNAPSHOT" |
| datasketches = "6.2.0" |
| delta-standalone = "3.3.0" |
| delta-spark = "3.3.0" |
| @@ -81,7 +82,7 @@ slf4j = "2.0.16" |
| snowflake-jdbc = "3.22.0" |
| spark-hive33 = "3.3.4" |
| spark-hive34 = "3.4.4" |
| -spark-hive35 = "3.5.4" |
| +spark-hive35 = "3.5.6" |
| sqlite-jdbc = "3.48.0.0" |
| testcontainers = "1.20.4" |
| tez010 = "0.10.4" |
| diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle |
| index 6eb26e8b73..38d9cf3ca3 100644 |
| --- a/spark/v3.4/build.gradle |
| +++ b/spark/v3.4/build.gradle |
| @@ -75,7 +75,7 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") { |
| exclude group: 'org.roaringbitmap' |
| } |
| |
| - compileOnly "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:0.5.0" |
| + compileOnly "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" |
| |
| implementation libs.parquet.column |
| implementation libs.parquet.hadoop |
| @@ -185,7 +185,7 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer |
| testImplementation libs.avro.avro |
| testImplementation libs.parquet.hadoop |
| testImplementation libs.junit.vintage.engine |
| - testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:0.5.0" |
| + testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" |
| |
| // Required because we remove antlr plugin dependencies from the compile configuration, see note above |
| runtimeOnly libs.antlr.runtime |
| @@ -260,6 +260,8 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio |
| integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') |
| integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts') |
| integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts') |
| + integrationImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" |
| + integrationImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" |
| |
| // runtime dependencies for running Hive Catalog based integration test |
| integrationRuntimeOnly project(':iceberg-hive-metastore') |
| diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java |
| index 4f137f5b8d..145ef2d00c 100644 |
| --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java |
| +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java |
| @@ -60,7 +60,23 @@ public abstract class SparkExtensionsTestBase extends SparkCatalogTestBase { |
| .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") |
| .config( |
| SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .enableHiveSupport() |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| |
| SparkTestBase.catalog = |
| diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java |
| index 55a413063e..a1895c2faf 100644 |
| --- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java |
| +++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java |
| @@ -61,6 +61,14 @@ public class TestCallStatementParser { |
| .master("local[2]") |
| .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) |
| .config("spark.extra.prop", "value") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| TestCallStatementParser.parser = spark.sessionState().sqlParser(); |
| } |
| diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java |
| index b6ade2bff3..b52d011691 100644 |
| --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java |
| +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java |
| @@ -170,6 +170,14 @@ public class DeleteOrphanFilesBenchmark { |
| .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") |
| .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .master("local"); |
| spark = builder.getOrCreate(); |
| } |
| diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java |
| index b08c352819..d2ca76e07a 100644 |
| --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java |
| +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java |
| @@ -386,6 +386,14 @@ public class IcebergSortCompactionBenchmark { |
| "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") |
| .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .master("local[*]"); |
| spark = builder.getOrCreate(); |
| Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); |
| diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java |
| index 68c537e34a..5ef031963a 100644 |
| --- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java |
| +++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java |
| @@ -94,7 +94,17 @@ public abstract class IcebergSourceBenchmark { |
| } |
| |
| protected void setupSpark(boolean enableDictionaryEncoding) { |
| - SparkSession.Builder builder = SparkSession.builder().config("spark.ui.enabled", false); |
| + SparkSession.Builder builder = |
| + SparkSession.builder() |
| + .config("spark.ui.enabled", false) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g"); |
| if (!enableDictionaryEncoding) { |
| builder |
| .config("parquet.dictionary.page.size", "1") |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java |
| deleted file mode 100644 |
| index 4794863ab1..0000000000 |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java |
| +++ /dev/null |
| @@ -1,150 +0,0 @@ |
| -/* |
| - * Licensed to the Apache Software Foundation (ASF) under one |
| - * or more contributor license agreements. See the NOTICE file |
| - * distributed with this work for additional information |
| - * regarding copyright ownership. The ASF licenses this file |
| - * to you under the Apache License, Version 2.0 (the |
| - * "License"); you may not use this file except in compliance |
| - * with the License. You may obtain a copy of the License at |
| - * |
| - * http://www.apache.org/licenses/LICENSE-2.0 |
| - * |
| - * Unless required by applicable law or agreed to in writing, |
| - * software distributed under the License is distributed on an |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| - * KIND, either express or implied. See the License for the |
| - * specific language governing permissions and limitations |
| - * under the License. |
| - */ |
| -package org.apache.iceberg.spark.data.vectorized; |
| - |
| -import java.io.IOException; |
| -import java.util.Map; |
| -import org.apache.comet.parquet.AbstractColumnReader; |
| -import org.apache.comet.parquet.ColumnReader; |
| -import org.apache.comet.parquet.TypeUtil; |
| -import org.apache.comet.parquet.Utils; |
| -import org.apache.comet.shaded.arrow.c.CometSchemaImporter; |
| -import org.apache.comet.shaded.arrow.memory.RootAllocator; |
| -import org.apache.iceberg.parquet.VectorizedReader; |
| -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; |
| -import org.apache.iceberg.spark.SparkSchemaUtil; |
| -import org.apache.iceberg.types.Types; |
| -import org.apache.parquet.column.ColumnDescriptor; |
| -import org.apache.parquet.column.page.PageReadStore; |
| -import org.apache.parquet.column.page.PageReader; |
| -import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; |
| -import org.apache.parquet.hadoop.metadata.ColumnPath; |
| -import org.apache.spark.sql.types.DataType; |
| -import org.apache.spark.sql.types.Metadata; |
| -import org.apache.spark.sql.types.StructField; |
| -import org.apache.spark.sql.vectorized.ColumnVector; |
| - |
| -class CometColumnReader implements VectorizedReader<ColumnVector> { |
| - // use the Comet default batch size |
| - public static final int DEFAULT_BATCH_SIZE = 8192; |
| - |
| - private final ColumnDescriptor descriptor; |
| - private final DataType sparkType; |
| - |
| - // The delegated ColumnReader from Comet side |
| - private AbstractColumnReader delegate; |
| - private boolean initialized = false; |
| - private int batchSize = DEFAULT_BATCH_SIZE; |
| - private CometSchemaImporter importer; |
| - |
| - CometColumnReader(DataType sparkType, ColumnDescriptor descriptor) { |
| - this.sparkType = sparkType; |
| - this.descriptor = descriptor; |
| - } |
| - |
| - CometColumnReader(Types.NestedField field) { |
| - DataType dataType = SparkSchemaUtil.convert(field.type()); |
| - StructField structField = new StructField(field.name(), dataType, false, Metadata.empty()); |
| - this.sparkType = dataType; |
| - this.descriptor = TypeUtil.convertToParquet(structField); |
| - } |
| - |
| - public AbstractColumnReader delegate() { |
| - return delegate; |
| - } |
| - |
| - void setDelegate(AbstractColumnReader delegate) { |
| - this.delegate = delegate; |
| - } |
| - |
| - void setInitialized(boolean initialized) { |
| - this.initialized = initialized; |
| - } |
| - |
| - public int batchSize() { |
| - return batchSize; |
| - } |
| - |
| - /** |
| - * This method is to initialized/reset the CometColumnReader. This needs to be called for each row |
| - * group after readNextRowGroup, so a new dictionary encoding can be set for each of the new row |
| - * groups. |
| - */ |
| - public void reset() { |
| - if (importer != null) { |
| - importer.close(); |
| - } |
| - |
| - if (delegate != null) { |
| - delegate.close(); |
| - } |
| - |
| - this.importer = new CometSchemaImporter(new RootAllocator()); |
| - this.delegate = Utils.getColumnReader(sparkType, descriptor, importer, batchSize, false, false); |
| - this.initialized = true; |
| - } |
| - |
| - public ColumnDescriptor descriptor() { |
| - return descriptor; |
| - } |
| - |
| - /** Returns the Spark data type for this column. */ |
| - public DataType sparkType() { |
| - return sparkType; |
| - } |
| - |
| - /** |
| - * Set the page reader to be 'pageReader'. |
| - * |
| - * <p>NOTE: this should be called before reading a new Parquet column chunk, and after {@link |
| - * CometColumnReader#reset} is called. |
| - */ |
| - public void setPageReader(PageReader pageReader) throws IOException { |
| - Preconditions.checkState(initialized, "Invalid state: 'reset' should be called first"); |
| - ((ColumnReader) delegate).setPageReader(pageReader); |
| - } |
| - |
| - @Override |
| - public void close() { |
| - // close resources on native side |
| - if (importer != null) { |
| - importer.close(); |
| - } |
| - |
| - if (delegate != null) { |
| - delegate.close(); |
| - } |
| - } |
| - |
| - @Override |
| - public void setBatchSize(int size) { |
| - this.batchSize = size; |
| - } |
| - |
| - @Override |
| - public void setRowGroupInfo( |
| - PageReadStore pageReadStore, Map<ColumnPath, ColumnChunkMetaData> map, long size) { |
| - throw new UnsupportedOperationException("Not supported"); |
| - } |
| - |
| - @Override |
| - public ColumnVector read(ColumnVector reuse, int numRowsToRead) { |
| - throw new UnsupportedOperationException("Not supported"); |
| - } |
| -} |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java |
| deleted file mode 100644 |
| index 1440e5d1d3..0000000000 |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java |
| +++ /dev/null |
| @@ -1,203 +0,0 @@ |
| -/* |
| - * Licensed to the Apache Software Foundation (ASF) under one |
| - * or more contributor license agreements. See the NOTICE file |
| - * distributed with this work for additional information |
| - * regarding copyright ownership. The ASF licenses this file |
| - * to you under the Apache License, Version 2.0 (the |
| - * "License"); you may not use this file except in compliance |
| - * with the License. You may obtain a copy of the License at |
| - * |
| - * http://www.apache.org/licenses/LICENSE-2.0 |
| - * |
| - * Unless required by applicable law or agreed to in writing, |
| - * software distributed under the License is distributed on an |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| - * KIND, either express or implied. See the License for the |
| - * specific language governing permissions and limitations |
| - * under the License. |
| - */ |
| -package org.apache.iceberg.spark.data.vectorized; |
| - |
| -import java.io.IOException; |
| -import java.io.UncheckedIOException; |
| -import java.util.List; |
| -import java.util.Map; |
| -import org.apache.comet.parquet.AbstractColumnReader; |
| -import org.apache.comet.parquet.BatchReader; |
| -import org.apache.iceberg.Schema; |
| -import org.apache.iceberg.data.DeleteFilter; |
| -import org.apache.iceberg.parquet.VectorizedReader; |
| -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; |
| -import org.apache.iceberg.spark.SparkSchemaUtil; |
| -import org.apache.iceberg.util.Pair; |
| -import org.apache.parquet.column.page.PageReadStore; |
| -import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; |
| -import org.apache.parquet.hadoop.metadata.ColumnPath; |
| -import org.apache.spark.sql.catalyst.InternalRow; |
| -import org.apache.spark.sql.vectorized.ColumnVector; |
| -import org.apache.spark.sql.vectorized.ColumnarBatch; |
| - |
| -/** |
| - * {@link VectorizedReader} that returns Spark's {@link ColumnarBatch} to support Spark's vectorized |
| - * read path. The {@link ColumnarBatch} returned is created by passing in the Arrow vectors |
| - * populated via delegated read calls to {@link CometColumnReader VectorReader(s)}. |
| - */ |
| -@SuppressWarnings("checkstyle:VisibilityModifier") |
| -class CometColumnarBatchReader implements VectorizedReader<ColumnarBatch> { |
| - |
| - private final CometColumnReader[] readers; |
| - private final boolean hasIsDeletedColumn; |
| - |
| - // The delegated BatchReader on the Comet side does the real work of loading a batch of rows. |
| - // The Comet BatchReader contains an array of ColumnReader. There is no need to explicitly call |
| - // ColumnReader.readBatch; instead, BatchReader.nextBatch will be called, which underneath calls |
| - // ColumnReader.readBatch. The only exception is DeleteColumnReader, because at the time of |
| - // calling BatchReader.nextBatch, the isDeleted value is not yet available, so |
| - // DeleteColumnReader.readBatch must be called explicitly later, after the isDeleted value is |
| - // available. |
| - private final BatchReader delegate; |
| - private DeleteFilter<InternalRow> deletes = null; |
| - private long rowStartPosInBatch = 0; |
| - |
| - CometColumnarBatchReader(List<VectorizedReader<?>> readers, Schema schema) { |
| - this.readers = |
| - readers.stream().map(CometColumnReader.class::cast).toArray(CometColumnReader[]::new); |
| - this.hasIsDeletedColumn = |
| - readers.stream().anyMatch(reader -> reader instanceof CometDeleteColumnReader); |
| - |
| - AbstractColumnReader[] abstractColumnReaders = new AbstractColumnReader[readers.size()]; |
| - this.delegate = new BatchReader(abstractColumnReaders); |
| - delegate.setSparkSchema(SparkSchemaUtil.convert(schema)); |
| - } |
| - |
| - @Override |
| - public void setRowGroupInfo( |
| - PageReadStore pageStore, Map<ColumnPath, ColumnChunkMetaData> metaData, long rowPosition) { |
| - setRowGroupInfo(pageStore, metaData); |
| - } |
| - |
| - @Override |
| - public void setRowGroupInfo( |
| - PageReadStore pageStore, Map<ColumnPath, ColumnChunkMetaData> metaData) { |
| - for (int i = 0; i < readers.length; i++) { |
| - try { |
| - if (!(readers[i] instanceof CometConstantColumnReader) |
| - && !(readers[i] instanceof CometPositionColumnReader) |
| - && !(readers[i] instanceof CometDeleteColumnReader)) { |
| - readers[i].reset(); |
| - readers[i].setPageReader(pageStore.getPageReader(readers[i].descriptor())); |
| - } |
| - } catch (IOException e) { |
| - throw new UncheckedIOException("Failed to setRowGroupInfo for Comet vectorization", e); |
| - } |
| - } |
| - |
| - for (int i = 0; i < readers.length; i++) { |
| - delegate.getColumnReaders()[i] = this.readers[i].delegate(); |
| - } |
| - |
| - this.rowStartPosInBatch = |
| - pageStore |
| - .getRowIndexOffset() |
| - .orElseThrow( |
| - () -> |
| - new IllegalArgumentException( |
| - "PageReadStore does not contain row index offset")); |
| - } |
| - |
| - public void setDeleteFilter(DeleteFilter<InternalRow> deleteFilter) { |
| - this.deletes = deleteFilter; |
| - } |
| - |
| - @Override |
| - public final ColumnarBatch read(ColumnarBatch reuse, int numRowsToRead) { |
| - ColumnarBatch columnarBatch = new ColumnBatchLoader(numRowsToRead).loadDataToColumnBatch(); |
| - rowStartPosInBatch += numRowsToRead; |
| - return columnarBatch; |
| - } |
| - |
| - @Override |
| - public void setBatchSize(int batchSize) { |
| - for (CometColumnReader reader : readers) { |
| - if (reader != null) { |
| - reader.setBatchSize(batchSize); |
| - } |
| - } |
| - } |
| - |
| - @Override |
| - public void close() { |
| - for (CometColumnReader reader : readers) { |
| - if (reader != null) { |
| - reader.close(); |
| - } |
| - } |
| - } |
| - |
| - private class ColumnBatchLoader { |
| - private final int batchSize; |
| - |
| - ColumnBatchLoader(int numRowsToRead) { |
| - Preconditions.checkArgument( |
| - numRowsToRead > 0, "Invalid number of rows to read: %s", numRowsToRead); |
| - this.batchSize = numRowsToRead; |
| - } |
| - |
| - ColumnarBatch loadDataToColumnBatch() { |
| - ColumnVector[] vectors = readDataToColumnVectors(); |
| - int numLiveRows = batchSize; |
| - |
| - if (hasIsDeletedColumn) { |
| - boolean[] isDeleted = buildIsDeleted(vectors); |
| - readDeletedColumn(vectors, isDeleted); |
| - } else { |
| - Pair<int[], Integer> pair = buildRowIdMapping(vectors); |
| - if (pair != null) { |
| - int[] rowIdMapping = pair.first(); |
| - numLiveRows = pair.second(); |
| - for (int i = 0; i < vectors.length; i++) { |
| - vectors[i] = new ColumnVectorWithFilter(vectors[i], rowIdMapping); |
| - } |
| - } |
| - } |
| - |
| - if (deletes != null && deletes.hasEqDeletes()) { |
| - vectors = ColumnarBatchUtil.removeExtraColumns(deletes, vectors); |
| - } |
| - |
| - ColumnarBatch batch = new ColumnarBatch(vectors); |
| - batch.setNumRows(numLiveRows); |
| - return batch; |
| - } |
| - |
| - private boolean[] buildIsDeleted(ColumnVector[] vectors) { |
| - return ColumnarBatchUtil.buildIsDeleted(vectors, deletes, rowStartPosInBatch, batchSize); |
| - } |
| - |
| - private Pair<int[], Integer> buildRowIdMapping(ColumnVector[] vectors) { |
| - return ColumnarBatchUtil.buildRowIdMapping(vectors, deletes, rowStartPosInBatch, batchSize); |
| - } |
| - |
| - ColumnVector[] readDataToColumnVectors() { |
| - ColumnVector[] columnVectors = new ColumnVector[readers.length]; |
| - // Fetch rows for all readers in the delegate |
| - delegate.nextBatch(batchSize); |
| - for (int i = 0; i < readers.length; i++) { |
| - columnVectors[i] = readers[i].delegate().currentBatch(); |
| - } |
| - |
| - return columnVectors; |
| - } |
| - |
| - void readDeletedColumn(ColumnVector[] columnVectors, boolean[] isDeleted) { |
| - for (int i = 0; i < readers.length; i++) { |
| - if (readers[i] instanceof CometDeleteColumnReader) { |
| - CometDeleteColumnReader deleteColumnReader = new CometDeleteColumnReader<>(isDeleted); |
| - deleteColumnReader.setBatchSize(batchSize); |
| - deleteColumnReader.delegate().readBatch(batchSize); |
| - columnVectors[i] = deleteColumnReader.delegate().currentBatch(); |
| - } |
| - } |
| - } |
| - } |
| -} |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java |
| deleted file mode 100644 |
| index c665002e8f..0000000000 |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java |
| +++ /dev/null |
| @@ -1,65 +0,0 @@ |
| -/* |
| - * Licensed to the Apache Software Foundation (ASF) under one |
| - * or more contributor license agreements. See the NOTICE file |
| - * distributed with this work for additional information |
| - * regarding copyright ownership. The ASF licenses this file |
| - * to you under the Apache License, Version 2.0 (the |
| - * "License"); you may not use this file except in compliance |
| - * with the License. You may obtain a copy of the License at |
| - * |
| - * http://www.apache.org/licenses/LICENSE-2.0 |
| - * |
| - * Unless required by applicable law or agreed to in writing, |
| - * software distributed under the License is distributed on an |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| - * KIND, either express or implied. See the License for the |
| - * specific language governing permissions and limitations |
| - * under the License. |
| - */ |
| -package org.apache.iceberg.spark.data.vectorized; |
| - |
| -import java.math.BigDecimal; |
| -import java.nio.ByteBuffer; |
| -import org.apache.comet.parquet.ConstantColumnReader; |
| -import org.apache.iceberg.types.Types; |
| -import org.apache.spark.sql.types.DataType; |
| -import org.apache.spark.sql.types.DataTypes; |
| -import org.apache.spark.sql.types.Decimal; |
| -import org.apache.spark.sql.types.DecimalType; |
| -import org.apache.spark.unsafe.types.UTF8String; |
| - |
| -class CometConstantColumnReader<T> extends CometColumnReader { |
| - |
| - CometConstantColumnReader(T value, Types.NestedField field) { |
| - super(field); |
| - // use delegate to set constant value on the native side to be consumed by native execution. |
| - setDelegate( |
| - new ConstantColumnReader(sparkType(), descriptor(), convertToSparkValue(value), false)); |
| - } |
| - |
| - @Override |
| - public void setBatchSize(int batchSize) { |
| - super.setBatchSize(batchSize); |
| - delegate().setBatchSize(batchSize); |
| - setInitialized(true); |
| - } |
| - |
| - private Object convertToSparkValue(T value) { |
| - DataType dataType = sparkType(); |
| - // Match the value to Spark internal type if necessary |
| - if (dataType == DataTypes.StringType && value instanceof String) { |
| - // the internal type for StringType is UTF8String |
| - return UTF8String.fromString((String) value); |
| - } else if (dataType instanceof DecimalType && value instanceof BigDecimal) { |
| - // the internal type for DecimalType is Decimal |
| - return Decimal.apply((BigDecimal) value); |
| - } else if (dataType == DataTypes.BinaryType && value instanceof ByteBuffer) { |
| - // the internal type for DecimalType is byte[] |
| - // Iceberg default value should always use HeapBufferBuffer, so calling ByteBuffer.array() |
| - // should be safe. |
| - return ((java.nio.ByteBuffer) value).array(); |
| - } else { |
| - return value; |
| - } |
| - } |
| -} |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java |
| deleted file mode 100644 |
| index 4a28fc51da..0000000000 |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java |
| +++ /dev/null |
| @@ -1,75 +0,0 @@ |
| -/* |
| - * Licensed to the Apache Software Foundation (ASF) under one |
| - * or more contributor license agreements. See the NOTICE file |
| - * distributed with this work for additional information |
| - * regarding copyright ownership. The ASF licenses this file |
| - * to you under the Apache License, Version 2.0 (the |
| - * "License"); you may not use this file except in compliance |
| - * with the License. You may obtain a copy of the License at |
| - * |
| - * http://www.apache.org/licenses/LICENSE-2.0 |
| - * |
| - * Unless required by applicable law or agreed to in writing, |
| - * software distributed under the License is distributed on an |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| - * KIND, either express or implied. See the License for the |
| - * specific language governing permissions and limitations |
| - * under the License. |
| - */ |
| -package org.apache.iceberg.spark.data.vectorized; |
| - |
| -import org.apache.comet.parquet.MetadataColumnReader; |
| -import org.apache.comet.parquet.Native; |
| -import org.apache.comet.parquet.TypeUtil; |
| -import org.apache.iceberg.MetadataColumns; |
| -import org.apache.iceberg.types.Types; |
| -import org.apache.spark.sql.types.DataTypes; |
| -import org.apache.spark.sql.types.Metadata; |
| -import org.apache.spark.sql.types.StructField; |
| - |
| -class CometDeleteColumnReader<T> extends CometColumnReader { |
| - CometDeleteColumnReader(Types.NestedField field) { |
| - super(field); |
| - setDelegate(new DeleteColumnReader()); |
| - } |
| - |
| - CometDeleteColumnReader(boolean[] isDeleted) { |
| - super(MetadataColumns.IS_DELETED); |
| - setDelegate(new DeleteColumnReader(isDeleted)); |
| - } |
| - |
| - @Override |
| - public void setBatchSize(int batchSize) { |
| - super.setBatchSize(batchSize); |
| - delegate().setBatchSize(batchSize); |
| - setInitialized(true); |
| - } |
| - |
| - private static class DeleteColumnReader extends MetadataColumnReader { |
| - private boolean[] isDeleted; |
| - |
| - DeleteColumnReader() { |
| - super( |
| - DataTypes.BooleanType, |
| - TypeUtil.convertToParquet( |
| - new StructField("_deleted", DataTypes.BooleanType, false, Metadata.empty())), |
| - false /* useDecimal128 = false */, |
| - false /* isConstant */); |
| - this.isDeleted = new boolean[0]; |
| - } |
| - |
| - DeleteColumnReader(boolean[] isDeleted) { |
| - this(); |
| - this.isDeleted = isDeleted; |
| - } |
| - |
| - @Override |
| - public void readBatch(int total) { |
| - Native.resetBatch(nativeHandle); |
| - // set isDeleted on the native side to be consumed by native execution |
| - Native.setIsDeleted(nativeHandle, isDeleted); |
| - |
| - super.readBatch(total); |
| - } |
| - } |
| -} |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java |
| deleted file mode 100644 |
| index 1949a71798..0000000000 |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java |
| +++ /dev/null |
| @@ -1,62 +0,0 @@ |
| -/* |
| - * Licensed to the Apache Software Foundation (ASF) under one |
| - * or more contributor license agreements. See the NOTICE file |
| - * distributed with this work for additional information |
| - * regarding copyright ownership. The ASF licenses this file |
| - * to you under the Apache License, Version 2.0 (the |
| - * "License"); you may not use this file except in compliance |
| - * with the License. You may obtain a copy of the License at |
| - * |
| - * http://www.apache.org/licenses/LICENSE-2.0 |
| - * |
| - * Unless required by applicable law or agreed to in writing, |
| - * software distributed under the License is distributed on an |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| - * KIND, either express or implied. See the License for the |
| - * specific language governing permissions and limitations |
| - * under the License. |
| - */ |
| -package org.apache.iceberg.spark.data.vectorized; |
| - |
| -import org.apache.comet.parquet.MetadataColumnReader; |
| -import org.apache.comet.parquet.Native; |
| -import org.apache.iceberg.types.Types; |
| -import org.apache.parquet.column.ColumnDescriptor; |
| -import org.apache.spark.sql.types.DataTypes; |
| - |
| -class CometPositionColumnReader extends CometColumnReader { |
| - CometPositionColumnReader(Types.NestedField field) { |
| - super(field); |
| - setDelegate(new PositionColumnReader(descriptor())); |
| - } |
| - |
| - @Override |
| - public void setBatchSize(int batchSize) { |
| - super.setBatchSize(batchSize); |
| - delegate().setBatchSize(batchSize); |
| - setInitialized(true); |
| - } |
| - |
| - private static class PositionColumnReader extends MetadataColumnReader { |
| - /** The current position value of the column that are used to initialize this column reader. */ |
| - private long position; |
| - |
| - PositionColumnReader(ColumnDescriptor descriptor) { |
| - super( |
| - DataTypes.LongType, |
| - descriptor, |
| - false /* useDecimal128 = false */, |
| - false /* isConstant */); |
| - } |
| - |
| - @Override |
| - public void readBatch(int total) { |
| - Native.resetBatch(nativeHandle); |
| - // set position on the native side to be consumed by native execution |
| - Native.setPosition(nativeHandle, position, total); |
| - position += total; |
| - |
| - super.readBatch(total); |
| - } |
| - } |
| -} |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java |
| deleted file mode 100644 |
| index d36f1a7274..0000000000 |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java |
| +++ /dev/null |
| @@ -1,147 +0,0 @@ |
| -/* |
| - * Licensed to the Apache Software Foundation (ASF) under one |
| - * or more contributor license agreements. See the NOTICE file |
| - * distributed with this work for additional information |
| - * regarding copyright ownership. The ASF licenses this file |
| - * to you under the Apache License, Version 2.0 (the |
| - * "License"); you may not use this file except in compliance |
| - * with the License. You may obtain a copy of the License at |
| - * |
| - * http://www.apache.org/licenses/LICENSE-2.0 |
| - * |
| - * Unless required by applicable law or agreed to in writing, |
| - * software distributed under the License is distributed on an |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| - * KIND, either express or implied. See the License for the |
| - * specific language governing permissions and limitations |
| - * under the License. |
| - */ |
| -package org.apache.iceberg.spark.data.vectorized; |
| - |
| -import java.util.List; |
| -import java.util.Map; |
| -import java.util.function.Function; |
| -import java.util.stream.IntStream; |
| -import org.apache.iceberg.MetadataColumns; |
| -import org.apache.iceberg.Schema; |
| -import org.apache.iceberg.data.DeleteFilter; |
| -import org.apache.iceberg.parquet.TypeWithSchemaVisitor; |
| -import org.apache.iceberg.parquet.VectorizedReader; |
| -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; |
| -import org.apache.iceberg.relocated.com.google.common.collect.Lists; |
| -import org.apache.iceberg.relocated.com.google.common.collect.Maps; |
| -import org.apache.iceberg.spark.SparkSchemaUtil; |
| -import org.apache.iceberg.types.Types; |
| -import org.apache.parquet.column.ColumnDescriptor; |
| -import org.apache.parquet.schema.GroupType; |
| -import org.apache.parquet.schema.MessageType; |
| -import org.apache.parquet.schema.PrimitiveType; |
| -import org.apache.parquet.schema.Type; |
| -import org.apache.spark.sql.catalyst.InternalRow; |
| - |
| -class CometVectorizedReaderBuilder extends TypeWithSchemaVisitor<VectorizedReader<?>> { |
| - |
| - private final MessageType parquetSchema; |
| - private final Schema icebergSchema; |
| - private final Map<Integer, ?> idToConstant; |
| - private final Function<List<VectorizedReader<?>>, VectorizedReader<?>> readerFactory; |
| - private final DeleteFilter<InternalRow> deleteFilter; |
| - |
| - CometVectorizedReaderBuilder( |
| - Schema expectedSchema, |
| - MessageType parquetSchema, |
| - Map<Integer, ?> idToConstant, |
| - Function<List<VectorizedReader<?>>, VectorizedReader<?>> readerFactory, |
| - DeleteFilter<InternalRow> deleteFilter) { |
| - this.parquetSchema = parquetSchema; |
| - this.icebergSchema = expectedSchema; |
| - this.idToConstant = idToConstant; |
| - this.readerFactory = readerFactory; |
| - this.deleteFilter = deleteFilter; |
| - } |
| - |
| - @Override |
| - public VectorizedReader<?> message( |
| - Types.StructType expected, MessageType message, List<VectorizedReader<?>> fieldReaders) { |
| - GroupType groupType = message.asGroupType(); |
| - Map<Integer, VectorizedReader<?>> readersById = Maps.newHashMap(); |
| - List<Type> fields = groupType.getFields(); |
| - |
| - IntStream.range(0, fields.size()) |
| - .filter(pos -> fields.get(pos).getId() != null) |
| - .forEach(pos -> readersById.put(fields.get(pos).getId().intValue(), fieldReaders.get(pos))); |
| - |
| - List<Types.NestedField> icebergFields = |
| - expected != null ? expected.fields() : ImmutableList.of(); |
| - |
| - List<VectorizedReader<?>> reorderedFields = |
| - Lists.newArrayListWithExpectedSize(icebergFields.size()); |
| - |
| - for (Types.NestedField field : icebergFields) { |
| - int id = field.fieldId(); |
| - VectorizedReader<?> reader = readersById.get(id); |
| - if (idToConstant.containsKey(id)) { |
| - CometConstantColumnReader constantReader = |
| - new CometConstantColumnReader<>(idToConstant.get(id), field); |
| - reorderedFields.add(constantReader); |
| - } else if (id == MetadataColumns.ROW_POSITION.fieldId()) { |
| - reorderedFields.add(new CometPositionColumnReader(field)); |
| - } else if (id == MetadataColumns.IS_DELETED.fieldId()) { |
| - CometColumnReader deleteReader = new CometDeleteColumnReader<>(field); |
| - reorderedFields.add(deleteReader); |
| - } else if (reader != null) { |
| - reorderedFields.add(reader); |
| - } else if (field.initialDefault() != null) { |
| - CometColumnReader constantReader = |
| - new CometConstantColumnReader<>(field.initialDefault(), field); |
| - reorderedFields.add(constantReader); |
| - } else if (field.isOptional()) { |
| - CometColumnReader constantReader = new CometConstantColumnReader<>(null, field); |
| - reorderedFields.add(constantReader); |
| - } else { |
| - throw new IllegalArgumentException( |
| - String.format("Missing required field: %s", field.name())); |
| - } |
| - } |
| - return vectorizedReader(reorderedFields); |
| - } |
| - |
| - protected VectorizedReader<?> vectorizedReader(List<VectorizedReader<?>> reorderedFields) { |
| - VectorizedReader<?> reader = readerFactory.apply(reorderedFields); |
| - if (deleteFilter != null) { |
| - ((CometColumnarBatchReader) reader).setDeleteFilter(deleteFilter); |
| - } |
| - return reader; |
| - } |
| - |
| - @Override |
| - public VectorizedReader<?> struct( |
| - Types.StructType expected, GroupType groupType, List<VectorizedReader<?>> fieldReaders) { |
| - if (expected != null) { |
| - throw new UnsupportedOperationException( |
| - "Vectorized reads are not supported yet for struct fields"); |
| - } |
| - return null; |
| - } |
| - |
| - @Override |
| - public VectorizedReader<?> primitive( |
| - org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) { |
| - |
| - if (primitive.getId() == null) { |
| - return null; |
| - } |
| - int parquetFieldId = primitive.getId().intValue(); |
| - ColumnDescriptor desc = parquetSchema.getColumnDescription(currentPath()); |
| - // Nested types not yet supported for vectorized reads |
| - if (desc.getMaxRepetitionLevel() > 0) { |
| - return null; |
| - } |
| - Types.NestedField icebergField = icebergSchema.findField(parquetFieldId); |
| - if (icebergField == null) { |
| - return null; |
| - } |
| - |
| - return new CometColumnReader(SparkSchemaUtil.convert(icebergField.type()), desc); |
| - } |
| -} |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java |
| index b523bc5bff..636ad3be7d 100644 |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java |
| +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java |
| @@ -70,23 +70,6 @@ public class VectorizedSparkParquetReaders { |
| deleteFilter)); |
| } |
| |
| - public static CometColumnarBatchReader buildCometReader( |
| - Schema expectedSchema, |
| - MessageType fileSchema, |
| - Map<Integer, ?> idToConstant, |
| - DeleteFilter<InternalRow> deleteFilter) { |
| - return (CometColumnarBatchReader) |
| - TypeWithSchemaVisitor.visit( |
| - expectedSchema.asStruct(), |
| - fileSchema, |
| - new CometVectorizedReaderBuilder( |
| - expectedSchema, |
| - fileSchema, |
| - idToConstant, |
| - readers -> new CometColumnarBatchReader(readers, expectedSchema), |
| - deleteFilter)); |
| - } |
| - |
| // enables unsafe memory access to avoid costly checks to see if index is within bounds |
| // as long as it is not configured explicitly (see BoundsChecking in Arrow) |
| private static void enableUnsafeMemoryAccess() { |
| diff --git a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java |
| index 780e1750a5..25f253eede 100644 |
| --- a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java |
| +++ b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java |
| @@ -34,7 +34,6 @@ import org.apache.iceberg.parquet.Parquet; |
| import org.apache.iceberg.relocated.com.google.common.collect.Sets; |
| import org.apache.iceberg.spark.OrcBatchReadConf; |
| import org.apache.iceberg.spark.ParquetBatchReadConf; |
| -import org.apache.iceberg.spark.ParquetReaderType; |
| import org.apache.iceberg.spark.data.vectorized.VectorizedSparkOrcReaders; |
| import org.apache.iceberg.spark.data.vectorized.VectorizedSparkParquetReaders; |
| import org.apache.iceberg.types.TypeUtil; |
| @@ -92,15 +91,9 @@ abstract class BaseBatchReader<T extends ScanTask> extends BaseReader<ColumnarBa |
| .project(requiredSchema) |
| .split(start, length) |
| .createBatchedReaderFunc( |
| - fileSchema -> { |
| - if (parquetConf.readerType() == ParquetReaderType.COMET) { |
| - return VectorizedSparkParquetReaders.buildCometReader( |
| - requiredSchema, fileSchema, idToConstant, deleteFilter); |
| - } else { |
| - return VectorizedSparkParquetReaders.buildReader( |
| - requiredSchema, fileSchema, idToConstant, deleteFilter); |
| - } |
| - }) |
| + fileSchema -> |
| + VectorizedSparkParquetReaders.buildReader( |
| + requiredSchema, fileSchema, idToConstant, deleteFilter)) |
| .recordsPerBatch(parquetConf.batchSize()) |
| .filter(residual) |
| .caseSensitive(caseSensitive()) |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java |
| index 404ba72846..5bd4e2b351 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java |
| @@ -90,6 +90,14 @@ public abstract class SparkDistributedDataScanTestBase |
| .master("local[2]") |
| .config("spark.serializer", serializer) |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| } |
| } |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java |
| index 9361c63176..44c59c8971 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java |
| @@ -69,6 +69,14 @@ public class TestSparkDistributedDataScanDeletes |
| .master("local[2]") |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| } |
| |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java |
| index a218f965ea..b1e0fcf2ca 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java |
| @@ -62,6 +62,14 @@ public class TestSparkDistributedDataScanFilterFiles |
| .master("local[2]") |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| } |
| |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java |
| index acd4688440..fffc604b48 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java |
| @@ -59,6 +59,14 @@ public class TestSparkDistributedDataScanReporting |
| .master("local[2]") |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| } |
| |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java |
| index 3e8953fb95..65a8d8ca35 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java |
| @@ -77,6 +77,14 @@ public abstract class SparkTestBase extends SparkTestHelperBase { |
| .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) |
| .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .enableHiveSupport() |
| .getOrCreate(); |
| |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTest.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTest.java |
| index ad969384c5..f3c2b833e3 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTest.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTest.java |
| @@ -41,7 +41,6 @@ import org.apache.iceberg.types.Types.MapType; |
| import org.apache.iceberg.types.Types.StructType; |
| import org.apache.iceberg.util.DateTimeUtil; |
| import org.assertj.core.api.Assumptions; |
| -import org.assertj.core.api.Condition; |
| import org.junit.jupiter.api.Test; |
| import org.junit.jupiter.api.io.TempDir; |
| import org.junit.jupiter.params.ParameterizedTest; |
| @@ -283,12 +282,6 @@ public abstract class AvroDataTest { |
| .build()); |
| |
| assertThatThrownBy(() -> writeAndValidate(writeSchema, expectedSchema)) |
| - .has( |
| - new Condition<>( |
| - t -> |
| - IllegalArgumentException.class.isInstance(t) |
| - || IllegalArgumentException.class.isInstance(t.getCause()), |
| - "Expecting a throwable or cause that is an instance of IllegalArgumentException")) |
| .hasMessageContaining("Missing required field: missing_str"); |
| } |
| |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java |
| index 3a269740b7..2035edf93a 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java |
| @@ -54,7 +54,18 @@ public abstract class ScanTestBase extends AvroDataTest { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - ScanTestBase.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + ScanTestBase.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| ScanTestBase.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); |
| } |
| |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java |
| index 724c6edde2..13c7be5103 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java |
| @@ -103,7 +103,18 @@ public class TestCompressionSettings extends SparkCatalogTestBase { |
| |
| @BeforeClass |
| public static void startSpark() { |
| - TestCompressionSettings.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestCompressionSettings.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @Parameterized.AfterParam |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java |
| index 013b8d4386..4c62e88fe9 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java |
| @@ -76,7 +76,18 @@ public class TestDataSourceOptions extends SparkTestBaseWithCatalog { |
| |
| @BeforeClass |
| public static void startSpark() { |
| - TestDataSourceOptions.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestDataSourceOptions.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterClass |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java |
| index ba13d005bd..8c1223f93c 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java |
| @@ -118,7 +118,18 @@ public class TestFilteredScan { |
| |
| @BeforeClass |
| public static void startSpark() { |
| - TestFilteredScan.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestFilteredScan.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| |
| // define UDFs used by partition tests |
| Function<Object, Integer> bucket4 = Transforms.bucket(4).bind(Types.LongType.get()); |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java |
| index 9f97753094..d715c6a3a5 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java |
| @@ -94,7 +94,18 @@ public class TestForwardCompatibility { |
| |
| @BeforeClass |
| public static void startSpark() { |
| - TestForwardCompatibility.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestForwardCompatibility.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterClass |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java |
| index 0154506f86..edd49d5e5b 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java |
| @@ -46,7 +46,18 @@ public class TestIcebergSpark { |
| |
| @BeforeClass |
| public static void startSpark() { |
| - TestIcebergSpark.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestIcebergSpark.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterClass |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java |
| index 639d37c793..0e69c8d992 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java |
| @@ -111,7 +111,18 @@ public class TestPartitionPruning { |
| |
| @BeforeClass |
| public static void startSpark() { |
| - TestPartitionPruning.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestPartitionPruning.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); |
| |
| String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java |
| index ad0984ef42..42a539ffd9 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java |
| @@ -104,7 +104,18 @@ public class TestPartitionValues { |
| |
| @BeforeClass |
| public static void startSpark() { |
| - TestPartitionValues.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestPartitionValues.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterClass |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java |
| index 9fc576dde5..69913e15cc 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java |
| @@ -83,7 +83,18 @@ public class TestSnapshotSelection { |
| |
| @BeforeClass |
| public static void startSpark() { |
| - TestSnapshotSelection.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestSnapshotSelection.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterClass |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java |
| index 16fde3c954..b28970ca18 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java |
| @@ -121,7 +121,18 @@ public class TestSparkDataFile { |
| |
| @BeforeClass |
| public static void startSpark() { |
| - TestSparkDataFile.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestSparkDataFile.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); |
| } |
| |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java |
| index 63c18277aa..3c00c4a20a 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java |
| @@ -89,7 +89,18 @@ public class TestSparkDataWrite { |
| |
| @BeforeClass |
| public static void startSpark() { |
| - TestSparkDataWrite.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestSparkDataWrite.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @Parameterized.AfterParam |
| @@ -138,7 +149,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); |
| Assert.assertEquals("Result rows should match", expected, actual); |
| for (ManifestFile manifest : |
| @@ -208,7 +219,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); |
| Assert.assertEquals("Result rows should match", expected, actual); |
| } |
| @@ -254,7 +265,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); |
| Assert.assertEquals("Result rows should match", expected, actual); |
| } |
| @@ -307,7 +318,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); |
| Assert.assertEquals("Result rows should match", expected, actual); |
| } |
| @@ -350,7 +361,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); |
| Assert.assertEquals("Result rows should match", expected, actual); |
| } |
| @@ -390,7 +401,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); |
| Assert.assertEquals("Result rows should match", expected, actual); |
| |
| @@ -455,7 +466,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); |
| Assert.assertEquals("Result rows should match", expected, actual); |
| } |
| @@ -619,7 +630,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| Assert.assertEquals("Number of rows should match", expected.size(), actual.size()); |
| Assert.assertEquals("Result rows should match", expected, actual); |
| |
| @@ -705,7 +716,7 @@ public class TestSparkDataWrite { |
| // Since write and commit succeeded, the rows should be readable |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| Assert.assertEquals( |
| "Number of rows should match", records.size() + records2.size(), actual.size()); |
| assertThat(actual) |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java |
| index 3a4b235c46..9beb95022d 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java |
| @@ -86,7 +86,18 @@ public class TestSparkReadProjection extends TestReadProjection { |
| |
| @BeforeClass |
| public static void startSpark() { |
| - TestSparkReadProjection.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestSparkReadProjection.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| ImmutableMap<String, String> config = |
| ImmutableMap.of( |
| "type", "hive", |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java |
| index dda49b4946..edf2b24a1c 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java |
| @@ -131,7 +131,23 @@ public class TestSparkReaderDeletes extends DeleteReadTests { |
| .config("spark.ui.liveUpdate.period", 0) |
| .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .enableHiveSupport() |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| |
| catalog = |
| @@ -199,7 +215,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { |
| } |
| |
| protected boolean countDeletes() { |
| - return true; |
| + // TODO: Enable once iceberg-rust exposes delete count metrics to Comet |
| + return false; |
| } |
| |
| @Override |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java |
| index e5831b76e4..bcc5cd0a58 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java |
| @@ -176,7 +176,23 @@ public class TestSparkReaderWithBloomFilter { |
| SparkSession.builder() |
| .master("local[2]") |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .enableHiveSupport() |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| |
| catalog = |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java |
| index f420f1b955..f5fd3cd91d 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java |
| @@ -66,6 +66,14 @@ public class TestStructuredStreaming { |
| SparkSession.builder() |
| .master("local[2]") |
| .config("spark.sql.shuffle.partitions", 4) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| } |
| |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java |
| index ac674e2e62..00e064c64d 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java |
| @@ -73,7 +73,18 @@ public class TestTimestampWithoutZone extends SparkTestBase { |
| |
| @BeforeClass |
| public static void startSpark() { |
| - TestTimestampWithoutZone.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestTimestampWithoutZone.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterClass |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java |
| index 73827b309b..6eaa915d5b 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java |
| @@ -80,7 +80,18 @@ public class TestWriteMetricsConfig { |
| |
| @BeforeClass |
| public static void startSpark() { |
| - TestWriteMetricsConfig.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestWriteMetricsConfig.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); |
| } |
| |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java |
| index 1a4e2f3e1c..f2228098ec 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java |
| @@ -65,7 +65,23 @@ public class TestAggregatePushDown extends SparkCatalogTestBase { |
| SparkSession.builder() |
| .master("local[2]") |
| .config("spark.sql.iceberg.aggregate_pushdown", "true") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .enableHiveSupport() |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| |
| SparkTestBase.catalog = |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java |
| index f92055ab7a..95fde91f19 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java |
| @@ -585,9 +585,7 @@ public class TestFilterPushDown extends SparkTestBaseWithCatalog { |
| String planAsString = sparkPlan.toString().replaceAll("#(\\d+L?)", ""); |
| |
| if (sparkFilter != null) { |
| - assertThat(planAsString) |
| - .as("Post scan filter should match") |
| - .contains("Filter (" + sparkFilter + ")"); |
| + assertThat(planAsString).as("Post scan filter should match").contains("CometFilter"); |
| } else { |
| assertThat(planAsString).as("Should be no post scan filter").doesNotContain("Filter ("); |
| } |
| diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java |
| index 8a1ec5060f..3611521cc4 100644 |
| --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java |
| +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java |
| @@ -605,7 +605,7 @@ public class TestStoragePartitionedJoins extends SparkTestBaseWithCatalog { |
| + "FROM %s t1 " |
| + "INNER JOIN %s t2 " |
| + "ON t1.id = t2.id AND t1.%s = t2.%s " |
| - + "ORDER BY t1.id, t1.%s", |
| + + "ORDER BY t1.id, t1.%s, t1.salary", |
| sourceColumnName, |
| tableName, |
| tableName(OTHER_TABLE_NAME), |
| diff --git a/spark/v3.5/build.gradle b/spark/v3.5/build.gradle |
| index e2d2c7a7ac..95cd3c08b5 100644 |
| --- a/spark/v3.5/build.gradle |
| +++ b/spark/v3.5/build.gradle |
| @@ -75,7 +75,7 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") { |
| exclude group: 'org.roaringbitmap' |
| } |
| |
| - compileOnly "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:0.5.0" |
| + compileOnly "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" |
| |
| implementation libs.parquet.column |
| implementation libs.parquet.hadoop |
| @@ -183,7 +183,7 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer |
| testImplementation libs.avro.avro |
| testImplementation libs.parquet.hadoop |
| testImplementation libs.awaitility |
| - testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:0.5.0" |
| + testImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" |
| |
| // Required because we remove antlr plugin dependencies from the compile configuration, see note above |
| runtimeOnly libs.antlr.runtime |
| @@ -263,6 +263,7 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio |
| integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') |
| integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts') |
| integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts') |
| + integrationImplementation "org.apache.datafusion:comet-spark-spark${sparkMajorVersion}_${scalaVersion}:${libs.versions.comet.get()}" |
| |
| // runtime dependencies for running Hive Catalog based integration test |
| integrationRuntimeOnly project(':iceberg-hive-metastore') |
| diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java |
| index 578845e3da..3cc1598035 100644 |
| --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java |
| +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/ExtensionsTestBase.java |
| @@ -57,6 +57,14 @@ public abstract class ExtensionsTestBase extends CatalogTestBase { |
| .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") |
| .config( |
| SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), String.valueOf(RANDOM.nextBoolean())) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .enableHiveSupport() |
| .getOrCreate(); |
| |
| diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java |
| index ade19de36f..366a094cfd 100644 |
| --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java |
| +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java |
| @@ -56,6 +56,14 @@ public class TestCallStatementParser { |
| .master("local[2]") |
| .config("spark.sql.extensions", IcebergSparkSessionExtensions.class.getName()) |
| .config("spark.extra.prop", "value") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| TestCallStatementParser.parser = spark.sessionState().sqlParser(); |
| } |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java |
| index 64edb1002e..4706106c2e 100644 |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java |
| @@ -179,6 +179,14 @@ public class DeleteOrphanFilesBenchmark { |
| .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") |
| .config("spark.sql.catalog.spark_catalog.warehouse", catalogWarehouse()) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .master("local"); |
| spark = builder.getOrCreate(); |
| } |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java |
| index a5d0456b0b..815de72f4a 100644 |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/action/IcebergSortCompactionBenchmark.java |
| @@ -392,6 +392,14 @@ public class IcebergSortCompactionBenchmark { |
| "spark.sql.catalog.spark_catalog", "org.apache.iceberg.spark.SparkSessionCatalog") |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") |
| .config("spark.sql.catalog.spark_catalog.warehouse", getCatalogWarehouse()) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .master("local[*]"); |
| spark = builder.getOrCreate(); |
| Configuration sparkHadoopConf = spark.sessionState().newHadoopConf(); |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java |
| index c6794e43c6..ada3a83e3c 100644 |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVReaderBenchmark.java |
| @@ -239,6 +239,14 @@ public class DVReaderBenchmark { |
| .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") |
| .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .master("local[*]") |
| .getOrCreate(); |
| } |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java |
| index ac74fb5a10..bafca20a1c 100644 |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/DVWriterBenchmark.java |
| @@ -223,6 +223,14 @@ public class DVWriterBenchmark { |
| .config("spark.sql.catalog.spark_catalog", SparkSessionCatalog.class.getName()) |
| .config("spark.sql.catalog.spark_catalog.type", "hadoop") |
| .config("spark.sql.catalog.spark_catalog.warehouse", newWarehouseDir()) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .master("local[*]") |
| .getOrCreate(); |
| } |
| diff --git a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java |
| index 68c537e34a..5ef031963a 100644 |
| --- a/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java |
| +++ b/spark/v3.5/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java |
| @@ -94,7 +94,17 @@ public abstract class IcebergSourceBenchmark { |
| } |
| |
| protected void setupSpark(boolean enableDictionaryEncoding) { |
| - SparkSession.Builder builder = SparkSession.builder().config("spark.ui.enabled", false); |
| + SparkSession.Builder builder = |
| + SparkSession.builder() |
| + .config("spark.ui.enabled", false) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g"); |
| if (!enableDictionaryEncoding) { |
| builder |
| .config("parquet.dictionary.page.size", "1") |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java |
| deleted file mode 100644 |
| index 4794863ab1..0000000000 |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnReader.java |
| +++ /dev/null |
| @@ -1,150 +0,0 @@ |
| -/* |
| - * Licensed to the Apache Software Foundation (ASF) under one |
| - * or more contributor license agreements. See the NOTICE file |
| - * distributed with this work for additional information |
| - * regarding copyright ownership. The ASF licenses this file |
| - * to you under the Apache License, Version 2.0 (the |
| - * "License"); you may not use this file except in compliance |
| - * with the License. You may obtain a copy of the License at |
| - * |
| - * http://www.apache.org/licenses/LICENSE-2.0 |
| - * |
| - * Unless required by applicable law or agreed to in writing, |
| - * software distributed under the License is distributed on an |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| - * KIND, either express or implied. See the License for the |
| - * specific language governing permissions and limitations |
| - * under the License. |
| - */ |
| -package org.apache.iceberg.spark.data.vectorized; |
| - |
| -import java.io.IOException; |
| -import java.util.Map; |
| -import org.apache.comet.parquet.AbstractColumnReader; |
| -import org.apache.comet.parquet.ColumnReader; |
| -import org.apache.comet.parquet.TypeUtil; |
| -import org.apache.comet.parquet.Utils; |
| -import org.apache.comet.shaded.arrow.c.CometSchemaImporter; |
| -import org.apache.comet.shaded.arrow.memory.RootAllocator; |
| -import org.apache.iceberg.parquet.VectorizedReader; |
| -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; |
| -import org.apache.iceberg.spark.SparkSchemaUtil; |
| -import org.apache.iceberg.types.Types; |
| -import org.apache.parquet.column.ColumnDescriptor; |
| -import org.apache.parquet.column.page.PageReadStore; |
| -import org.apache.parquet.column.page.PageReader; |
| -import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; |
| -import org.apache.parquet.hadoop.metadata.ColumnPath; |
| -import org.apache.spark.sql.types.DataType; |
| -import org.apache.spark.sql.types.Metadata; |
| -import org.apache.spark.sql.types.StructField; |
| -import org.apache.spark.sql.vectorized.ColumnVector; |
| - |
| -class CometColumnReader implements VectorizedReader<ColumnVector> { |
| - // use the Comet default batch size |
| - public static final int DEFAULT_BATCH_SIZE = 8192; |
| - |
| - private final ColumnDescriptor descriptor; |
| - private final DataType sparkType; |
| - |
| - // The delegated ColumnReader from Comet side |
| - private AbstractColumnReader delegate; |
| - private boolean initialized = false; |
| - private int batchSize = DEFAULT_BATCH_SIZE; |
| - private CometSchemaImporter importer; |
| - |
| - CometColumnReader(DataType sparkType, ColumnDescriptor descriptor) { |
| - this.sparkType = sparkType; |
| - this.descriptor = descriptor; |
| - } |
| - |
| - CometColumnReader(Types.NestedField field) { |
| - DataType dataType = SparkSchemaUtil.convert(field.type()); |
| - StructField structField = new StructField(field.name(), dataType, false, Metadata.empty()); |
| - this.sparkType = dataType; |
| - this.descriptor = TypeUtil.convertToParquet(structField); |
| - } |
| - |
| - public AbstractColumnReader delegate() { |
| - return delegate; |
| - } |
| - |
| - void setDelegate(AbstractColumnReader delegate) { |
| - this.delegate = delegate; |
| - } |
| - |
| - void setInitialized(boolean initialized) { |
| - this.initialized = initialized; |
| - } |
| - |
| - public int batchSize() { |
| - return batchSize; |
| - } |
| - |
| - /** |
| - * This method is to initialized/reset the CometColumnReader. This needs to be called for each row |
| - * group after readNextRowGroup, so a new dictionary encoding can be set for each of the new row |
| - * groups. |
| - */ |
| - public void reset() { |
| - if (importer != null) { |
| - importer.close(); |
| - } |
| - |
| - if (delegate != null) { |
| - delegate.close(); |
| - } |
| - |
| - this.importer = new CometSchemaImporter(new RootAllocator()); |
| - this.delegate = Utils.getColumnReader(sparkType, descriptor, importer, batchSize, false, false); |
| - this.initialized = true; |
| - } |
| - |
| - public ColumnDescriptor descriptor() { |
| - return descriptor; |
| - } |
| - |
| - /** Returns the Spark data type for this column. */ |
| - public DataType sparkType() { |
| - return sparkType; |
| - } |
| - |
| - /** |
| - * Set the page reader to be 'pageReader'. |
| - * |
| - * <p>NOTE: this should be called before reading a new Parquet column chunk, and after {@link |
| - * CometColumnReader#reset} is called. |
| - */ |
| - public void setPageReader(PageReader pageReader) throws IOException { |
| - Preconditions.checkState(initialized, "Invalid state: 'reset' should be called first"); |
| - ((ColumnReader) delegate).setPageReader(pageReader); |
| - } |
| - |
| - @Override |
| - public void close() { |
| - // close resources on native side |
| - if (importer != null) { |
| - importer.close(); |
| - } |
| - |
| - if (delegate != null) { |
| - delegate.close(); |
| - } |
| - } |
| - |
| - @Override |
| - public void setBatchSize(int size) { |
| - this.batchSize = size; |
| - } |
| - |
| - @Override |
| - public void setRowGroupInfo( |
| - PageReadStore pageReadStore, Map<ColumnPath, ColumnChunkMetaData> map, long size) { |
| - throw new UnsupportedOperationException("Not supported"); |
| - } |
| - |
| - @Override |
| - public ColumnVector read(ColumnVector reuse, int numRowsToRead) { |
| - throw new UnsupportedOperationException("Not supported"); |
| - } |
| -} |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java |
| deleted file mode 100644 |
| index 1440e5d1d3..0000000000 |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometColumnarBatchReader.java |
| +++ /dev/null |
| @@ -1,203 +0,0 @@ |
| -/* |
| - * Licensed to the Apache Software Foundation (ASF) under one |
| - * or more contributor license agreements. See the NOTICE file |
| - * distributed with this work for additional information |
| - * regarding copyright ownership. The ASF licenses this file |
| - * to you under the Apache License, Version 2.0 (the |
| - * "License"); you may not use this file except in compliance |
| - * with the License. You may obtain a copy of the License at |
| - * |
| - * http://www.apache.org/licenses/LICENSE-2.0 |
| - * |
| - * Unless required by applicable law or agreed to in writing, |
| - * software distributed under the License is distributed on an |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| - * KIND, either express or implied. See the License for the |
| - * specific language governing permissions and limitations |
| - * under the License. |
| - */ |
| -package org.apache.iceberg.spark.data.vectorized; |
| - |
| -import java.io.IOException; |
| -import java.io.UncheckedIOException; |
| -import java.util.List; |
| -import java.util.Map; |
| -import org.apache.comet.parquet.AbstractColumnReader; |
| -import org.apache.comet.parquet.BatchReader; |
| -import org.apache.iceberg.Schema; |
| -import org.apache.iceberg.data.DeleteFilter; |
| -import org.apache.iceberg.parquet.VectorizedReader; |
| -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; |
| -import org.apache.iceberg.spark.SparkSchemaUtil; |
| -import org.apache.iceberg.util.Pair; |
| -import org.apache.parquet.column.page.PageReadStore; |
| -import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; |
| -import org.apache.parquet.hadoop.metadata.ColumnPath; |
| -import org.apache.spark.sql.catalyst.InternalRow; |
| -import org.apache.spark.sql.vectorized.ColumnVector; |
| -import org.apache.spark.sql.vectorized.ColumnarBatch; |
| - |
| -/** |
| - * {@link VectorizedReader} that returns Spark's {@link ColumnarBatch} to support Spark's vectorized |
| - * read path. The {@link ColumnarBatch} returned is created by passing in the Arrow vectors |
| - * populated via delegated read calls to {@link CometColumnReader VectorReader(s)}. |
| - */ |
| -@SuppressWarnings("checkstyle:VisibilityModifier") |
| -class CometColumnarBatchReader implements VectorizedReader<ColumnarBatch> { |
| - |
| - private final CometColumnReader[] readers; |
| - private final boolean hasIsDeletedColumn; |
| - |
| - // The delegated BatchReader on the Comet side does the real work of loading a batch of rows. |
| - // The Comet BatchReader contains an array of ColumnReader. There is no need to explicitly call |
| - // ColumnReader.readBatch; instead, BatchReader.nextBatch will be called, which underneath calls |
| - // ColumnReader.readBatch. The only exception is DeleteColumnReader, because at the time of |
| - // calling BatchReader.nextBatch, the isDeleted value is not yet available, so |
| - // DeleteColumnReader.readBatch must be called explicitly later, after the isDeleted value is |
| - // available. |
| - private final BatchReader delegate; |
| - private DeleteFilter<InternalRow> deletes = null; |
| - private long rowStartPosInBatch = 0; |
| - |
| - CometColumnarBatchReader(List<VectorizedReader<?>> readers, Schema schema) { |
| - this.readers = |
| - readers.stream().map(CometColumnReader.class::cast).toArray(CometColumnReader[]::new); |
| - this.hasIsDeletedColumn = |
| - readers.stream().anyMatch(reader -> reader instanceof CometDeleteColumnReader); |
| - |
| - AbstractColumnReader[] abstractColumnReaders = new AbstractColumnReader[readers.size()]; |
| - this.delegate = new BatchReader(abstractColumnReaders); |
| - delegate.setSparkSchema(SparkSchemaUtil.convert(schema)); |
| - } |
| - |
| - @Override |
| - public void setRowGroupInfo( |
| - PageReadStore pageStore, Map<ColumnPath, ColumnChunkMetaData> metaData, long rowPosition) { |
| - setRowGroupInfo(pageStore, metaData); |
| - } |
| - |
| - @Override |
| - public void setRowGroupInfo( |
| - PageReadStore pageStore, Map<ColumnPath, ColumnChunkMetaData> metaData) { |
| - for (int i = 0; i < readers.length; i++) { |
| - try { |
| - if (!(readers[i] instanceof CometConstantColumnReader) |
| - && !(readers[i] instanceof CometPositionColumnReader) |
| - && !(readers[i] instanceof CometDeleteColumnReader)) { |
| - readers[i].reset(); |
| - readers[i].setPageReader(pageStore.getPageReader(readers[i].descriptor())); |
| - } |
| - } catch (IOException e) { |
| - throw new UncheckedIOException("Failed to setRowGroupInfo for Comet vectorization", e); |
| - } |
| - } |
| - |
| - for (int i = 0; i < readers.length; i++) { |
| - delegate.getColumnReaders()[i] = this.readers[i].delegate(); |
| - } |
| - |
| - this.rowStartPosInBatch = |
| - pageStore |
| - .getRowIndexOffset() |
| - .orElseThrow( |
| - () -> |
| - new IllegalArgumentException( |
| - "PageReadStore does not contain row index offset")); |
| - } |
| - |
| - public void setDeleteFilter(DeleteFilter<InternalRow> deleteFilter) { |
| - this.deletes = deleteFilter; |
| - } |
| - |
| - @Override |
| - public final ColumnarBatch read(ColumnarBatch reuse, int numRowsToRead) { |
| - ColumnarBatch columnarBatch = new ColumnBatchLoader(numRowsToRead).loadDataToColumnBatch(); |
| - rowStartPosInBatch += numRowsToRead; |
| - return columnarBatch; |
| - } |
| - |
| - @Override |
| - public void setBatchSize(int batchSize) { |
| - for (CometColumnReader reader : readers) { |
| - if (reader != null) { |
| - reader.setBatchSize(batchSize); |
| - } |
| - } |
| - } |
| - |
| - @Override |
| - public void close() { |
| - for (CometColumnReader reader : readers) { |
| - if (reader != null) { |
| - reader.close(); |
| - } |
| - } |
| - } |
| - |
| - private class ColumnBatchLoader { |
| - private final int batchSize; |
| - |
| - ColumnBatchLoader(int numRowsToRead) { |
| - Preconditions.checkArgument( |
| - numRowsToRead > 0, "Invalid number of rows to read: %s", numRowsToRead); |
| - this.batchSize = numRowsToRead; |
| - } |
| - |
| - ColumnarBatch loadDataToColumnBatch() { |
| - ColumnVector[] vectors = readDataToColumnVectors(); |
| - int numLiveRows = batchSize; |
| - |
| - if (hasIsDeletedColumn) { |
| - boolean[] isDeleted = buildIsDeleted(vectors); |
| - readDeletedColumn(vectors, isDeleted); |
| - } else { |
| - Pair<int[], Integer> pair = buildRowIdMapping(vectors); |
| - if (pair != null) { |
| - int[] rowIdMapping = pair.first(); |
| - numLiveRows = pair.second(); |
| - for (int i = 0; i < vectors.length; i++) { |
| - vectors[i] = new ColumnVectorWithFilter(vectors[i], rowIdMapping); |
| - } |
| - } |
| - } |
| - |
| - if (deletes != null && deletes.hasEqDeletes()) { |
| - vectors = ColumnarBatchUtil.removeExtraColumns(deletes, vectors); |
| - } |
| - |
| - ColumnarBatch batch = new ColumnarBatch(vectors); |
| - batch.setNumRows(numLiveRows); |
| - return batch; |
| - } |
| - |
| - private boolean[] buildIsDeleted(ColumnVector[] vectors) { |
| - return ColumnarBatchUtil.buildIsDeleted(vectors, deletes, rowStartPosInBatch, batchSize); |
| - } |
| - |
| - private Pair<int[], Integer> buildRowIdMapping(ColumnVector[] vectors) { |
| - return ColumnarBatchUtil.buildRowIdMapping(vectors, deletes, rowStartPosInBatch, batchSize); |
| - } |
| - |
| - ColumnVector[] readDataToColumnVectors() { |
| - ColumnVector[] columnVectors = new ColumnVector[readers.length]; |
| - // Fetch rows for all readers in the delegate |
| - delegate.nextBatch(batchSize); |
| - for (int i = 0; i < readers.length; i++) { |
| - columnVectors[i] = readers[i].delegate().currentBatch(); |
| - } |
| - |
| - return columnVectors; |
| - } |
| - |
| - void readDeletedColumn(ColumnVector[] columnVectors, boolean[] isDeleted) { |
| - for (int i = 0; i < readers.length; i++) { |
| - if (readers[i] instanceof CometDeleteColumnReader) { |
| - CometDeleteColumnReader deleteColumnReader = new CometDeleteColumnReader<>(isDeleted); |
| - deleteColumnReader.setBatchSize(batchSize); |
| - deleteColumnReader.delegate().readBatch(batchSize); |
| - columnVectors[i] = deleteColumnReader.delegate().currentBatch(); |
| - } |
| - } |
| - } |
| - } |
| -} |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java |
| deleted file mode 100644 |
| index 047c96314b..0000000000 |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometConstantColumnReader.java |
| +++ /dev/null |
| @@ -1,65 +0,0 @@ |
| -/* |
| - * Licensed to the Apache Software Foundation (ASF) under one |
| - * or more contributor license agreements. See the NOTICE file |
| - * distributed with this work for additional information |
| - * regarding copyright ownership. The ASF licenses this file |
| - * to you under the Apache License, Version 2.0 (the |
| - * "License"); you may not use this file except in compliance |
| - * with the License. You may obtain a copy of the License at |
| - * |
| - * http://www.apache.org/licenses/LICENSE-2.0 |
| - * |
| - * Unless required by applicable law or agreed to in writing, |
| - * software distributed under the License is distributed on an |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| - * KIND, either express or implied. See the License for the |
| - * specific language governing permissions and limitations |
| - * under the License. |
| - */ |
| -package org.apache.iceberg.spark.data.vectorized; |
| - |
| -import java.math.BigDecimal; |
| -import java.nio.ByteBuffer; |
| -import org.apache.comet.parquet.ConstantColumnReader; |
| -import org.apache.iceberg.types.Types; |
| -import org.apache.spark.sql.types.DataType; |
| -import org.apache.spark.sql.types.DataTypes; |
| -import org.apache.spark.sql.types.Decimal; |
| -import org.apache.spark.sql.types.DecimalType; |
| -import org.apache.spark.unsafe.types.UTF8String; |
| - |
| -class CometConstantColumnReader<T> extends CometColumnReader { |
| - |
| - CometConstantColumnReader(T value, Types.NestedField field) { |
| - super(field); |
| - // use delegate to set constant value on the native side to be consumed by native execution. |
| - setDelegate( |
| - new ConstantColumnReader(sparkType(), descriptor(), convertToSparkValue(value), false)); |
| - } |
| - |
| - @Override |
| - public void setBatchSize(int batchSize) { |
| - super.setBatchSize(batchSize); |
| - delegate().setBatchSize(batchSize); |
| - setInitialized(true); |
| - } |
| - |
| - private Object convertToSparkValue(T value) { |
| - DataType dataType = sparkType(); |
| - // Match the value to Spark internal type if necessary |
| - if (dataType == DataTypes.StringType && value instanceof String) { |
| - // the internal type for StringType is UTF8String |
| - return UTF8String.fromString((String) value); |
| - } else if (dataType instanceof DecimalType && value instanceof BigDecimal) { |
| - // the internal type for DecimalType is Decimal |
| - return Decimal.apply((BigDecimal) value); |
| - } else if (dataType == DataTypes.BinaryType && value instanceof ByteBuffer) { |
| - // the internal type for DecimalType is byte[] |
| - // Iceberg default value should always use HeapBufferBuffer, so calling ByteBuffer.array() |
| - // should be safe. |
| - return ((ByteBuffer) value).array(); |
| - } else { |
| - return value; |
| - } |
| - } |
| -} |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java |
| deleted file mode 100644 |
| index 6235bfe486..0000000000 |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometDeleteColumnReader.java |
| +++ /dev/null |
| @@ -1,75 +0,0 @@ |
| -/* |
| - * Licensed to the Apache Software Foundation (ASF) under one |
| - * or more contributor license agreements. See the NOTICE file |
| - * distributed with this work for additional information |
| - * regarding copyright ownership. The ASF licenses this file |
| - * to you under the Apache License, Version 2.0 (the |
| - * "License"); you may not use this file except in compliance |
| - * with the License. You may obtain a copy of the License at |
| - * |
| - * http://www.apache.org/licenses/LICENSE-2.0 |
| - * |
| - * Unless required by applicable law or agreed to in writing, |
| - * software distributed under the License is distributed on an |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| - * KIND, either express or implied. See the License for the |
| - * specific language governing permissions and limitations |
| - * under the License. |
| - */ |
| -package org.apache.iceberg.spark.data.vectorized; |
| - |
| -import org.apache.comet.parquet.MetadataColumnReader; |
| -import org.apache.comet.parquet.Native; |
| -import org.apache.comet.parquet.TypeUtil; |
| -import org.apache.iceberg.MetadataColumns; |
| -import org.apache.iceberg.types.Types; |
| -import org.apache.spark.sql.types.DataTypes; |
| -import org.apache.spark.sql.types.Metadata; |
| -import org.apache.spark.sql.types.StructField; |
| - |
| -class CometDeleteColumnReader<T> extends CometColumnReader { |
| - CometDeleteColumnReader(Types.NestedField field) { |
| - super(field); |
| - setDelegate(new DeleteColumnReader()); |
| - } |
| - |
| - CometDeleteColumnReader(boolean[] isDeleted) { |
| - super(MetadataColumns.IS_DELETED); |
| - setDelegate(new DeleteColumnReader(isDeleted)); |
| - } |
| - |
| - @Override |
| - public void setBatchSize(int batchSize) { |
| - super.setBatchSize(batchSize); |
| - delegate().setBatchSize(batchSize); |
| - setInitialized(true); |
| - } |
| - |
| - private static class DeleteColumnReader extends MetadataColumnReader { |
| - private boolean[] isDeleted; |
| - |
| - DeleteColumnReader() { |
| - super( |
| - DataTypes.BooleanType, |
| - TypeUtil.convertToParquet( |
| - new StructField("_deleted", DataTypes.BooleanType, false, Metadata.empty())), |
| - false /* useDecimal128 = false */, |
| - false /* isConstant = false */); |
| - this.isDeleted = new boolean[0]; |
| - } |
| - |
| - DeleteColumnReader(boolean[] isDeleted) { |
| - this(); |
| - this.isDeleted = isDeleted; |
| - } |
| - |
| - @Override |
| - public void readBatch(int total) { |
| - Native.resetBatch(nativeHandle); |
| - // set isDeleted on the native side to be consumed by native execution |
| - Native.setIsDeleted(nativeHandle, isDeleted); |
| - |
| - super.readBatch(total); |
| - } |
| - } |
| -} |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java |
| deleted file mode 100644 |
| index bcc0e514c2..0000000000 |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometPositionColumnReader.java |
| +++ /dev/null |
| @@ -1,62 +0,0 @@ |
| -/* |
| - * Licensed to the Apache Software Foundation (ASF) under one |
| - * or more contributor license agreements. See the NOTICE file |
| - * distributed with this work for additional information |
| - * regarding copyright ownership. The ASF licenses this file |
| - * to you under the Apache License, Version 2.0 (the |
| - * "License"); you may not use this file except in compliance |
| - * with the License. You may obtain a copy of the License at |
| - * |
| - * http://www.apache.org/licenses/LICENSE-2.0 |
| - * |
| - * Unless required by applicable law or agreed to in writing, |
| - * software distributed under the License is distributed on an |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| - * KIND, either express or implied. See the License for the |
| - * specific language governing permissions and limitations |
| - * under the License. |
| - */ |
| -package org.apache.iceberg.spark.data.vectorized; |
| - |
| -import org.apache.comet.parquet.MetadataColumnReader; |
| -import org.apache.comet.parquet.Native; |
| -import org.apache.iceberg.types.Types; |
| -import org.apache.parquet.column.ColumnDescriptor; |
| -import org.apache.spark.sql.types.DataTypes; |
| - |
| -class CometPositionColumnReader extends CometColumnReader { |
| - CometPositionColumnReader(Types.NestedField field) { |
| - super(field); |
| - setDelegate(new PositionColumnReader(descriptor())); |
| - } |
| - |
| - @Override |
| - public void setBatchSize(int batchSize) { |
| - super.setBatchSize(batchSize); |
| - delegate().setBatchSize(batchSize); |
| - setInitialized(true); |
| - } |
| - |
| - private static class PositionColumnReader extends MetadataColumnReader { |
| - /** The current position value of the column that are used to initialize this column reader. */ |
| - private long position; |
| - |
| - PositionColumnReader(ColumnDescriptor descriptor) { |
| - super( |
| - DataTypes.LongType, |
| - descriptor, |
| - false /* useDecimal128 = false */, |
| - false /* isConstant = false */); |
| - } |
| - |
| - @Override |
| - public void readBatch(int total) { |
| - Native.resetBatch(nativeHandle); |
| - // set position on the native side to be consumed by native execution |
| - Native.setPosition(nativeHandle, position, total); |
| - position += total; |
| - |
| - super.readBatch(total); |
| - } |
| - } |
| -} |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java |
| deleted file mode 100644 |
| index d36f1a7274..0000000000 |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/CometVectorizedReaderBuilder.java |
| +++ /dev/null |
| @@ -1,147 +0,0 @@ |
| -/* |
| - * Licensed to the Apache Software Foundation (ASF) under one |
| - * or more contributor license agreements. See the NOTICE file |
| - * distributed with this work for additional information |
| - * regarding copyright ownership. The ASF licenses this file |
| - * to you under the Apache License, Version 2.0 (the |
| - * "License"); you may not use this file except in compliance |
| - * with the License. You may obtain a copy of the License at |
| - * |
| - * http://www.apache.org/licenses/LICENSE-2.0 |
| - * |
| - * Unless required by applicable law or agreed to in writing, |
| - * software distributed under the License is distributed on an |
| - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| - * KIND, either express or implied. See the License for the |
| - * specific language governing permissions and limitations |
| - * under the License. |
| - */ |
| -package org.apache.iceberg.spark.data.vectorized; |
| - |
| -import java.util.List; |
| -import java.util.Map; |
| -import java.util.function.Function; |
| -import java.util.stream.IntStream; |
| -import org.apache.iceberg.MetadataColumns; |
| -import org.apache.iceberg.Schema; |
| -import org.apache.iceberg.data.DeleteFilter; |
| -import org.apache.iceberg.parquet.TypeWithSchemaVisitor; |
| -import org.apache.iceberg.parquet.VectorizedReader; |
| -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; |
| -import org.apache.iceberg.relocated.com.google.common.collect.Lists; |
| -import org.apache.iceberg.relocated.com.google.common.collect.Maps; |
| -import org.apache.iceberg.spark.SparkSchemaUtil; |
| -import org.apache.iceberg.types.Types; |
| -import org.apache.parquet.column.ColumnDescriptor; |
| -import org.apache.parquet.schema.GroupType; |
| -import org.apache.parquet.schema.MessageType; |
| -import org.apache.parquet.schema.PrimitiveType; |
| -import org.apache.parquet.schema.Type; |
| -import org.apache.spark.sql.catalyst.InternalRow; |
| - |
| -class CometVectorizedReaderBuilder extends TypeWithSchemaVisitor<VectorizedReader<?>> { |
| - |
| - private final MessageType parquetSchema; |
| - private final Schema icebergSchema; |
| - private final Map<Integer, ?> idToConstant; |
| - private final Function<List<VectorizedReader<?>>, VectorizedReader<?>> readerFactory; |
| - private final DeleteFilter<InternalRow> deleteFilter; |
| - |
| - CometVectorizedReaderBuilder( |
| - Schema expectedSchema, |
| - MessageType parquetSchema, |
| - Map<Integer, ?> idToConstant, |
| - Function<List<VectorizedReader<?>>, VectorizedReader<?>> readerFactory, |
| - DeleteFilter<InternalRow> deleteFilter) { |
| - this.parquetSchema = parquetSchema; |
| - this.icebergSchema = expectedSchema; |
| - this.idToConstant = idToConstant; |
| - this.readerFactory = readerFactory; |
| - this.deleteFilter = deleteFilter; |
| - } |
| - |
| - @Override |
| - public VectorizedReader<?> message( |
| - Types.StructType expected, MessageType message, List<VectorizedReader<?>> fieldReaders) { |
| - GroupType groupType = message.asGroupType(); |
| - Map<Integer, VectorizedReader<?>> readersById = Maps.newHashMap(); |
| - List<Type> fields = groupType.getFields(); |
| - |
| - IntStream.range(0, fields.size()) |
| - .filter(pos -> fields.get(pos).getId() != null) |
| - .forEach(pos -> readersById.put(fields.get(pos).getId().intValue(), fieldReaders.get(pos))); |
| - |
| - List<Types.NestedField> icebergFields = |
| - expected != null ? expected.fields() : ImmutableList.of(); |
| - |
| - List<VectorizedReader<?>> reorderedFields = |
| - Lists.newArrayListWithExpectedSize(icebergFields.size()); |
| - |
| - for (Types.NestedField field : icebergFields) { |
| - int id = field.fieldId(); |
| - VectorizedReader<?> reader = readersById.get(id); |
| - if (idToConstant.containsKey(id)) { |
| - CometConstantColumnReader constantReader = |
| - new CometConstantColumnReader<>(idToConstant.get(id), field); |
| - reorderedFields.add(constantReader); |
| - } else if (id == MetadataColumns.ROW_POSITION.fieldId()) { |
| - reorderedFields.add(new CometPositionColumnReader(field)); |
| - } else if (id == MetadataColumns.IS_DELETED.fieldId()) { |
| - CometColumnReader deleteReader = new CometDeleteColumnReader<>(field); |
| - reorderedFields.add(deleteReader); |
| - } else if (reader != null) { |
| - reorderedFields.add(reader); |
| - } else if (field.initialDefault() != null) { |
| - CometColumnReader constantReader = |
| - new CometConstantColumnReader<>(field.initialDefault(), field); |
| - reorderedFields.add(constantReader); |
| - } else if (field.isOptional()) { |
| - CometColumnReader constantReader = new CometConstantColumnReader<>(null, field); |
| - reorderedFields.add(constantReader); |
| - } else { |
| - throw new IllegalArgumentException( |
| - String.format("Missing required field: %s", field.name())); |
| - } |
| - } |
| - return vectorizedReader(reorderedFields); |
| - } |
| - |
| - protected VectorizedReader<?> vectorizedReader(List<VectorizedReader<?>> reorderedFields) { |
| - VectorizedReader<?> reader = readerFactory.apply(reorderedFields); |
| - if (deleteFilter != null) { |
| - ((CometColumnarBatchReader) reader).setDeleteFilter(deleteFilter); |
| - } |
| - return reader; |
| - } |
| - |
| - @Override |
| - public VectorizedReader<?> struct( |
| - Types.StructType expected, GroupType groupType, List<VectorizedReader<?>> fieldReaders) { |
| - if (expected != null) { |
| - throw new UnsupportedOperationException( |
| - "Vectorized reads are not supported yet for struct fields"); |
| - } |
| - return null; |
| - } |
| - |
| - @Override |
| - public VectorizedReader<?> primitive( |
| - org.apache.iceberg.types.Type.PrimitiveType expected, PrimitiveType primitive) { |
| - |
| - if (primitive.getId() == null) { |
| - return null; |
| - } |
| - int parquetFieldId = primitive.getId().intValue(); |
| - ColumnDescriptor desc = parquetSchema.getColumnDescription(currentPath()); |
| - // Nested types not yet supported for vectorized reads |
| - if (desc.getMaxRepetitionLevel() > 0) { |
| - return null; |
| - } |
| - Types.NestedField icebergField = icebergSchema.findField(parquetFieldId); |
| - if (icebergField == null) { |
| - return null; |
| - } |
| - |
| - return new CometColumnReader(SparkSchemaUtil.convert(icebergField.type()), desc); |
| - } |
| -} |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java |
| index b523bc5bff..636ad3be7d 100644 |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java |
| +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java |
| @@ -70,23 +70,6 @@ public class VectorizedSparkParquetReaders { |
| deleteFilter)); |
| } |
| |
| - public static CometColumnarBatchReader buildCometReader( |
| - Schema expectedSchema, |
| - MessageType fileSchema, |
| - Map<Integer, ?> idToConstant, |
| - DeleteFilter<InternalRow> deleteFilter) { |
| - return (CometColumnarBatchReader) |
| - TypeWithSchemaVisitor.visit( |
| - expectedSchema.asStruct(), |
| - fileSchema, |
| - new CometVectorizedReaderBuilder( |
| - expectedSchema, |
| - fileSchema, |
| - idToConstant, |
| - readers -> new CometColumnarBatchReader(readers, expectedSchema), |
| - deleteFilter)); |
| - } |
| - |
| // enables unsafe memory access to avoid costly checks to see if index is within bounds |
| // as long as it is not configured explicitly (see BoundsChecking in Arrow) |
| private static void enableUnsafeMemoryAccess() { |
| diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java |
| index 780e1750a5..25f253eede 100644 |
| --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java |
| +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/BaseBatchReader.java |
| @@ -34,7 +34,6 @@ import org.apache.iceberg.parquet.Parquet; |
| import org.apache.iceberg.relocated.com.google.common.collect.Sets; |
| import org.apache.iceberg.spark.OrcBatchReadConf; |
| import org.apache.iceberg.spark.ParquetBatchReadConf; |
| -import org.apache.iceberg.spark.ParquetReaderType; |
| import org.apache.iceberg.spark.data.vectorized.VectorizedSparkOrcReaders; |
| import org.apache.iceberg.spark.data.vectorized.VectorizedSparkParquetReaders; |
| import org.apache.iceberg.types.TypeUtil; |
| @@ -92,15 +91,9 @@ abstract class BaseBatchReader<T extends ScanTask> extends BaseReader<ColumnarBa |
| .project(requiredSchema) |
| .split(start, length) |
| .createBatchedReaderFunc( |
| - fileSchema -> { |
| - if (parquetConf.readerType() == ParquetReaderType.COMET) { |
| - return VectorizedSparkParquetReaders.buildCometReader( |
| - requiredSchema, fileSchema, idToConstant, deleteFilter); |
| - } else { |
| - return VectorizedSparkParquetReaders.buildReader( |
| - requiredSchema, fileSchema, idToConstant, deleteFilter); |
| - } |
| - }) |
| + fileSchema -> |
| + VectorizedSparkParquetReaders.buildReader( |
| + requiredSchema, fileSchema, idToConstant, deleteFilter)) |
| .recordsPerBatch(parquetConf.batchSize()) |
| .filter(residual) |
| .caseSensitive(caseSensitive()) |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java |
| index 404ba72846..5bd4e2b351 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/SparkDistributedDataScanTestBase.java |
| @@ -90,6 +90,14 @@ public abstract class SparkDistributedDataScanTestBase |
| .master("local[2]") |
| .config("spark.serializer", serializer) |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| } |
| } |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java |
| index 659507e4c5..74182efbb2 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanDeletes.java |
| @@ -73,6 +73,14 @@ public class TestSparkDistributedDataScanDeletes |
| .master("local[2]") |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| } |
| |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java |
| index a218f965ea..b1e0fcf2ca 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanFilterFiles.java |
| @@ -62,6 +62,14 @@ public class TestSparkDistributedDataScanFilterFiles |
| .master("local[2]") |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| } |
| |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java |
| index 2665d7ba8d..3539020102 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/TestSparkDistributedDataScanReporting.java |
| @@ -63,6 +63,14 @@ public class TestSparkDistributedDataScanReporting |
| .master("local[2]") |
| .config("spark.serializer", "org.apache.spark.serializer.KryoSerializer") |
| .config(SQLConf.SHUFFLE_PARTITIONS().key(), "4") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| } |
| |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java |
| index de68351f6e..785f7a3b58 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestBase.java |
| @@ -77,6 +77,14 @@ public abstract class TestBase extends SparkTestHelperBase { |
| .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) |
| .config("spark.sql.legacy.respectNullabilityInTextDatasetConversion", "true") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .enableHiveSupport() |
| .getOrCreate(); |
| |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java |
| index bc4e722bc8..c7a417e105 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java |
| @@ -59,7 +59,18 @@ public class TestParquetDictionaryEncodedVectorizedReads extends TestParquetVect |
| |
| @BeforeAll |
| public static void startSpark() { |
| - spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterAll |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java |
| index 3a269740b7..2035edf93a 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/ScanTestBase.java |
| @@ -54,7 +54,18 @@ public abstract class ScanTestBase extends AvroDataTest { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - ScanTestBase.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + ScanTestBase.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| ScanTestBase.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); |
| } |
| |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java |
| index f411920a5d..cf3b710d2d 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java |
| @@ -144,7 +144,18 @@ public class TestCompressionSettings extends CatalogTestBase { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - TestCompressionSettings.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestCompressionSettings.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @BeforeEach |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java |
| index c4ba96e634..5faeec110c 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java |
| @@ -75,7 +75,18 @@ public class TestDataSourceOptions extends TestBaseWithCatalog { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - TestDataSourceOptions.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestDataSourceOptions.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterAll |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java |
| index 348173596e..828fe716a5 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java |
| @@ -110,7 +110,18 @@ public class TestFilteredScan { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - TestFilteredScan.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestFilteredScan.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterAll |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java |
| index 84c99a575c..79dfc3aff3 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java |
| @@ -93,7 +93,18 @@ public class TestForwardCompatibility { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - TestForwardCompatibility.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestForwardCompatibility.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterAll |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java |
| index 7eff93d204..aead2a6924 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java |
| @@ -46,7 +46,18 @@ public class TestIcebergSpark { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - TestIcebergSpark.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestIcebergSpark.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterAll |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java |
| index 9464f687b0..4850759d61 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java |
| @@ -112,7 +112,18 @@ public class TestPartitionPruning { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - TestPartitionPruning.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestPartitionPruning.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); |
| |
| String optionKey = String.format("fs.%s.impl", CountOpenLocalFileSystem.scheme); |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java |
| index 5c218f21c4..dd4aeca9ef 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java |
| @@ -107,7 +107,18 @@ public class TestPartitionValues { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - TestPartitionValues.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestPartitionValues.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterAll |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java |
| index a7334a580c..f6f034240b 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java |
| @@ -87,7 +87,18 @@ public class TestSnapshotSelection { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - TestSnapshotSelection.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestSnapshotSelection.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterAll |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java |
| index 182b1ef8f5..47e6f0bd01 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java |
| @@ -120,7 +120,18 @@ public class TestSparkDataFile { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - TestSparkDataFile.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestSparkDataFile.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); |
| } |
| |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java |
| index fb2b312bed..25b6170fd3 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java |
| @@ -96,7 +96,18 @@ public class TestSparkDataWrite { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - TestSparkDataWrite.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestSparkDataWrite.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterEach |
| @@ -140,7 +151,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); |
| for (ManifestFile manifest : |
| @@ -210,7 +221,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); |
| } |
| @@ -256,7 +267,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); |
| } |
| @@ -309,7 +320,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); |
| } |
| @@ -352,7 +363,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); |
| } |
| @@ -392,7 +403,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); |
| |
| @@ -458,7 +469,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); |
| } |
| @@ -622,7 +633,7 @@ public class TestSparkDataWrite { |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| assertThat(actual).as("Number of rows should match").hasSameSizeAs(expected); |
| assertThat(actual).as("Result rows should match").isEqualTo(expected); |
| |
| @@ -708,7 +719,7 @@ public class TestSparkDataWrite { |
| // Since write and commit succeeded, the rows should be readable |
| Dataset<Row> result = spark.read().format("iceberg").load(targetLocation); |
| List<SimpleRecord> actual = |
| - result.orderBy("id").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| + result.orderBy("id", "data").as(Encoders.bean(SimpleRecord.class)).collectAsList(); |
| assertThat(actual).as("Number of rows should match").hasSize(records.size() + records2.size()); |
| assertThat(actual) |
| .describedAs("Result rows should match") |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java |
| index becf6a064d..f00afdf707 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java |
| @@ -83,7 +83,18 @@ public class TestSparkReadProjection extends TestReadProjection { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - TestSparkReadProjection.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestSparkReadProjection.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| ImmutableMap<String, String> config = |
| ImmutableMap.of( |
| "type", "hive", |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java |
| index 4f1cef5d37..e1b1716dcd 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java |
| @@ -136,6 +136,14 @@ public class TestSparkReaderDeletes extends DeleteReadTests { |
| .config("spark.ui.liveUpdate.period", 0) |
| .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .enableHiveSupport() |
| .getOrCreate(); |
| |
| @@ -204,7 +212,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { |
| } |
| |
| protected boolean countDeletes() { |
| - return true; |
| + // TODO: Enable once iceberg-rust exposes delete count metrics to Comet |
| + return false; |
| } |
| |
| @Override |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java |
| index baf7fa8f88..5ada29b4af 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderWithBloomFilter.java |
| @@ -182,6 +182,14 @@ public class TestSparkReaderWithBloomFilter { |
| SparkSession.builder() |
| .master("local[2]") |
| .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .enableHiveSupport() |
| .getOrCreate(); |
| |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java |
| index 17db46b85c..93dfa45e9c 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java |
| @@ -65,6 +65,14 @@ public class TestStructuredStreaming { |
| SparkSession.builder() |
| .master("local[2]") |
| .config("spark.sql.shuffle.partitions", 4) |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .getOrCreate(); |
| } |
| |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java |
| index 306444b9f2..e4c828110e 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java |
| @@ -75,7 +75,18 @@ public class TestTimestampWithoutZone extends TestBase { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - TestTimestampWithoutZone.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestTimestampWithoutZone.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| } |
| |
| @AfterAll |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java |
| index 841268a6be..ec29f30204 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java |
| @@ -80,7 +80,18 @@ public class TestWriteMetricsConfig { |
| |
| @BeforeAll |
| public static void startSpark() { |
| - TestWriteMetricsConfig.spark = SparkSession.builder().master("local[2]").getOrCreate(); |
| + TestWriteMetricsConfig.spark = |
| + SparkSession.builder() |
| + .master("local[2]") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| + .getOrCreate(); |
| TestWriteMetricsConfig.sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); |
| } |
| |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java |
| index 6e09252704..2ed54a2e60 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestAggregatePushDown.java |
| @@ -60,6 +60,14 @@ public class TestAggregatePushDown extends CatalogTestBase { |
| SparkSession.builder() |
| .master("local[2]") |
| .config("spark.sql.iceberg.aggregate_pushdown", "true") |
| + .config("spark.plugins", "org.apache.spark.CometPlugin") |
| + .config( |
| + "spark.shuffle.manager", |
| + "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager") |
| + .config("spark.comet.explainFallback.enabled", "true") |
| + .config("spark.comet.scan.icebergNative.enabled", "true") |
| + .config("spark.memory.offHeap.enabled", "true") |
| + .config("spark.memory.offHeap.size", "10g") |
| .enableHiveSupport() |
| .getOrCreate(); |
| |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java |
| index 9d2ce2b388..5e23368848 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestFilterPushDown.java |
| @@ -598,9 +598,7 @@ public class TestFilterPushDown extends TestBaseWithCatalog { |
| String planAsString = sparkPlan.toString().replaceAll("#(\\d+L?)", ""); |
| |
| if (sparkFilter != null) { |
| - assertThat(planAsString) |
| - .as("Post scan filter should match") |
| - .contains("Filter (" + sparkFilter + ")"); |
| + assertThat(planAsString).as("Post scan filter should match").contains("CometFilter"); |
| } else { |
| assertThat(planAsString).as("Should be no post scan filter").doesNotContain("Filter ("); |
| } |
| diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java |
| index 6719c45ca9..2515454401 100644 |
| --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java |
| +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestStoragePartitionedJoins.java |
| @@ -616,7 +616,7 @@ public class TestStoragePartitionedJoins extends TestBaseWithCatalog { |
| + "FROM %s t1 " |
| + "INNER JOIN %s t2 " |
| + "ON t1.id = t2.id AND t1.%s = t2.%s " |
| - + "ORDER BY t1.id, t1.%s", |
| + + "ORDER BY t1.id, t1.%s, t1.salary", |
| sourceColumnName, |
| tableName, |
| tableName(OTHER_TABLE_NAME), |
