| # | |
| # Licensed to the Apache Software Foundation (ASF) under one or more | |
| # contributor license agreements. See the NOTICE file distributed with | |
| # this work for additional information regarding copyright ownership. | |
| # The ASF licenses this file to You under the Apache License, Version 2.0 | |
| # (the "License"); you may not use this file except in compliance with | |
| # the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # | |
| spark.sql.extensions org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions | |
| # Configure Iceberg REST catalog | |
| spark.sql.catalog.rest org.apache.iceberg.spark.SparkCatalog | |
| spark.sql.catalog.rest.type rest | |
| spark.sql.catalog.rest.uri http://rest:8181 | |
| spark.sql.catalog.rest.io-impl org.apache.iceberg.aws.s3.S3FileIO | |
| spark.sql.catalog.rest.warehouse s3://warehouse/rest/ | |
| spark.sql.catalog.rest.s3.endpoint http://object-store:9000 | |
| spark.sql.catalog.rest.cache-enabled false | |
| # Configure Iceberg Hive catalog | |
| spark.sql.catalog.hive org.apache.iceberg.spark.SparkCatalog | |
| spark.sql.catalog.hive.type hive | |
| spark.sql.catalog.hive.uri thrift://hive:9083 | |
| spark.sql.catalog.hive.io-impl org.apache.iceberg.aws.s3.S3FileIO | |
| spark.sql.catalog.hive.warehouse s3://warehouse/hive/ | |
| spark.sql.catalog.hive.s3.endpoint http://object-store:9000 | |
| # Configure Spark's default session catalog (spark_catalog) to use Iceberg backed by the Hive Metastore | |
| spark.sql.catalog.spark_catalog org.apache.iceberg.spark.SparkSessionCatalog | |
| spark.sql.catalog.spark_catalog.type hive | |
| spark.sql.catalog.spark_catalog.uri thrift://hive:9083 | |
| spark.hadoop.fs.s3a.endpoint http://object-store:9000 | |
| spark.sql.catalogImplementation hive | |
| spark.sql.warehouse.dir s3a://warehouse/hive/ | |
| # Spark's own Hive client speaks the Hive 2.3 API, so keep it off the Hive 4 client on the class path | |
| spark.sql.hive.metastore.version 2.3.10 | |
| spark.sql.hive.metastore.jars path | |
| spark.sql.hive.metastore.jars.path file:///opt/spark/hive-metastore-jars/*,file:///opt/spark/jars/* | |
| spark.sql.defaultCatalog rest | |
| # Configure Spark UI and event logging | |
| spark.ui.enabled true | |
| spark.eventLog.enabled true | |
| spark.eventLog.dir /home/iceberg/spark-events | |
| spark.sql.ansi.enabled false |