dev/docker-compose-integration.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

services:
  spark-iceberg:
    image: pyiceberg-spark:latest
    build:
      context: spark/
      args:
        MAVEN_MIRROR: ${MAVEN_MIRROR:-https://repo.maven.apache.org/maven2}
    container_name: pyiceberg-spark
    networks:
      iceberg_net:
    depends_on:
      - rest
      - hive
      - object-store
    ports:
      - 15002:15002 # Spark Connect
      - 4040:4040 # Spark UI
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
    links:
      - rest:rest
      - hive:hive
      - object-store:object-store
    healthcheck:
      test: ["CMD", "sh", "-c", "netstat -an | grep 15002 | grep LISTEN"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 90s
  rest:
    image: apache/iceberg-rest-fixture
    container_name: pyiceberg-rest
    networks:
      iceberg_net:
    ports:
      - 8181:8181
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
      - CATALOG_WAREHOUSE=s3://warehouse/
      - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
      - CATALOG_S3_ENDPOINT=http://object-store:9000
      - CATALOG_JDBC_STRICT__MODE=true
  object-store:
    image: rustfs/rustfs:1.0.0-rc.5
    container_name: pyiceberg-object-store
    networks:
      iceberg_net:
        aliases:
          - warehouse.object-store
    ports:
      - 9001:9001
      - 9000:9000
    environment:
      - RUSTFS_ACCESS_KEY=admin
      - RUSTFS_SECRET_KEY=password
      - RUSTFS_VOLUMES=/data
      - RUSTFS_ADDRESS=:9000
      - RUSTFS_CONSOLE_ENABLE=true
      - RUSTFS_CONSOLE_ADDRESS=:9001
      # Resolves virtual-hosted-style `warehouse.object-store` to the warehouse
      # bucket. The port is required until rustfs/rustfs#7051 ships.
      - RUSTFS_SERVER_DOMAINS=object-store:9000
    healthcheck:
      test: ["CMD-SHELL", "curl --fail http://127.0.0.1:9000/health"]
      interval: 5s
      timeout: 5s
      retries: 10
      start_period: 5s
  aws-cli:
    image: amazon/aws-cli:2.36.44
    container_name: pyiceberg-aws-cli
    networks:
      iceberg_net:
    depends_on:
      object-store:
        condition: service_healthy
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
      - AWS_ENDPOINT_URL=http://object-store:9000
    entrypoint: /bin/sh
    command:
      - -c
      - "aws s3 rb s3://warehouse --force 2>/dev/null; aws s3 mb s3://warehouse"
  hive:
    image: pyiceberg-hive:latest
    build:
      context: hive/
      args:
        MAVEN_MIRROR: ${MAVEN_MIRROR:-https://repo1.maven.org/maven2}
    container_name: pyiceberg-hive
    hostname: hive
    networks:
      iceberg_net:
    ports:
      - 9083:9083
    environment:
      SERVICE_NAME: "metastore"
      SERVICE_OPTS: "-Dmetastore.warehouse.dir=s3a://warehouse/hive/"

networks:
  iceberg_net: