dev/docker/livy-dev-cluster/docker-compose.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

version: '2'

services:
  spark-master:
    image: livy-dev-spark:latest
    command: bin/spark-class org.apache.spark.deploy.master.Master -h master
    hostname: master
    container_name: spark-master
    environment:
      MASTER: spark://master:7077
      SPARK_CONF_DIR: /conf
      SPARK_PUBLIC_DNS: localhost
    expose:
      - 7001
      - 7002
      - 7003
      - 7004
      - 7005
      - 7077
      - 6066
      - 4040
    ports:
      - 4040:4040
      - 6066:6066
      - 7077:7077
      - 8080:8080
    volumes:
      - ./conf/master:/conf
      - ./data:/tmp/data
      - ./logs/master:/logs

  spark-worker-1:
    image: livy-dev-spark:latest
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077
    container_name: spark-worker-1
    environment:
      SPARK_CONF_DIR: /conf
      SPARK_WORKER_CORES: 1
      SPARK_WORKER_MEMORY: 1g
      SPARK_WORKER_PORT: 8881
      SPARK_WORKER_WEBUI_PORT: 8081
      SPARK_PUBLIC_DNS: localhost
    expose:
      - 7012
      - 7013
      - 7014
      - 7015
      - 8881
    ports:
      - 8081:8081
    volumes:
      - ./conf/worker:/conf
      - ./data:/tmp/data
      - ./logs/worker:/logs
    depends_on:
      - "spark-master"

  livy:
    image: livy-dev-server:latest
    command: bin/livy-server
    container_name: livy
    environment:
      SPARK_CONF_DIR: /conf
      SPARK_DIVER_CORES: 1
      SPARK_DRIVER_MEMORY: 1g
      SPARK_MASTER_ENDPOINT: master
      SPARK_MASTER_PORT: 7077
      LIVY_CONF_DIR: /conf
      LIVY_LOG_DIR: /logs
      LIVY_FILE_LOCAL_DIR_WHITELIST: /opt/jars
    expose:
      # remote debug port
      - 9010
    ports:
      - 8998:8998
      - 9010:9010
    volumes:
      - ./conf/livy:/conf
      - ./logs/livy:/logs
    depends_on:
      - "spark-master"
      - "spark-worker-1"