| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| # |
| |
| version: '3.4' |
| |
| networks: |
| pulsar: |
| driver: bridge |
| |
| services: |
| |
| zk1: |
| container_name: zk1 |
| hostname: zk1 |
| image: apachepulsar/pulsar-all:latest |
| command: > |
| bash -c "bin/apply-config-from-env.py conf/zookeeper.conf && \ |
| bin/apply-config-from-env.py conf/pulsar_env.sh && \ |
| bin/generate-zookeeper-config.sh conf/zookeeper.conf && \ |
| exec bin/pulsar zookeeper" |
| environment: |
| ZOOKEEPER_SERVERS: zk1,zk2,zk3 |
| volumes: |
| - ./../../docker/pulsar/scripts/apply-config-from-env.py:/pulsar/bin/apply-config-from-env.py |
| networks: |
| pulsar: |
| |
| zk2: |
| container_name: zk2 |
| hostname: zk2 |
| image: apachepulsar/pulsar-all:latest |
| command: > |
| bash -c "bin/apply-config-from-env.py conf/zookeeper.conf && \ |
| bin/apply-config-from-env.py conf/pulsar_env.sh && \ |
| bin/generate-zookeeper-config.sh conf/zookeeper.conf && \ |
| exec bin/pulsar zookeeper" |
| environment: |
| ZOOKEEPER_SERVERS: zk1,zk2,zk3 |
| volumes: |
| - ./../../docker/pulsar/scripts/apply-config-from-env.py:/pulsar/bin/apply-config-from-env.py |
| networks: |
| pulsar: |
| |
| zk3: |
| container_name: zk3 |
| hostname: zk3 |
| image: apachepulsar/pulsar-all:latest |
| command: > |
| bash -c "bin/apply-config-from-env.py conf/zookeeper.conf && \ |
| bin/apply-config-from-env.py conf/pulsar_env.sh && \ |
| bin/generate-zookeeper-config.sh conf/zookeeper.conf && \ |
| exec bin/pulsar zookeeper" |
| environment: |
| ZOOKEEPER_SERVERS: zk1,zk2,zk3 |
| volumes: |
| - ./../../docker/pulsar/scripts/apply-config-from-env.py:/pulsar/bin/apply-config-from-env.py |
| networks: |
| pulsar: |
| |
| pulsar-init: |
| container_name: pulsar-init |
| hostname: pulsar-init |
| image: apachepulsar/pulsar-all:latest |
| command: bin/init-cluster.sh |
| environment: |
| clusterName: test |
| zkServers: zk1:2181 |
| configurationStore: zk1:2181 |
| pulsarNode: proxy1 |
| volumes: |
| - ./scripts/init-cluster.sh/:/pulsar/bin/init-cluster.sh |
| depends_on: |
| - zk1 |
| - zk2 |
| - zk3 |
| networks: |
| pulsar: |
| |
| bk1: |
| hostname: bk1 |
| container_name: bk1 |
| image: apachepulsar/pulsar-all:latest |
| command: > |
| bash -c "export dbStorage_writeCacheMaxSizeMb="$${dbStorage_writeCacheMaxSizeMb:-16}" && \ |
| export dbStorage_readAheadCacheMaxSizeMb="$${dbStorage_readAheadCacheMaxSizeMb:-16}" && \ |
| bin/apply-config-from-env.py conf/bookkeeper.conf && \ |
| bin/apply-config-from-env.py conf/pulsar_env.sh && \ |
| bin/watch-znode.py -z $$zkServers -p /initialized-$$clusterName -w && \ |
| exec bin/pulsar bookie" |
| environment: |
| clusterName: test |
| zkServers: zk1:2181,zk2:2181,zk3:2181 |
| numAddWorkerThreads: 8 |
| useHostNameAsBookieID: "true" |
| volumes: |
| - ./../../docker/pulsar/scripts/apply-config-from-env.py:/pulsar/bin/apply-config-from-env.py |
| depends_on: |
| - zk1 |
| - zk2 |
| - zk3 |
| - pulsar-init |
| networks: |
| pulsar: |
| |
| bk2: |
| hostname: bk2 |
| container_name: bk2 |
| image: apachepulsar/pulsar-all:latest |
| command: > |
| bash -c "export dbStorage_writeCacheMaxSizeMb="${dbStorage_writeCacheMaxSizeMb:-16}" && \ |
| export dbStorage_readAheadCacheMaxSizeMb="${dbStorage_readAheadCacheMaxSizeMb:-16}" && \ |
| bin/apply-config-from-env.py conf/bookkeeper.conf && \ |
| bin/apply-config-from-env.py conf/pulsar_env.sh && \ |
| bin/watch-znode.py -z $$zkServers -p /initialized-$$clusterName -w && \ |
| exec bin/pulsar bookie" |
| environment: |
| clusterName: test |
| zkServers: zk1:2181,zk2:2181,zk3:2181 |
| numAddWorkerThreads: 8 |
| useHostNameAsBookieID: "true" |
| volumes: |
| - ./../../docker/pulsar/scripts/apply-config-from-env.py:/pulsar/bin/apply-config-from-env.py |
| depends_on: |
| - zk1 |
| - zk2 |
| - zk3 |
| - pulsar-init |
| - bk1 |
| networks: |
| pulsar: |
| |
| bk3: |
| hostname: bk3 |
| container_name: bk3 |
| image: apachepulsar/pulsar-all:latest |
| command: > |
| bash -c "export dbStorage_writeCacheMaxSizeMb="${dbStorage_writeCacheMaxSizeMb:-16}" && \ |
| export dbStorage_readAheadCacheMaxSizeMb="${dbStorage_readAheadCacheMaxSizeMb:-16}" && \ |
| bin/apply-config-from-env.py conf/bookkeeper.conf && \ |
| bin/apply-config-from-env.py conf/pulsar_env.sh && \ |
| bin/watch-znode.py -z $$zkServers -p /initialized-$$clusterName -w && \ |
| exec bin/pulsar bookie" |
| environment: |
| clusterName: test |
| zkServers: zk1:2181,zk2:2181,zk3:2181 |
| numAddWorkerThreads: 8 |
| useHostNameAsBookieID: "true" |
| volumes: |
| - ./../../docker/pulsar/scripts/apply-config-from-env.py:/pulsar/bin/apply-config-from-env.py |
| depends_on: |
| - zk1 |
| - zk2 |
| - zk3 |
| - pulsar-init |
| - bk1 |
| - bk2 |
| networks: |
| pulsar: |
| |
| broker1: |
| hostname: broker1 |
| container_name: broker1 |
| image: apachepulsar/pulsar-all:latest |
| restart: on-failure |
| command: > |
| bash -c "bin/apply-config-from-env.py conf/broker.conf && \ |
| bin/apply-config-from-env.py conf/pulsar_env.sh && \ |
| bin/watch-znode.py -z $$zookeeperServers -p /initialized-$$clusterName -w && \ |
| exec bin/pulsar broker" |
| environment: |
| clusterName: test |
| zookeeperServers: zk1:2181,zk2:2181,zk3:2181 |
| configurationStore: zk1:2181,zk2:2181,zk3:2181 |
| webSocketServiceEnabled: "false" |
| functionsWorkerEnabled: "false" |
| volumes: |
| - ./../../docker/pulsar/scripts/apply-config-from-env.py:/pulsar/bin/apply-config-from-env.py |
| depends_on: |
| - zk1 |
| - zk2 |
| - zk3 |
| - pulsar-init |
| - bk1 |
| - bk2 |
| - bk3 |
| networks: |
| pulsar: |
| |
| broker2: |
| hostname: broker2 |
| container_name: broker2 |
| image: apachepulsar/pulsar-all:latest |
| restart: on-failure |
| command: > |
| bash -c "bin/apply-config-from-env.py conf/broker.conf && \ |
| bin/apply-config-from-env.py conf/pulsar_env.sh && \ |
| bin/watch-znode.py -z $$zookeeperServers -p /initialized-$$clusterName -w && \ |
| exec bin/pulsar broker" |
| environment: |
| clusterName: test |
| zookeeperServers: zk1:2181,zk2:2181,zk3:2181 |
| configurationStore: zk1:2181,zk2:2181,zk3:2181 |
| webSocketServiceEnabled: "false" |
| functionsWorkerEnabled: "false" |
| volumes: |
| - ./../../docker/pulsar/scripts/apply-config-from-env.py:/pulsar/bin/apply-config-from-env.py |
| depends_on: |
| - zk1 |
| - zk2 |
| - zk3 |
| - pulsar-init |
| - bk1 |
| - bk2 |
| - bk3 |
| - broker1 |
| networks: |
| pulsar: |
| |
| broker3: |
| hostname: broker3 |
| container_name: broker3 |
| image: apachepulsar/pulsar-all:latest |
| restart: on-failure |
| command: > |
| bash -c "bin/apply-config-from-env.py conf/broker.conf && \ |
| bin/apply-config-from-env.py conf/pulsar_env.sh && \ |
| bin/watch-znode.py -z $$zookeeperServers -p /initialized-$$clusterName -w && \ |
| exec bin/pulsar broker" |
| environment: |
| clusterName: test |
| zookeeperServers: zk1:2181,zk2:2181,zk3:2181 |
| configurationStore: zk1:2181,zk2:2181,zk3:2181 |
| webSocketServiceEnabled: "false" |
| functionsWorkerEnabled: "false" |
| volumes: |
| - ./../../docker/pulsar/scripts/apply-config-from-env.py:/pulsar/bin/apply-config-from-env.py |
| depends_on: |
| - zk1 |
| - zk2 |
| - zk3 |
| - pulsar-init |
| - bk1 |
| - bk2 |
| - bk3 |
| - broker1 |
| - broker2 |
| networks: |
| pulsar: |
| |
| proxy1: |
| hostname: proxy1 |
| container_name: proxy1 |
| restart: on-failure |
| image: apachepulsar/pulsar-all:latest |
| command: > |
| bash -c "bin/apply-config-from-env.py conf/proxy.conf && \ |
| bin/apply-config-from-env.py conf/pulsar_env.sh && \ |
| bin/watch-znode.py -z $$zookeeperServers -p /initialized-$$clusterName -w && \ |
| exec bin/pulsar proxy" |
| environment: |
| clusterName: test |
| zookeeperServers: zk1:2181,zk2:2181,zk3:2181 |
| configurationStoreServers: zk1:2181,zk2:2181,zk3:2181 |
| webSocketServiceEnabled: "true" |
| functionWorkerWebServiceURL: http://fnc1:6750 |
| volumes: |
| - ./../../docker/pulsar/scripts/apply-config-from-env.py:/pulsar/bin/apply-config-from-env.py |
| ports: |
| - "6650:6650" |
| - "8080:8080" |
| depends_on: |
| - zk1 |
| - zk2 |
| - zk3 |
| - pulsar-init |
| - bk1 |
| - bk2 |
| - bk3 |
| - broker1 |
| networks: |
| pulsar: |
| |
| websocket1: |
| hostname: websocket1 |
| container_name: websocket1 |
| restart: on-failure |
| image: apachepulsar/pulsar-all:latest |
| command: > |
| bash -c "bin/apply-config-from-env.py conf/websocket.conf && \ |
| bin/apply-config-from-env.py conf/pulsar_env.sh && \ |
| bin/watch-znode.py -z $$zookeeperServers -p /initialized-$$clusterName -w && \ |
| exec bin/pulsar websocket" |
| environment: |
| clusterName: test |
| zookeeperServers: zk1:2181,zk2:2181,zk3:2181 |
| configurationStoreServers: zk1:2181,zk2:2181,zk3:2181 |
| volumes: |
| - ./../../docker/pulsar/scripts/apply-config-from-env.py:/pulsar/bin/apply-config-from-env.py |
| depends_on: |
| - zk1 |
| - zk2 |
| - zk3 |
| - pulsar-init |
| - bk1 |
| - bk2 |
| - bk3 |
| - broker1 |
| - proxy1 |
| networks: |
| pulsar: |
| |
| fnc1: |
| hostname: fnc1 |
| container_name: fnc1 |
| image: apachepulsar/pulsar-all:latest |
| restart: on-failure |
| command: > |
| bash -c "bin/apply-config-from-env.py conf/client.conf && \ |
| bin/gen-yml-from-env.py conf/functions_worker.yml && \ |
| bin/apply-config-from-env.py conf/pulsar_env.sh && \ |
| bin/watch-znode.py -z $$zookeeperServers -p /initialized-$$clusterName -w && \ |
| exec bin/pulsar functions-worker" |
| environment: |
| clusterName: test |
| zookeeperServers: zk1:2181,zk2:2181,zk3:2181 |
| # Requires PF_ prefix for some reason in the code |
| PF_pulsarFunctionsCluster: test |
| PF_workerId: fnc1 |
| PF_pulsarFunctionsCluster: test |
| # This setting does not appear to accept more than one host |
| PF_configurationStoreServers: zk1:2181 |
| PF_pulsarServiceUrl: pulsar://proxy1:6650 |
| PF_pulsarWebServiceUrl: http://proxy1:8080 |
| volumes: |
| - ./../../docker/pulsar/scripts/apply-config-from-env.py:/pulsar/bin/apply-config-from-env.py |
| depends_on: |
| - zk1 |
| - zk2 |
| - zk3 |
| - pulsar-init |
| - bk1 |
| - bk2 |
| - bk3 |
| - broker1 |
| - proxy1 |
| networks: |
| pulsar: |
| |
| sql1: |
| hostname: sql1 |
| container_name: sql1 |
| image: apachepulsar/pulsar-all:latest |
| restart: on-failure |
| command: > |
| bash -c "bin/apply-config-from-env-with-prefix.py SQL_PREFIX_ conf/presto/catalog/pulsar.properties && \ |
| bin/apply-config-from-env.py conf/pulsar_env.sh && \ |
| bin/watch-znode.py -z $$zookeeperServers -p /initialized-$$clusterName -w && \ |
| exec bin/pulsar sql-worker run" |
| environment: |
| clusterName: test |
| zookeeperServers: zk1:2181,zk2:2181,zk3:2181 |
| configurationStoreServers: zk1:2181,zk2:2181,zk3:2181 |
| pulsar.zookeeper-uri: zk1:2181,zk2:2181,zk3:2181 |
| coordinator: "true" |
| volumes: |
| - ./../../docker/pulsar/scripts/apply-config-from-env-with-prefix.py:/pulsar/bin/apply-config-from-env-with-prefix.py |
| - ./../../docker/pulsar/scripts/apply-config-from-env.py:/pulsar/bin/apply-config-from-env.py |
| depends_on: |
| - zk1 |
| - zk2 |
| - zk3 |
| - pulsar-init |
| - bk1 |
| - bk2 |
| - bk3 |
| - broker1 |
| - proxy1 |
| ports: |
| - "8081:8081" |
| networks: |
| pulsar: |
| |
| manager: |
| hostname: manager |
| container_name: manager |
| image: apachepulsar/pulsar-manager:v0.1.0 |
| ports: |
| - "9527:9527" |
| - "7750:7750" |
| depends_on: |
| - broker1 |
| volumes: |
| - "./data/:/data" |
| environment: |
| REDIRECT_HOST: "http://127.0.0.1" |
| REDIRECT_PORT: "9527" |
| DRIVER_CLASS_NAME: "org.postgresql.Driver" |
| URL: "jdbc:postgresql://127.0.0.1:5432/pulsar_manager" |
| USERNAME: "pulsar" |
| PASSWORD: "pulsar" |
| LOG_LEVEL: "DEBUG" |
| networks: |
| pulsar: |