conf/cluster-master/application.conf

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Cluster configuration properties
gobblin.cluster.app.name=GobblinStandaloneCluster
gobblin.cluster.email.notification.on.shutdown=false
gobblin.cluster.helix.instance.max.retries=2
gobblin.cluster.work.dir=/tmp/gobblin-cluster

# Helix/Zookeeper configuration properties
gobblin.cluster.helix.cluster.name=GobblinStandaloneCluster
gobblin.cluster.zk.connection.string="localhost:2181"

jobconf.monitor.interval=30000

# Sample configuration properties for the Gobblin Standalone cluster
gobblin.cluster.workDir=${gobblin.cluster.work.dir}/GobblinStandaloneCluster

# default is the JobConfigurationManager
# use this manager to accept jobs from Kafka. It requires some additional Kafka related parameters.
#gobblin.cluster.job.configuration.manager=org.apache.gobblin.cluster.StreamingJobConfigurationManager
#spec.kafka.topics=ruyang_test_kafka_gobblin
#kafka.brokers="hostname:12913/kafka-queuing"
#jobSpecMonitor.kafka.zookeeper.connect="hostname:12913/kafka-queuing"

# Cluster configuration properties
gobblin.cluster.helix.cluster.name=GobblinStandaloneClusterCli

# used by the JobConfigurationManager
gobblin.cluster.job.conf.path=${gobblin.cluster.work.dir}/jobs

gobblin.cluster.jobconf.fullyQualifiedPath=${gobblin.cluster.work.dir}/jobs
gobblin.cluster.job.catalog=org.apache.gobblin.runtime.job_catalog.FSJobCatalog

# File system URIs
fs.uri="file:///"
writer.fs.uri=${fs.uri}
state.store.fs.uri=${fs.uri}

# Writer related configuration properties
writer.destination.type=HDFS
writer.output.format=AVRO
writer.staging.dir=${gobblin.cluster.work.dir}/task-staging
writer.output.dir=${gobblin.cluster.work.dir}/task-output

# Data publisher related configuration properties
data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher
data.publisher.final.dir=${gobblin.cluster.work.dir}/job-output
data.publisher.replace.final.dir=false

# Directory where job/task state files are stored
state.store.dir=${gobblin.cluster.work.dir}/state-store

# Directory where error files from the quality checkers are stored
qualitychecker.row.err.file=${gobblin.cluster.work.dir}/err

# Disable job locking for now
job.lock.enabled=false

# Directory where job locks are stored
job.lock.dir=${gobblin.cluster.work.dir}/locks

# Directory where metrics log files are stored
metrics.log.dir=${gobblin.cluster.work.dir}/metrics

# Interval of task state reporting in milliseconds
task.status.reportintervalinms=1000

# Enable metrics / events
metrics.enabled=true

# UI
admin.server.enabled=true
admin.server.port=9000

#  is this required/redundent ?
rest.server.host=localhost
rest.server.port=9090

# job history store
job.execinfo.server.enabled=false
job.history.store.enabled=false