| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| configfiles: |
| - file: client |
| configs: |
| - name: heron.package.core.uri |
| description: Location of the core Heron package |
| default: file:///vagrant/.herondata/dist/heron-core-release.tar.gz |
| - name: heron.config.is.role.required |
| description: Whether a role is required to submit a topology |
| default: "False" |
| - name: heron.config.is.env.required |
| description: Whether an environment is required to submit a topology |
| default: "False" |
| - file: heron_internals |
| configs: |
| - name: heron.logging.directory |
| description: The relative path to the logging directory |
| default: log-files |
| - name: heron.logging.maximum.size.mb |
| description: The maximum log file size (in MB) |
| default: 100 |
| - name: heron.logging.maximum.files |
| description: The maximum number of log files |
| default: 5 |
| - name: heron.check.tmaster.location.interval.sec |
| description: The interval, in seconds, after which to check if the topology master location has been fetched or not |
| default: 120 |
| - name: heron.logging.prune.interval.sec |
| description: The interval, in seconds, at which to prune C++ log files |
| default: 300 |
| - name: heron.logging.flush.interval.sec |
| description: The interval, in seconds, at which to flush C++ log files |
| default: 10 |
| - name: heron.logging.err.threshold |
| description: The threshold level at which to log errors |
| default: 3 |
| - name: heron.metrics.export.interval.sec |
| description: The interval, in seconds, at which different components export metrics to the metrics manager |
| default: 60 |
| - name: heron.metrics.max.exceptions.per.message.count |
| description: The maximum count of exceptions in one `MetricPublisherPublishMessage` protobuf message |
| default: 1024 |
| - name: heron.streammgr.cache.drain.frequency.ms |
| description: The frequency, in milliseconds, at which to drain the tuple cache in the stream manager |
| default: 10 |
| - name: heron.streammgr.stateful.buffer.size.mb |
| description: The sized-based threshold (in MB) for buffering data tuples waiting for checkpoint markers before giving up |
| default: 100 |
| - name: heron.streammgr.cache.drain.size.mb |
| description: The sized-based threshold (in MB) for draining the tuple cache |
| default: 100 |
| - name: heron.streammgr.xormgr.rotatingmap.nbuckets |
| description: For efficient acknowledgements |
| default: 3 |
| - name: heron.streammgr.mempool.max.message.number |
| description: The max number of messages in the memory pool for each message type |
| default: 512 |
| - name: heron.streammgr.client.reconnect.interval.sec |
| description: The reconnect interval to other stream managers (in seconds) for the stream manager client |
| default: 1 |
| - name: heron.streammgr.client.reconnect.tmaster.interval.sec |
| description: The reconnect interval to the topology master (in seconds) for the stream manager client |
| default: 10 |
| - name: heron.streammgr.client.reconnect.tmaster.max.attempts |
| description: The max reconnect attempts to tmaster for stream manager client |
| default: 30 |
| - name: heron.streammgr.network.options.maximum.packet.mb |
| description: The maximum packet size (in MB) of the stream manager's network options |
| default: 10 |
| - name: heron.streammgr.tmaster.heartbeat.interval.sec |
| description: The interval (in seconds) at which to send heartbeats |
| default: 10 |
| - name: heron.streammgr.connection.read.batch.size.mb |
| description: The maximum batch size (in MB) for the stream manager to read from socket |
| default: 1 |
| - name: heron.streammgr.connection.write.batch.size.mb |
| description: Maximum batch size (in MB) for the stream manager to write to socket |
| default: 1 |
| - name: heron.streammgr.network.backpressure.threshold |
| description: The number of times Heron should wait to see a buffer full while enqueueing data before declaring the start of backpressure |
| default: 3 |
| - name: heron.streammgr.network.backpressure.highwatermark.mb |
| description: The high-water mark on the number (in MB) that can be left outstanding on a connection |
| default: 100 |
| - name: heron.streammgr.network.backpressure.lowwatermark.mb |
| description: The low-water mark on the number (in MB) that can be left outstanding on a connection |
| default: |
| - name: heron.tmaster.metrics.collector.maximum.interval.min |
| description: The maximum interval (in minutes) for metrics to be kept in the topology master |
| default: 180 |
| - name: heron.tmaster.establish.retry.times |
| description: The maximum number of times to retry establishing connection with the topology master |
| default: 30 |
| - name: heron.tmaster.establish.retry.interval.sec |
| description: The interval at which to retry establishing connection with the topology master |
| default: 1 |
| - name: heron.tmaster.network.master.options.maximum.packet.mb |
| description: Maximum packet size (in MB) of topology master's network options to connect to stream managers |
| default: 16 |
| - name: heron.tmaster.network.controller.options.maximum.packet.mb |
| description: Maximum packet size (in MB) of the topology master's network options to connect to scheduler |
| default: 1 |
| - name: heron.tmaster.network.stats.options.maximum.packet.mb |
| description: Maximum packet size (in MB) of the topology master's network options for stat queries |
| default: 1 |
| - name: heron.tmaster.metrics.collector.purge.interval.sec |
| description: The interval (in seconds) at which the topology master purges metrics from socket |
| default: 60 |
| - name: heron.tmaster.metrics.collector.maximum.exception |
| description: The maximum number of exceptions to be stored in the topology metrics collector, to prevent out-of-memory errors |
| default: 256 |
| - name: heron.tmaster.metrics.network.bindallinterfaces |
| description: Whether the metrics reporter should bind on all interfaces |
| default: "False" |
| - name: heron.tmaster.stmgr.state.timeout.sec |
| description: The timeout (in seconds) for the stream manager, compared with (current time - last heartbeat time) |
| default: 60 |
| - name: heron.metricsmgr.network.read.batch.time.ms |
| description: The maximum batch time (in milliseconds) for the metrics manager to read from socket |
| default: 16 |
| - name: heron.metricsmgr.network.read.batch.size.bytes |
| description: The maximum batch size (in bytes) to read from socket |
| default: 32768 |
| - name: heron.metricsmgr.network.write.batch.time.ms |
| description: The maximum batch time (in milliseconds) for the metrics manager to write to socket |
| default: 32768 |
| - name: heron.metricsmgr.network.options.socket.send.buffer.size.bytes |
| description: The maximum socket send buffer size (in bytes) |
| default: 6553600 |
| - name: heron.metricsmgr.network.options.socket.received.buffer.size.bytes |
| description: The maximum socket received buffer size (in bytes) for the metrics manager's network options |
| default: 8738000 |
| - name: heron.metricsmgr.network.options.maximum.packetsize.bytes |
| description: The maximum packet size that the metrics manager can read |
| default: 1048576 |
| - name: heron.instance.network.options.maximum.packetsize.bytes |
| description: The maximum size of packets that Heron instances can read |
| default: 10485760 |
| - name: heron.instance.internal.bolt.read.queue.capacity |
| description: The queue capacity (num of items) in bolt for buffer packets to read from stream manager |
| default: 128 |
| - name: heron.instance.internal.bolt.write.queue.capacity |
| description: The queue capacity (num of items) in bolt for buffer packets to write to stream manager |
| default: 128 |
| - name: heron.instance.internal.spout.read.queue.capacity |
| description: The queue capacity (num of items) in spout for buffer packets to read from stream manager |
| default: 1024 |
| - name: heron.instance.internal.spout.write.queue.capacity |
| description: The queue capacity (num of items) in spout for buffer packets to write to stream manager |
| default: 128 |
| - name: heron.instance.internal.metrics.write.queue.capacity |
| description: The queue capacity (num of items) for metrics packets to write to metrics manager |
| default: 128 |
| - name: heron.instance.network.read.batch.time.ms |
| description: Time based, the maximum batch time in ms for instance to read from stream manager per attempt |
| default: 16 |
| - name: heron.instance.network.read.batch.size.bytes |
| description: Size based, the maximum batch size in bytes to read from stream manager |
| default: 32768 |
| - name: heron.instance.network.write.batch.time.ms |
| description: Time based, the maximum batch time (in milliseconds) for the instance to write to the stream manager per attempt |
| default: 16 |
| - name: heron.instance.network.write.batch.size.bytes |
| description: Size based, the maximum batch size in bytes to write to stream manager |
| default: 32768 |
| - name: heron.instance.network.options.socket.send.buffer.size.bytes |
| description: The maximum socket's send buffer size in bytes |
| default: 6553600 |
| - name: heron.instance.network.options.socket.received.buffer.size.bytes |
| description: The maximum socket's received buffer size in bytes of instance's network options |
| default: 8738000 |
| - name: heron.instance.set.data.tuple.capacity |
| description: The maximum number of data tuple to batch in a HeronDataTupleSet protobuf |
| default: 1024 |
| - name: heron.instance.set.data.tuple.size.bytes |
| description: The maximum size in bytes of data tuple to batch in a HeronDataTupleSet protobuf |
| default: 8388608 |
| - name: heron.instance.set.control.tuple.capacity |
| description: The maximum number of control tuple to batch in a HeronControlTupleSet protobuf |
| default: 1024 |
| - name: heron.instance.ack.batch.time.ms |
| description: The maximum time in ms for a spout to do acknowledgement per attempt, the ack batch could also break if there are no more ack tuples to process |
| default: 128 |
| - name: heron.instance.emit.batch.time.ms |
| description: The maximum time in ms for an spout instance to emit tuples per attempt |
| default: 16 |
| - name: heron.instance.emit.batch.size.bytes |
| description: The maximum batch size in bytes for an spout to emit tuples per attempt |
| default: 32768 |
| - name: heron.instance.execute.batch.time.ms |
| description: The maximum time in ms for an bolt instance to execute tuples per attempt |
| default: 16 |
| - name: heron.instance.execute.batch.size.bytes |
| description: The maximum batch size in bytes for an bolt instance to execute tuples per attempt |
| default: 32768 |
| - name: heron.instance.state.check.interval.sec |
| description: The time interval for an instance to check the state change, for example, the interval a spout uses to check whether activate/deactivate is invoked |
| default: 5 |
| - name: heron.instance.force.exit.timeout.ms |
| description: The time to wait before the instance exits forcibly when uncaught exception happens |
| default: 2000 |
| - name: heron.instance.reconnect.streammgr.interval.sec |
| description: Interval in seconds to reconnect to the stream manager, including the request timeout in connecting |
| default: 5 |
| - name: heron.instance.reconnect.streammgr.interval.sec |
| description: Interval in seconds to reconnect to the stream manager, including the request timeout in connecting |
| default: 60 |
| - name: heron.instance.reconnect.metricsmgr.interval.sec |
| description: Interval in seconds to reconnect to the metrics manager, including the request timeout in connecting |
| default: 5 |
| - name: heron.instance.reconnect.metricsmgr.times |
| description: Interval in seconds to reconnect to the metrics manager, including the request timeout in connecting |
| default: 60 |
| - name: heron.instance.metrics.system.sample.interval.sec |
| description: The interval in second for an instance to sample its system metrics, for instance, CPU load. |
| default: 10 |
| - name: heron.instance.slave.fetch.pplan.interval.sec |
| description: The time interval (in seconds) at which Heron instances fetch the physical plan from slaves |
| default: 1 |
| - name: heron.instance.acknowledgement.nbuckets |
| description: For efficient acknowledgement |
| default: 10 |
| - name: heron.instance.tuning.expected.bolt.read.queue.size |
| description: The expected size on read queue in bolt |
| default: 8 |
| - name: heron.instance.tuning.expected.bolt.write.queue.size |
| description: The expected size on write queue in bolt |
| default: 8 |
| - name: heron.instance.tuning.expected.spout.read.queue.size |
| description: The expected size on read queue in spout |
| default: 512 |
| - name: heron.instance.tuning.expected.spout.write.queue.size |
| description: The exepected size on write queue in spout |
| default: 8 |
| - name: heron.instance.tuning.expected.metrics.write.queue.size |
| description: The expected size on metrics write queue |
| default: 8 |
| - name: heron.instance.tuning.current.sample.weight |
| description: |
| default: 0.8 |
| - name: heron.instance.tuning.interval.ms |
| description: Interval in ms to tune the size of in & out data queue in instance |
| default: 100 |
| - file: packing |
| configs: |
| - name: heron.class.packing.algorithm |
| description: Packing algorithm for packing instances into containers |
| default: org.apache.heron.packing.roundrobin.RoundRobinPacking |
| - file: scheduler |
| configs: |
| - name: heron.class.scheduler |
| description: scheduler class for distributing the topology for execution |
| default: org.apache.heron.scheduler.kubernetes.KubernetesScheduler |
| - name: heron.class.launcher |
| description: launcher class for submitting and launching the topology |
| default: org.apache.heron.scheduler.kubernetes.KubernetesLauncher |
| - name: heron.directory.sandbox.java.home |
| description: location of java - pick it up from shell environment |
| default: $JAVA_HOME |
| - name: heron.kubernetes.scheduler.uri |
| description: The URI of the Kubernetes API |
| - name: heron.scheduler.is.service |
| description: Invoke the IScheduler as a library directly |
| default: False |
| - name: heron.executor.docker.image |
| description: docker repo for executor |
| default: heron/heron:latest |
| - file: stateful |
| configs: |
| - name: heron.statefulstorage.classname |
| description: The type of storage to be used for state checkpointing |
| default: org.apache.heron.statefulstorage.localfs.LocalFileSystemStorage |
| - file: statemgr |
| configs: |
| - name: heron.class.state.manager |
| description: local state manager class for managing state in a persistent fashion |
| default: org.apache.heron.statemgr.zookeeper.curator.CuratorStateManager |
| - name: heron.statemgr.connection.string |
| description: local state manager connection string |
| - name: heron.statemgr.root.path |
| description: path of the root address to store the state in a local file system |
| default: /heron |
| - name: heron.statemgr.zookeeper.is.initialize.tree |
| description: create the zookeeper nodes, if they do not exist |
| default: "True" |
| - name: heron.statemgr.zookeeper.session.timeout.ms |
| description: timeout in ms to wait before considering zookeeper session is dead |
| default: 30000 |
| - name: heron.statemgr.zookeeper.connection.timeout.ms |
| description: timeout in ms to wait before considering zookeeper connection is dead |
| default: 30000 |
| - name: heron.statemgr.zookeeper.retry.count |
| description: timeout in ms to wait before considering zookeeper connection is dead |
| default: 10 |
| - name: heron.statemgr.zookeeper.retry.interval.ms |
| description: duration of time to wait until the next retry |
| default: 10000 |
| - file: uploader |
| configs: |
| - name: heron.class.uploader |
| description: uploader class for transferring the topology files (jars, tars, PEXes, etc.) to storage |
| default: org.apache.heron.uploader.s3.S3Uploader |
| - name: heron.uploader.s3.bucket |
| description: S3 bucket in which topology assets will be stored (if AWS S3 is being used) |
| - name: heron.uploader.s3.access_key |
| description: AWS access key (if AWS S3 is being used) |
| - name: heron.uploader.s3.secret_key |
| description: AWS secret access key (if AWS S3 is being used) |
