| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| ########### These all have default values as shown |
| ########### All the config associated with time is in the unit of milli-seconds, unless otherwise specified. |
| ########### All the config associated with data size is in the unit of bytes, unless otherwise specified. |
| |
| ### system level configs for UNIT TESTS (mainly for speed up things) |
| |
| ### heron.* configs are general configurations over all componenets |
| |
| # The relative path to the logging directory |
| heron.logging.directory: "log-files" |
| |
| # The maximum log file size in MB |
| heron.logging.maximum.size.mb: 100 |
| |
| # The maximum number of log files |
| heron.logging.maximum.files: 5 |
| |
| # The interval in seconds after which to check if the tmaster location has been fetched or not |
| heron.check.tmaster.location.interval.sec: 120 |
| |
| # The interval in seconds to prune logging files in C++ |
| heron.logging.prune.interval.sec: 300 |
| |
| # The interval in seconds to flush log files in C++ |
| heron.logging.flush.interval.sec: 10 |
| |
| # The threadhold level to log error |
| heron.logging.err.threshold: 3 |
| |
| # The interval in seconds for different components to export metrics to metrics manager |
| heron.metrics.export.interval.sec: 60 |
| |
| # The maximum count of exceptions in one MetricPublisherPublishMessage protobuf |
| heron.metrics.max.exceptions.per.message.count: 1024 |
| |
| ### heron.streammgr.* configs are for the stream manager |
| |
| # The tuple cache (used for batching) can be drained in two ways: (a) Time based (b) size based |
| |
| # The frequency in ms to drain the tuple cache in stream manager |
| heron.streammgr.cache.drain.frequency.ms: 10 |
| |
| # The sized based threshold in MB for buffering data tuples |
| # waiting for checkpoint markers before giving up |
| heron.streammgr.stateful.buffer.size.mb: 100 |
| |
| # The sized based threshold in MB for draining the tuple cache |
| heron.streammgr.cache.drain.size.mb: 100 |
| |
| # The max number of messages in the memory pool for each message type |
| heron.streammgr.mempool.max.message.number: 512 |
| |
| # For efficient acknowledgement |
| heron.streammgr.xormgr.rotatingmap.nbuckets: 3 |
| |
| # The reconnect interval to other stream managers in second for stream manager client |
| heron.streammgr.client.reconnect.interval.sec: 1 |
| |
| # The reconnect interval to tamster in second for stream manager client |
| heron.streammgr.client.reconnect.tmaster.interval.sec: 1 |
| |
| # The max reconnect attempts to tmaster for stream manager client |
| heron.streammgr.client.reconnect.tmaster.max.attempts: 30 |
| |
| # The maximum packet size in MB of stream manager's network options |
| heron.streammgr.network.options.maximum.packet.mb: 10 |
| |
| # The interval in seconds to send heartbeat |
| heron.streammgr.tmaster.heartbeat.interval.sec: 10 |
| |
| # Maximum batch size in MB to read by stream manager from socket |
| heron.streammgr.connection.read.batch.size.mb: 1 |
| |
| # Maximum batch size in MB to write by stream manager to socket |
| heron.streammgr.connection.write.batch.size.mb: 1 |
| |
| # Number of times we should wait to see a buffer full while enqueueing data before declaring start of back pressure |
| heron.streammgr.network.backpressure.threshold: 3 |
| |
| # High water mark on the num in MB that can be left outstanding on a connection |
| heron.streammgr.network.backpressure.highwatermark.mb: 50 |
| |
| # Low water mark on the num in MB that can be left outstanding on a connection |
| heron.streammgr.network.backpressure.lowwatermark.mb: 30 |
| |
| |
| ### heron.tmaster.* configs are for the tmaster |
| |
| # The maximum interval in minutes of metrics to be kept in tmaster |
| heron.tmaster.metrics.collector.maximum.interval.min: 180 |
| |
| # The maximum time to retry to establish the tmaster |
| heron.tmaster.establish.retry.times: 30 |
| |
| # The interval to retry to establish the tmaster |
| heron.tmaster.establish.retry.interval.sec: 1 |
| |
| # The maximum packet size in MB of tmaster's network options for stmgrs to connect to |
| heron.tmaster.network.master.options.maximum.packet.mb: 16 |
| |
| # The maximum packet size in MB of tmaster's network options for scheduler to connect to |
| heron.tmaster.network.controller.options.maximum.packet.mb: 1 |
| |
| # The maximum packet size in MB of tmaster's network options for stat queries |
| heron.tmaster.network.stats.options.maximum.packet.mb: 1 |
| |
| # The inteval for tmaster to purge metrics from socket |
| heron.tmaster.metrics.collector.purge.interval.sec: 60 |
| |
| # The maximum # of exception to be stored in tmetrics collector, to prevent potential OOM |
| heron.tmaster.metrics.collector.maximum.exception: 256 |
| |
| # Should the metrics reporter bind on all interfaces |
| heron.tmaster.metrics.network.bindallinterfaces: False |
| |
| # The timeout in seconds for stream mgr, compared with (current time - last heartbeat time) |
| heron.tmaster.stmgr.state.timeout.sec: 60 |
| |
| |
| ### heron.metricsmgr.* configs are for the metrics manager |
| |
| # The size of packets to read from socket will be determined by the minimal of: (a) time based (b) size based |
| |
| # Time based, the maximum batch time in ms for metricsmgr to read from socket |
| heron.metricsmgr.network.read.batch.time.ms: 16 |
| |
| # Size based, the maximum batch size in bytes to read from socket |
| heron.metricsmgr.network.read.batch.size.bytes: 32768 |
| |
| # The size of packets to write to socket will be determined by the minimal of: (a) time based (b) size based |
| |
| # Time based, the maximum batch time in ms for metricsmgr to write to socket |
| heron.metricsmgr.network.write.batch.time.ms: 16 |
| |
| # Size based, the maximum batch size in bytes to write to socket |
| heron.metricsmgr.network.write.batch.size.bytes: 32768 |
| |
| # The maximum socket's send buffer size in bytes |
| heron.metricsmgr.network.options.socket.send.buffer.size.bytes: 6553600 |
| |
| # The maximum socket's received buffer size in bytes of metricsmgr's network options |
| heron.metricsmgr.network.options.socket.received.buffer.size.bytes: 8738000 |
| |
| # The maximum packet size that metrics mgr can read |
| heron.metricsmgr.network.options.maximum.packetsize.bytes: 1048576 |
| |
| # Attempts to restart IMetricsSink once failure ia occurred |
| # heron.metricsmgr.sink.retry.attempts: 10 |
| |
| # # When sinks' failure exceeds retry attempts, 1 means we would exit the metrics manager, while 0 means we close just the bad sink but keep metrics manager running |
| # heron.metricsmgr.exit.in.sinks.failure: 0 |
| |
| ### heron.instance.* configs are for the instances |
| |
| # The maximum size of packets that instance can read |
| heron.instance.network.options.maximum.packetsize.bytes: 10485760 |
| |
| # The queue capacity (num of items) in bolt for buffer packets to read from stream manager |
| heron.instance.internal.bolt.read.queue.capacity: 128 |
| |
| # The queue capacity (num of items) in bolt for buffer packets to write to stream manager |
| heron.instance.internal.bolt.write.queue.capacity: 128 |
| |
| # The queue capacity (num of items) in spout for buffer packets to read from stream manager |
| heron.instance.internal.spout.read.queue.capacity: 1024 |
| |
| # The queue capacity (num of items) in spout for buffer packets to write to stream manager |
| heron.instance.internal.spout.write.queue.capacity: 128 |
| |
| # The queue capacity (num of items) for metrics packets to write to metrics manager |
| heron.instance.internal.metrics.write.queue.capacity: 128 |
| |
| # The size of packets read from stream manager will be determined by the minimal of: (a) time based (b) size based |
| |
| # Time based, the maximum batch time in ms for instance to read from stream manager per attempt |
| heron.instance.network.read.batch.time.ms: 16 |
| |
| # Size based, the maximum batch size in bytes to read from stream manager |
| heron.instance.network.read.batch.size.bytes: 32768 |
| |
| # The size of packets written to stream manager will be determined by the minimal of: (a) time based (b) size based |
| |
| # Time based, the maximum batch time in ms for instance to write to stream manager per attempt |
| heron.instance.network.write.batch.time.ms: 16 |
| |
| # Size based, the maximum batch size in bytes to write to stream manager |
| heron.instance.network.write.batch.size.bytes: 32768 |
| |
| # The maximum socket's send buffer size in bytes |
| heron.instance.network.options.socket.send.buffer.size.bytes: 6553600 |
| |
| # The maximum socket's received buffer size in bytes of instance's network options |
| heron.instance.network.options.socket.received.buffer.size.bytes: 8738000 |
| |
| # The maximum # of data tuple to batch in a HeronDataTupleSet protobuf |
| heron.instance.set.data.tuple.capacity: 256 |
| |
| # The maximum size in bytes of data tuple to batch in a HeronDataTupleSet protobuf |
| heron.instance.set.data.tuple.size.bytes: 8388608 |
| |
| # The maximum # of control tuple to batch in a HeronControlTupleSet protobuf |
| heron.instance.set.control.tuple.capacity: 256 |
| |
| # The maximum time in ms for an spout to do acknowledgement per attempt, the ack batch could also break if there are no more ack tuples to process |
| heron.instance.ack.batch.time.ms: 128 |
| |
| |
| # The maximum time in ms for an spout instance to emit tuples per attempt |
| heron.instance.emit.batch.time.ms: 16 |
| |
| # The maximum batch size in bytes for an spout to emit tuples per attempt |
| heron.instance.emit.batch.size.bytes: 32768 |
| |
| # The maximum time in ms for an bolt instance to execute tuples per attempt |
| heron.instance.execute.batch.time.ms: 16 |
| |
| # The maximum batch size in bytes for an bolt instance to execute tuples per attempt |
| heron.instance.execute.batch.size.bytes: 32768 |
| |
| # The time to wait before the instance exits forcibly when uncaught exception happens |
| heron.instance.force.exit.timeout.ms: 2000 |
| |
| # Interval in seconds to reconnect to the stream manager, including the request timeout in connecting |
| heron.instance.reconnect.streammgr.interval.sec: 5 |
| heron.instance.reconnect.streammgr.times: 60 |
| |
| # Interval in seconds to reconnect to the metrics manager, including the request timeout in connecting |
| heron.instance.reconnect.metricsmgr.interval.sec: 5 |
| heron.instance.reconnect.metricsmgr.times: 60 |
| |
| # The interval in second for an instance to sample its system metrics, for instance, CPU load. |
| heron.instance.metrics.system.sample.interval.sec: 10 |
| |
| # For efficient acknowledgement |
| heron.instance.acknowledgement.nbuckets: 10 |
| |
| ## For dynamically tuning the available sizes in the interval read&write queues to keep high performance while avoid gc issues |
| |
| # The expected size on read queue in bolt |
| heron.instance.tuning.expected.bolt.read.queue.size: 5 |
| |
| # The expected size on write queue in bolt |
| heron.instance.tuning.expected.bolt.write.queue.size: 5 |
| |
| # The expected size on read queue in spout |
| heron.instance.tuning.expected.spout.read.queue.size: 512 |
| |
| # The exepected size on write queue in spout |
| heron.instance.tuning.expected.spout.write.queue.size: 5 |
| |
| # The expected size on metrics write queue |
| heron.instance.tuning.expected.metrics.write.queue.size: 5 |
| heron.instance.tuning.current.sample.weight: 0.8 |