heron/config/src/yaml/conf/sandbox/heron_internals.yaml - incubator-heron - Git at Google

 ################################################################################
 # Default values for various configs used inside Heron.
 ################################################################################
 # All the config associated with time is in the unit of milli-seconds,
 # unless otherwise specified.
 ################################################################################
 # All the config associated with data size is in the unit of bytes, unless
 # otherwise specified.
 ################################################################################

 ################################################################################
 # System level configs
 ################################################################################

 ### heron.* configs are general configurations over all componenets

 # The relative path to the logging directory
 heron.logging.directory: "log-files"

 # The maximum log file size in MB
 heron.logging.maximum.size.mb: 100

 # The maximum number of log files
 heron.logging.maximum.files: 5

 # The interval in seconds after which to check if the tmaster location has been fetched or not
 heron.check.tmaster.location.interval.sec: 120

 # The interval in seconds to prune logging files in C++
 heron.logging.prune.interval.sec: 300

 # The interval in seconds to flush log files in C++
 heron.logging.flush.interval.sec: 10

 # The threshold level to log error
 heron.logging.err.threshold: 3

 # The interval in seconds for different components to export metrics to metrics manager
 heron.metrics.export.interval.sec: 60

 # The maximum count of exceptions in one MetricPublisherPublishMessage protobuf
 heron.metrics.max.exceptions.per.message.count: 1024

 ################################################################################
 # Configs related to Stream Manager, starts with heron.streammgr.*
 ################################################################################

 # The tuple cache (used for batching) can be drained in two ways:
 # (a) Time based
 # (b) size based

 # The frequency in ms to drain the tuple cache in stream manager
 heron.streammgr.cache.drain.frequency.ms: 10

 # The sized based threshold in MB for buffering data tuples
 # waiting for checkpoint markers before giving up
 heron.streammgr.stateful.buffer.size.mb: 100

 # The sized based threshold in MB for draining the tuple cache
 heron.streammgr.cache.drain.size.mb: 100

 # For efficient acknowledgements
 heron.streammgr.xormgr.rotatingmap.nbuckets: 3

 # The max number of messages in the memory pool for each message type
 heron.streammgr.mempool.max.message.number: 512

 # The max reconnect attempts to other stream managers for stream manager client
 heron.streammgr.client.reconnect.max.attempts: 300

 # The reconnect interval to other stream managers in secs for stream manager client
 heron.streammgr.client.reconnect.interval.sec: 1

 # The reconnect interval to tamster in second for stream manager client
 heron.streammgr.client.reconnect.tmaster.interval.sec: 10

 # The maximum packet size in MB of stream manager's network options
 heron.streammgr.network.options.maximum.packet.mb: 10

 # The interval in seconds to send heartbeat
 heron.streammgr.tmaster.heartbeat.interval.sec: 10

 # Maximum batch size in MB to read by stream manager from socket
 heron.streammgr.connection.read.batch.size.mb: 1

 # Maximum batch size in MB to write by stream manager to socket
 heron.streammgr.connection.write.batch.size.mb: 1

 # Number of times we should wait to see a buffer full while enqueueing data
 # before declaring start of back pressure
 heron.streammgr.network.backpressure.threshold: 3

 # High water mark on the num in MB that can be left outstanding on a connection
 heron.streammgr.network.backpressure.highwatermark.mb: 100

 # Low water mark on the num in MB that can be left outstanding on a connection
 heron.streammgr.network.backpressure.lowwatermark.mb: 50

 ################################################################################
 # Configs related to Topology Master, starts with heron.tmaster.*
 ################################################################################

 # The maximum interval in minutes of metrics to be kept in tmaster
 heron.tmaster.metrics.collector.maximum.interval.min: 180

 # The maximum time to retry to establish the tmaster
 heron.tmaster.establish.retry.times: 30

 # The interval to retry to establish the tmaster
 heron.tmaster.establish.retry.interval.sec: 1

 # Maximum packet size in MB of tmaster's network options to connect to stream managers
 heron.tmaster.network.master.options.maximum.packet.mb: 16

 # Maximum packet size in MB of tmaster's network options to connect to scheduler
 heron.tmaster.network.controller.options.maximum.packet.mb: 1

 # Maximum packet size in MB of tmaster's network options for stat queries
 heron.tmaster.network.stats.options.maximum.packet.mb: 1

 # The interval for tmaster to purge metrics from socket
 heron.tmaster.metrics.collector.purge.interval.sec: 60

 # The maximum # of exceptions to be stored in tmetrics collector, to prevent potential OOM
 heron.tmaster.metrics.collector.maximum.exception: 256

 # Should the metrics reporter bind on all interfaces
 heron.tmaster.metrics.network.bindallinterfaces: False

 # The timeout in seconds for stream mgr, compared with (current time - last heartbeat time)
 heron.tmaster.stmgr.state.timeout.sec: 60

 ################################################################################
 # Configs related to Topology Master, starts with heron.metricsmgr.*
 ################################################################################

 # The size of packets to read from socket will be determined by the minimal of:
 # (a) time based
 # (b) size based

 # Time based, the maximum batch time in ms for metricsmgr to read from socket
 heron.metricsmgr.network.read.batch.time.ms: 16

 # Size based, the maximum batch size in bytes to read from socket
 heron.metricsmgr.network.read.batch.size.bytes: 32768

 # The size of packets to write to socket will be determined by the minimum of
 # (a) time based
 # (b) size based

 # Time based, the maximum batch time in ms for metricsmgr to write to socket
 heron.metricsmgr.network.write.batch.time.ms: 16

 # Size based, the maximum batch size in bytes to write to socket
 heron.metricsmgr.network.write.batch.size.bytes: 32768

 # The maximum socket's send buffer size in bytes
 heron.metricsmgr.network.options.socket.send.buffer.size.bytes: 655360

 # The maximum socket's received buffer size in bytes of metricsmgr's network options
 heron.metricsmgr.network.options.socket.received.buffer.size.bytes: 873800

 # The maximum packet size that metrics mgr can read
 heron.metricsmgr.network.options.maximum.packetsize.bytes: 1048576

 ################################################################################
 # Configs related to Heron Instance, starts with heron.instance.*
 ################################################################################

 # The maximum size of packets that instance can read
 heron.instance.network.options.maximum.packetsize.bytes: 10485760

 # The queue capacity (num of items) in bolt for buffer packets to read from stream manager
 heron.instance.internal.bolt.read.queue.capacity: 128

 # The queue capacity (num of items) in bolt for buffer packets to write to stream manager
 heron.instance.internal.bolt.write.queue.capacity: 128

 # The queue capacity (num of items) in spout for buffer packets to read from stream manager
 heron.instance.internal.spout.read.queue.capacity: 1024

 # The queue capacity (num of items) in spout for buffer packets to write to stream manager
 heron.instance.internal.spout.write.queue.capacity: 128

 # The queue capacity (num of items) for metrics packets to write to metrics manager
 heron.instance.internal.metrics.write.queue.capacity: 128

 # The size of packets read from stream manager will be determined by the minimal of
 # (a) time based
 # (b) size based

 # Time based, the maximum batch time in ms for instance to read from stream manager per attempt
 heron.instance.network.read.batch.time.ms: 16

 # Size based, the maximum batch size in bytes to read from stream manager
 heron.instance.network.read.batch.size.bytes: 32768

 # The size of packets written to stream manager will be determined by the minimum of
 # (a) time based
 # (b) size based

 # Time based, the maximum batch time in ms for instance to write to stream manager per attempt
 heron.instance.network.write.batch.time.ms: 16

 # Size based, the maximum batch size in bytes to write to stream manager
 heron.instance.network.write.batch.size.bytes: 32768

 # The maximum socket's send buffer size in bytes
 heron.instance.network.options.socket.send.buffer.size.bytes: 6553600

 # The maximum socket's received buffer size in bytes of instance's network options
 heron.instance.network.options.socket.received.buffer.size.bytes: 8738000

 # The maximum # of data tuple to batch in a HeronDataTupleSet protobuf
 heron.instance.set.data.tuple.capacity: 10240

 # The maximum size in bytes of data tuple to batch in a HeronDataTupleSet protobuf
 heron.instance.set.data.tuple.size.bytes: 1310720

 # The maximum # of control tuple to batch in a HeronControlTupleSet protobuf
 heron.instance.set.control.tuple.capacity: 1024

 # The maximum time in ms for a spout to do acknowledgement per attempt, the ack batch could
 # also break if there are no more ack tuples to process
 heron.instance.ack.batch.time.ms: 128

 # The maximum time in ms for an spout instance to emit tuples per attempt
 heron.instance.emit.batch.time.ms: 160

 # The maximum batch size in bytes for an spout to emit tuples per attempt
 heron.instance.emit.batch.size.bytes: 327680

 # The maximum time in ms for an bolt instance to execute tuples per attempt
 heron.instance.execute.batch.time.ms: 16

 # The maximum batch size in bytes for an bolt instance to execute tuples per attempt
 heron.instance.execute.batch.size.bytes: 32768

 # The time to wait before the instance exits forcibly when uncaught exception happens
 heron.instance.force.exit.timeout.ms: 2000

 # Interval in seconds to reconnect to the stream manager, including the request timeout in connecting
 heron.instance.reconnect.streammgr.interval.sec: 5
 heron.instance.reconnect.streammgr.times: 60

 # Interval in seconds to reconnect to the metrics manager, including the request timeout in connecting
 heron.instance.reconnect.metricsmgr.interval.sec: 5
 heron.instance.reconnect.metricsmgr.times: 60

 # The interval in second for an instance to sample its system metrics, for instance, cpu load.
 heron.instance.metrics.system.sample.interval.sec: 10

 # For efficient acknowledgement
 heron.instance.acknowledgement.nbuckets: 10

 ################################################################################
 # For dynamically tuning the available sizes in the interval read & write queues
 # to provide high performance while avoiding GC issues
 ################################################################################

 # The expected size on read queue in bolt
 heron.instance.tuning.expected.bolt.read.queue.size: 8

 # The expected size on write queue in bolt
 heron.instance.tuning.expected.bolt.write.queue.size: 8

 # The expected size on read queue in spout
 heron.instance.tuning.expected.spout.read.queue.size: 512

 # The exepected size on write queue in spout
 heron.instance.tuning.expected.spout.write.queue.size: 8

 # The expected size on metrics write queue
 heron.instance.tuning.expected.metrics.write.queue.size: 8

 heron.instance.tuning.current.sample.weight: 0.8

 # Interval in ms to tune the size of in & out data queue in instance
 heron.instance.tuning.interval.ms: 100
	################################################################################
	# Default values for various configs used inside Heron.
	################################################################################
	# All the config associated with time is in the unit of milli-seconds,
	# unless otherwise specified.
	################################################################################
	# All the config associated with data size is in the unit of bytes, unless
	# otherwise specified.
	################################################################################

	################################################################################
	# System level configs
	################################################################################

	### heron.* configs are general configurations over all componenets

	# The relative path to the logging directory
	heron.logging.directory: "log-files"

	# The maximum log file size in MB
	heron.logging.maximum.size.mb: 100

	# The maximum number of log files
	heron.logging.maximum.files: 5

	# The interval in seconds after which to check if the tmaster location has been fetched or not
	heron.check.tmaster.location.interval.sec: 120

	# The interval in seconds to prune logging files in C++
	heron.logging.prune.interval.sec: 300

	# The interval in seconds to flush log files in C++
	heron.logging.flush.interval.sec: 10

	# The threshold level to log error
	heron.logging.err.threshold: 3

	# The interval in seconds for different components to export metrics to metrics manager
	heron.metrics.export.interval.sec: 60

	# The maximum count of exceptions in one MetricPublisherPublishMessage protobuf
	heron.metrics.max.exceptions.per.message.count: 1024

	################################################################################
	# Configs related to Stream Manager, starts with heron.streammgr.*
	################################################################################

	# The tuple cache (used for batching) can be drained in two ways:
	# (a) Time based
	# (b) size based

	# The frequency in ms to drain the tuple cache in stream manager
	heron.streammgr.cache.drain.frequency.ms: 10

	# The sized based threshold in MB for buffering data tuples
	# waiting for checkpoint markers before giving up
	heron.streammgr.stateful.buffer.size.mb: 100

	# The sized based threshold in MB for draining the tuple cache
	heron.streammgr.cache.drain.size.mb: 100

	# For efficient acknowledgements
	heron.streammgr.xormgr.rotatingmap.nbuckets: 3

	# The max number of messages in the memory pool for each message type
	heron.streammgr.mempool.max.message.number: 512

	# The max reconnect attempts to other stream managers for stream manager client
	heron.streammgr.client.reconnect.max.attempts: 300

	# The reconnect interval to other stream managers in secs for stream manager client
	heron.streammgr.client.reconnect.interval.sec: 1

	# The reconnect interval to tamster in second for stream manager client
	heron.streammgr.client.reconnect.tmaster.interval.sec: 10

	# The maximum packet size in MB of stream manager's network options
	heron.streammgr.network.options.maximum.packet.mb: 10

	# The interval in seconds to send heartbeat
	heron.streammgr.tmaster.heartbeat.interval.sec: 10

	# Maximum batch size in MB to read by stream manager from socket
	heron.streammgr.connection.read.batch.size.mb: 1

	# Maximum batch size in MB to write by stream manager to socket
	heron.streammgr.connection.write.batch.size.mb: 1

	# Number of times we should wait to see a buffer full while enqueueing data
	# before declaring start of back pressure
	heron.streammgr.network.backpressure.threshold: 3

	# High water mark on the num in MB that can be left outstanding on a connection
	heron.streammgr.network.backpressure.highwatermark.mb: 100

	# Low water mark on the num in MB that can be left outstanding on a connection
	heron.streammgr.network.backpressure.lowwatermark.mb: 50

	################################################################################
	# Configs related to Topology Master, starts with heron.tmaster.*
	################################################################################

	# The maximum interval in minutes of metrics to be kept in tmaster
	heron.tmaster.metrics.collector.maximum.interval.min: 180

	# The maximum time to retry to establish the tmaster
	heron.tmaster.establish.retry.times: 30

	# The interval to retry to establish the tmaster
	heron.tmaster.establish.retry.interval.sec: 1

	# Maximum packet size in MB of tmaster's network options to connect to stream managers
	heron.tmaster.network.master.options.maximum.packet.mb: 16

	# Maximum packet size in MB of tmaster's network options to connect to scheduler
	heron.tmaster.network.controller.options.maximum.packet.mb: 1

	# Maximum packet size in MB of tmaster's network options for stat queries
	heron.tmaster.network.stats.options.maximum.packet.mb: 1

	# The interval for tmaster to purge metrics from socket
	heron.tmaster.metrics.collector.purge.interval.sec: 60

	# The maximum # of exceptions to be stored in tmetrics collector, to prevent potential OOM
	heron.tmaster.metrics.collector.maximum.exception: 256

	# Should the metrics reporter bind on all interfaces
	heron.tmaster.metrics.network.bindallinterfaces: False

	# The timeout in seconds for stream mgr, compared with (current time - last heartbeat time)
	heron.tmaster.stmgr.state.timeout.sec: 60

	################################################################################
	# Configs related to Topology Master, starts with heron.metricsmgr.*
	################################################################################

	# The size of packets to read from socket will be determined by the minimal of:
	# (a) time based
	# (b) size based

	# Time based, the maximum batch time in ms for metricsmgr to read from socket
	heron.metricsmgr.network.read.batch.time.ms: 16

	# Size based, the maximum batch size in bytes to read from socket
	heron.metricsmgr.network.read.batch.size.bytes: 32768

	# The size of packets to write to socket will be determined by the minimum of
	# (a) time based
	# (b) size based

	# Time based, the maximum batch time in ms for metricsmgr to write to socket
	heron.metricsmgr.network.write.batch.time.ms: 16

	# Size based, the maximum batch size in bytes to write to socket
	heron.metricsmgr.network.write.batch.size.bytes: 32768

	# The maximum socket's send buffer size in bytes
	heron.metricsmgr.network.options.socket.send.buffer.size.bytes: 655360

	# The maximum socket's received buffer size in bytes of metricsmgr's network options
	heron.metricsmgr.network.options.socket.received.buffer.size.bytes: 873800

	# The maximum packet size that metrics mgr can read
	heron.metricsmgr.network.options.maximum.packetsize.bytes: 1048576

	################################################################################
	# Configs related to Heron Instance, starts with heron.instance.*
	################################################################################

	# The maximum size of packets that instance can read
	heron.instance.network.options.maximum.packetsize.bytes: 10485760

	# The queue capacity (num of items) in bolt for buffer packets to read from stream manager
	heron.instance.internal.bolt.read.queue.capacity: 128

	# The queue capacity (num of items) in bolt for buffer packets to write to stream manager
	heron.instance.internal.bolt.write.queue.capacity: 128

	# The queue capacity (num of items) in spout for buffer packets to read from stream manager
	heron.instance.internal.spout.read.queue.capacity: 1024

	# The queue capacity (num of items) in spout for buffer packets to write to stream manager
	heron.instance.internal.spout.write.queue.capacity: 128

	# The queue capacity (num of items) for metrics packets to write to metrics manager
	heron.instance.internal.metrics.write.queue.capacity: 128

	# The size of packets read from stream manager will be determined by the minimal of
	# (a) time based
	# (b) size based

	# Time based, the maximum batch time in ms for instance to read from stream manager per attempt
	heron.instance.network.read.batch.time.ms: 16

	# Size based, the maximum batch size in bytes to read from stream manager
	heron.instance.network.read.batch.size.bytes: 32768

	# The size of packets written to stream manager will be determined by the minimum of
	# (a) time based
	# (b) size based

	# Time based, the maximum batch time in ms for instance to write to stream manager per attempt
	heron.instance.network.write.batch.time.ms: 16

	# Size based, the maximum batch size in bytes to write to stream manager
	heron.instance.network.write.batch.size.bytes: 32768

	# The maximum socket's send buffer size in bytes
	heron.instance.network.options.socket.send.buffer.size.bytes: 6553600

	# The maximum socket's received buffer size in bytes of instance's network options
	heron.instance.network.options.socket.received.buffer.size.bytes: 8738000

	# The maximum # of data tuple to batch in a HeronDataTupleSet protobuf
	heron.instance.set.data.tuple.capacity: 10240

	# The maximum size in bytes of data tuple to batch in a HeronDataTupleSet protobuf
	heron.instance.set.data.tuple.size.bytes: 1310720

	# The maximum # of control tuple to batch in a HeronControlTupleSet protobuf
	heron.instance.set.control.tuple.capacity: 1024

	# The maximum time in ms for a spout to do acknowledgement per attempt, the ack batch could
	# also break if there are no more ack tuples to process
	heron.instance.ack.batch.time.ms: 128

	# The maximum time in ms for an spout instance to emit tuples per attempt
	heron.instance.emit.batch.time.ms: 160

	# The maximum batch size in bytes for an spout to emit tuples per attempt
	heron.instance.emit.batch.size.bytes: 327680

	# The maximum time in ms for an bolt instance to execute tuples per attempt
	heron.instance.execute.batch.time.ms: 16

	# The maximum batch size in bytes for an bolt instance to execute tuples per attempt
	heron.instance.execute.batch.size.bytes: 32768

	# The time to wait before the instance exits forcibly when uncaught exception happens
	heron.instance.force.exit.timeout.ms: 2000

	# Interval in seconds to reconnect to the stream manager, including the request timeout in connecting
	heron.instance.reconnect.streammgr.interval.sec: 5
	heron.instance.reconnect.streammgr.times: 60

	# Interval in seconds to reconnect to the metrics manager, including the request timeout in connecting
	heron.instance.reconnect.metricsmgr.interval.sec: 5
	heron.instance.reconnect.metricsmgr.times: 60

	# The interval in second for an instance to sample its system metrics, for instance, cpu load.
	heron.instance.metrics.system.sample.interval.sec: 10

	# For efficient acknowledgement
	heron.instance.acknowledgement.nbuckets: 10

	################################################################################
	# For dynamically tuning the available sizes in the interval read & write queues
	# to provide high performance while avoiding GC issues
	################################################################################

	# The expected size on read queue in bolt
	heron.instance.tuning.expected.bolt.read.queue.size: 8

	# The expected size on write queue in bolt
	heron.instance.tuning.expected.bolt.write.queue.size: 8

	# The expected size on read queue in spout
	heron.instance.tuning.expected.spout.read.queue.size: 512

	# The exepected size on write queue in spout
	heron.instance.tuning.expected.spout.write.queue.size: 8

	# The expected size on metrics write queue
	heron.instance.tuning.expected.metrics.write.queue.size: 8

	heron.instance.tuning.current.sample.weight: 0.8

	# Interval in ms to tune the size of in & out data queue in instance
	heron.instance.tuning.interval.ms: 100