core/server/config.toml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Configuration for consumer group cooperative partition rebalancing.
[consumer_group]
# Maximum time a partition can remain in pending revocation before being force-transferred to the target member.
rebalancing_timeout = "30s"

[data_maintenance.messages]
# Enables or disables the segment cleaner. It deletes the oldest sealed segments of
# topics with a finite message_expiry or max_topic_size, per partition, best-effort.
cleaner_enabled = true

# Interval for running the message cleaner.
interval = "1 m"

# This node's own client-facing identity, used when 'cluster.enabled' is false.
[node]
# A literal IP or a DNS hostname, without a port. Named to match its roster
# counterpart 'cluster.nodes.advertised_address', which answers the same
# question per node. The unspecified address ("0.0.0.0", "::") is rejected: it
# is the bind address's answer to a question clients are not asking. Commented
# out here because the shipped 'tcp.address' binds a concrete address and needs
# no declaration.
#advertised_address = "broker-1.example.com"

# HTTP server configuration
[http]
# Determines if the HTTP server is active.
# `true` enables the server, allowing it to handle HTTP requests.
# `false` disables the server, preventing it from handling HTTP requests.
# In cluster mode, followers forward control-plane requests (streams, topics,
# users, ...) to the current primary when a cluster-wide JWT key exists (see
# http.jwt / cluster.auth below).
# TODO: forwarding does not cover the partition-plane APIs yet - message
# produce and consumer-offset writes are never forwarded and must reach the
# partition's primary node directly (message polls read locally on any node).
enabled = true

# Specifies the network address and port for the HTTP server.
# The format is "HOST:PORT". For example, "127.0.0.1:3000" listens on localhost only on port 3000.
# In cluster mode the HOST still picks the bind interface, while the port
# comes from this node's cluster.nodes ports.http entry.
address = "127.0.0.1:3000"

# Maximum size of the request body in bytes. For security reasons, the default limit is 2 MB.
# HTTP produce requests are not framed by the message bus, so this body cap is
# what bounds the widest batch record the HTTP path can persist. Boot refuses a
# value above the frozen 256 MiB recovery ceiling (see
# message_bus.max_message_size), and a body above message_bus.max_message_size
# can still admit a batch that no peer accepts a replication frame for -- keep
# this at or below that cap unless replica_count = 1.
max_request_size = "2 MB"

# Enables the embedded Web UI dashboard at '/ui'.
# When set to `true` and the server is compiled with the 'iggy-web' feature,
# the Svelte dashboard will be served at the '/ui' endpoint, providing a
# browser-based interface for managing streams, topics, and viewing messages.
# If the server is compiled without 'iggy-web' feature and this is set to `true`,
# a warning will be logged at startup but the server will continue to run.
# `true` enables the embedded Web UI (requires server built with 'iggy-web' feature).
# `false` disables the embedded Web UI (default).
web_ui = false

# Configuration for Cross-Origin Resource Sharing (CORS).
[http.cors]
# Controls whether CORS is enabled for the HTTP server.
# `true` allows handling cross-origin requests with specified rules.
# `false` blocks cross-origin requests, enhancing security.
enabled = true

# Specifies which HTTP methods are allowed when CORS is enabled.
# For example, ["GET", "POST"] would allow only GET and POST requests.
allowed_methods = ["GET", "POST", "PUT", "DELETE"]

# Defines which origins are permitted to make cross-origin requests.
# An asterisk "*" as the first entry allows all origins (any entries after it
# are ignored); "*" in any other position fails the config. Specific domains
# can be listed to restrict access.
allowed_origins = ["*"]

# Lists allowed headers that can be used in CORS requests.
# For example, ["content-type"] permits only the content-type header.
allowed_headers = ["content-type", "authorization"]

# Headers that browsers are allowed to access in CORS responses.
# `iggy-view` carries the current VSR view number; exposing it lets browser
# clients read it on cross-origin responses.
exposed_headers = ["iggy-view"]

# Determines if credentials like cookies or HTTP auth can be included in CORS requests.
# `true` allows credentials to be included, useful for authenticated sessions;
# it requires explicit (non-wildcard) allowed_origins, allowed_headers, and
# exposed_headers.
# `false` prevents credentials, enhancing privacy and security.
allow_credentials = false

# Allows or blocks requests from private networks in CORS.
# `true` permits requests from private networks.
# `false` disallows such requests, providing additional security.
allow_private_network = false

# JWT (JSON Web Token) configuration for HTTP.
[http.jwt]
# Specifies the algorithm used for signing JWTs.
# For example, "HS256" indicates HMAC with SHA-256.
algorithm = "HS256"

# The issuer of the JWT, typically a URL or an identifier of the issuing entity.
issuer = "iggy.apache.org"

# Intended audience for the JWT, usually the recipient or system intended to process the token.
audience = "iggy.apache.org"

# Lists valid issuers for JWT validation to ensure tokens are from trusted sources.
valid_issuers = ["iggy.apache.org"]

# Lists valid audiences for JWT validation to confirm tokens are for the intended recipient.
valid_audiences = ["iggy.apache.org"]

# Expiry time for access tokens.
access_token_expiry = "1 h"

# Tolerance for timing discrepancies during token validation.
clock_skew = "5 s"

# Time before which the token should not be considered valid.
not_before = "0 s"

# Secret key for encoding JWTs.
# If left empty, a secure random secret will be generated on each server start.
# In cluster mode a configured secret (identical on every node) makes bearers
# valid cluster-wide and activates follower-to-primary HTTP forwarding; with
# cluster.auth enabled the key is instead derived from the shared PSK. Without
# either, tokens are node-local and forwarding stays disabled.
encoding_secret = ""

# Secret key for decoding JWTs.
# If left empty, a secure random secret will be generated on each server start.
decoding_secret = ""

# Indicates if the secret key is base64 encoded.
# `true` means the secret is base64 encoded.
# `false` means the secret is in plain text.
use_base64_secret = false

# Trusted issuers for A2A (Application-to-Application) authentication. Opt-in:
# with none configured the listener accepts only self-issued HS256 tokens.
# `issuer`, `audience` and `jwks_url` are required per entry; `user_id` is
# optional but defaults to 0 (root), which is rejected - set it to the non-zero
# iggy user every token from that issuer is remapped onto.
#
# Operational note: enabling an issuer opens an outbound JWKS fetch that is
# reachable before a token's signature is verified - a token naming this issuer
# with an unknown key id can trigger a fetch to `jwks_url`. The target is fixed
# (not attacker-chosen); concurrent misses coalesce onto one fetch and repeats
# are rate-limited to at most one outbound request per issuer per short window,
# so an unknown-key-id flood cannot amplify. The same window bounds how quickly a
# freshly rotated signing key is picked up.
# [[http.jwt.trusted_issuers]]
# issuer = "test-issuer"
# jwks_url = "http://127.0.0.1:8081/.well-known/jwks.json"
# audience = "iggy.apache.org"
# user_id = 1

# Metrics configuration for HTTP.
[http.metrics]
# Enable or disable the metrics endpoint.
# `true` makes metrics available at the specified endpoint.
# `false` disables metrics collection.
enabled = true

# Specifies the endpoint for accessing metrics, e.g., "/metrics".
# This route requires authentication like every other read: scrapers must
# present a bearer credential (JWT or personal access token).
endpoint = "/metrics"

# TLS (Transport Layer Security) configuration for HTTP.
[http.tls]
# Controls the use of TLS for encrypted HTTP connections.
# `true` enables TLS, enhancing security.
# `false` disables TLS, which may be appropriate in secure internal networks.
enabled = false

# Path to the TLS certificate file.
cert_file = "core/certs/iggy_cert.pem"

# Path to the TLS key file.
key_file = "core/certs/iggy_key.pem"

# TCP server configuration.
[tcp]
# Determines if the TCP server is active.
# `true` enables the TCP server for handling TCP connections.
# `false` disables it, preventing any TCP communication.
enabled = true

# Defines the network address and port for the TCP server.
# For example, "127.0.0.1:8090" listens on localhost only on port 8090.
address = "127.0.0.1:8090"

# TLS configuration for the TCP server.
[tcp.tls]
# Enables or disables TLS for TCP connections.
# `true` secures TCP connections with TLS.
# `false` leaves TCP connections unencrypted.
enabled = false

# Certificate source for the TCP TLS listener.
# `true` generates an ephemeral loopback certificate, but only while cert_file
# does not exist on disk: an existing PEM pair is loaded instead (the server
# logs which way it went).
# `false` loads cert_file / key_file, which must both exist.
self_signed = true

# Path to the TLS certificate file. Loaded when the file exists or
# self_signed = false.
cert_file = "core/certs/iggy_cert.pem"

# Path to the TLS key file. Read together with cert_file.
key_file = "core/certs/iggy_key.pem"

# QUIC protocol configuration.
[quic]
# Controls whether the QUIC server is enabled.
# `true` enables QUIC for fast, secure connections.
# `false` disables QUIC, possibly for compatibility or simplicity.
enabled = true

# Network address and port for the QUIC server.
# For example, "127.0.0.1:8080" binds to localhost on port 8080.
address = "127.0.0.1:8080"

# How many bidirectional streams a client may keep open at once on a single
# QUIC connection. The SDK opens a fresh stream per command, so 1 caps a
# connection at one in-flight command: commands do not pipeline, and the next
# one waits for the current reply. The known cost is one stream-credit round
# trip between back-to-back requests on the same connection. Raising it is a
# transport decision only: the server's accept loop takes one bidi at a time
# and waits for its reply before accepting the next, which is what keeps
# partition operations (they all share the connection's current request id)
# distinguishable, and a higher cap does not make the handlers concurrent.
max_concurrent_bidi_streams = 1

# Initial path MTU for QUIC connections, before MTU discovery has run.
# 1200 bytes is the QUIC minimum and quinn's own default: safe on every
# path, and MTU discovery probes upward from it automatically. Values
# above the real path MTU cause packet loss until black-hole detection
# resets the connection back to 1200.
initial_mtu = "1200 B"

# Server-side flow-control budget for unacknowledged data the server may have
# in flight on one connection. A ceiling, not a reservation, and rarely the
# operative limit: the receiver's own window binds first, and the Rust SDK's
# QUIC client sets both of its windows to about 100 KB.
send_window = "64 MiB"

# Server-side flow-control budget for data the server will buffer from one
# connection, shared by all of its streams. Symmetric with `send_window`.
receive_window = "64 MiB"

# Per-stream slice of `receive_window`. Equal to it, because
# `max_concurrent_bidi_streams` is 1: with a single stream per connection a
# smaller carve-out protects nothing, it only caps the one stream early.
# Lower it below `receive_window` if that cap is ever raised, so one unread
# stream cannot pin the whole connection window. Must not exceed
# `receive_window`; equality is allowed.
stream_receive_window = "64 MiB"

# Interval for sending QUIC keep-alive PINGs. One third of
# `max_idle_timeout` so up to two consecutive losses fit before the
# idle timer closes the connection. Set to "0 s" to disable.
keep_alive_interval = "10 s"

# Maximum idle time before a QUIC connection is closed. Set to
# "0 s" to disable (not recommended).
max_idle_timeout = "30 s"

# QUIC certificate configuration.
[quic.certificate]
# Certificate source for the QUIC listener.
# `true` always generates an ephemeral loopback certificate and ignores
# cert_file / key_file even when those files exist on disk; the server logs a
# warning naming the files it skipped.
# `false` loads cert_file / key_file, which must both exist.
self_signed = true

# Path to the QUIC TLS certificate file. Required when self_signed = false,
# left empty here so the shipped self-signed default has nothing to skip.
cert_file = ""

# Path to the QUIC TLS key file. Required when self_signed = false.
key_file = ""

# Personal access token configuration.
[personal_access_token]
# Sets the maximum number of active tokens allowed per user.
max_tokens_per_user = 100

# Personal access token cleaner configuration.
[personal_access_token.cleaner]
# Enables or disables the token cleaner process.
# `true` activates periodic token cleaning.
# `false` disables it, tokens remain active until manually revoked or expired.
enabled = true

# Interval for running the token cleaner.
interval = "1 m"

# Heartbeat configuration
[heartbeat]
# Enables or disables the client heartbeat verification process. When
# enabled, a connection that sends nothing (no request, no PING) for
# 1.2 x interval has its session released so its consumer groups
# rebalance off it. Only a connection that still holds a group
# membership is evicted; one that holds none is left alone and is
# reaped when its socket closes.
enabled = true
# Interval for expected client heartbeats. The Rust, Go, Python, Node and
# async Java SDKs ping automatically every 5 s, well inside the resulting 36 s
# staleness window, but only from a connected high-level client: the Rust and
# async Java pingers are armed by connect(), so a session that logs in without
# it never pings. The blocking Java and the C# SDK have no automatic heartbeat
# at all, only a manual ping. Wherever nothing pings, an idle consumer-group
# member is evicted, and only the application can keep it alive (ping) or
# bring it back afterwards (reconnect).
interval = "30 s"

# OpenTelemetry configuration
[telemetry]
# Enables or disables telemetry.
enabled = false
# Service name for telemetry.
service_name = "iggy"

# OpenTelemetry logs configuration
[telemetry.logs]
# Transport for sending logs. Options: "grpc", "http".
transport = "grpc"
# Endpoint for sending logs.
endpoint = "http://localhost:7281/v1/logs"

# OpenTelemetry traces configuration
[telemetry.traces]
# Transport for sending traces. Options: "grpc", "http".
transport = "grpc"
# Endpoint for sending traces.
endpoint = "http://localhost:7281/v1/traces"

# System configuration.
[system]
# Base path for system data storage.
path = "local_data"

# Runtime configuration.
[system.runtime]
# Path for storing runtime data.
# Specifies the directory where any runtime data is stored, relative to `system.path`.
path = "runtime"

# Logging configuration.
[system.logging]
# Path for storing log files.
path = "logs"

# Log filtering directive using the same syntax as the RUST_LOG environment variable.
# Supports simple levels ("trace", "debug", "info", "warn", "error", "off" or "none")
# as well as complex directives like "warn,server=debug,iggy=trace".
# Note: RUST_LOG environment variable always takes precedence over this setting.
level = "info"

# Whether to write logs to file. When false, logs are only written to stdout.
# When enabled, logs are stored in {system.path}/{system.logging.path} (default: local_data/logs).
file_enabled = true

# Maximum size of a single log file before rotation occurs. When a log
# file reaches this size,  it will be rotated   (closed and a new file
# created). This setting works together with max_total_size to control
# log storage.  You can set it to 0 to enable unlimited size of single
# log,  but all logs will be written to a single file,  thus disabling
# log rotation. Please configure 0 with caution, esp. RUST_LOG > debug
max_file_size = "500 MB"

# Maximum total size of all log files.  When this size is reached,
# the oldest log files will be deleted first. Set it to 0 to allow
# an unlimited number of archived logs. This does not disable time
# based log rotation or per-log-file size limits.
max_total_size = "4 GB"

# Time interval for checking log rotation status. Avoid less than 1s.
rotation_check_interval = "1 h"

# Time to retain log files before deletion. Avoid less than 1s, too.
retention = "7 days"

# Encryption configuration
[system.encryption]
# Determines whether server-side data encryption for the messages payloads and state commands is enabled (boolean).
# `true` enables encryption for stored data using AES-256-GCM.
# `false` means data is stored without encryption.
enabled = false

# The encryption key used when encryption is enabled (string).
# Should be a 32 bytes length key, provided as a base64 encoded string.
# This key is required and used only if encryption is enabled.
key = ""

# Stream configuration
[system.stream]
# Path for storing stream-related data (string).
# Specifies the directory where stream data is stored, relative to `system.path`.
path = "streams"

# Topic configuration
[system.topic]
# Path for storing topic-related data, relative to `stream.path`.
path = "topics"

# Retention is per topic, set at CreateTopic and readable on GetTopic:
#   max_topic_size   - delete oldest sealed segments past this size
#                      ("unlimited" when unset)
#   message_expiry   - delete sealed segments older than this ("none" when unset)
# Both policies can be active at once; the active segment is never touched.
# Call GET /options/topic (or the SDK's describe_options) for the full catalog
# with this server's defaults.

# Partition configuration
[system.partition]
# Path for storing partition-related data (string).
# Specifies the directory where partition data is stored, relative to `topic.path`.
path = "partitions"

# Enables checksum validation for data integrity (boolean).
# `true` re-hashes every batch a disk poll reads and fails the poll closed on a
# mismatch, so a segment damaged at rest is reported instead of served.
# `false` skips the re-hash and serves whatever decodes, which hands a consumer
# bytes provably not the ones written. Only turn it off with a corruption guard
# somewhere else in the stack.
validate_checksum = true

# Durability and flush cadence are per topic, set at CreateTopic:
#   enforce_fsync                     - fdatasync each flush (false when unset)
#   messages_required_to_save         - flush after this many messages (1024)
#   size_of_messages_required_to_save - flush after this many bytes (1 MiB)
# enforce_fsync decides whether a flush syncs, never whether one happens. A
# write is acknowledged when it commits, which precedes its flush unless that
# commit trips a threshold, and the partition journal holding the unflushed
# remainder is in memory and is not replayed at boot. Acks are therefore
# durability-gated only at messages_required_to_save = 1.

# Segment configuration
[system.segment]
# Segment size is per topic, set at CreateTopic as `segment_size` (1 GiB when
# unset). It is a soft limit: a segment may close one whole batch past it.
# Bounds are enforced at creation - a 512 B multiple, at least 1 MiB, and no
# larger than 1 GiB.
#
# `preallocate_segments` is per topic too: it reserves exactly `segment_size`
# up front where the filesystem supports it. Off unless a topic asks for it --
# with the default 1 GiB segment size it costs 1 GiB of real disk per
# partition at creation.

# Configures whether expired segments are archived (boolean) or just deleted without archiving.
# Unsupported: setting this to `true` aborts boot.
archive_expired = false

# Recovery configuration in case of lost data
[system.recovery]
# Controls whether streams/topics/partitions should be recreated if the expected data for existing state is missing (boolean).
# Unsupported: setting this to `true` aborts boot.
recreate_missing_state = false

# At boot, segment recovery walks each partition's segments: bytes after the
# last decodable batch of a genuinely torn tail are physically truncated from
# the .log/.index files. Every index entry is derived from a batch the log
# already held, so an index is never evidence about the log: one the log
# contradicts is repaired, not believed. It is DROPPED whole and rebuilt from
# a byte-0 walk when it holds no whole 24-byte entry, when its entries
# regress in offset or position, when an entry does not match the log batch's
# offset, timestamp, partition and position, when its first entry is not the
# segment's own start offset at position 0, or when the log cannot back its
# LAST entry. Only an index whose
# last entry the log proves anchors a walk, which then runs from that entry
# to the file end. Under the topic's enforce_fsync = true an index the log
# cannot back is measured against the log first: a gap of more than ONE entry
# refuses the partition instead of rebuilding, because fsync lets the index
# outrun the log by at most the chunk in flight at the crash, so a wider gap
# is previously durable data the log lost.
#
# The byte-0 walk is deliberately not narrowed to the entries the log still
# proves. An anchor picked that way can sit ABOVE damage, and the residue
# probe only ever looks forward from where a walk stopped, so a hole under
# the anchor would go unread while end_offset still advertised the offsets
# over it. Reaching that branch means the index and the log already disagree,
# so the extra bytes read are bounded by the segments a crash actually tore.
#
# Every batch a walk accepts passes its own checksum. With enforce_fsync =
# false, the index only locates data because page-cache writeback can preserve
# a later log page while losing an earlier one, so recovery walks every segment
# from byte 0. A clean walk preserves the existing index and performs no disk
# mutation. With enforce_fsync = true, completed serialized log fdatasyncs
# prove the prefix before the last index entry, so a healthy boot starts at
# that last entry and leaves earlier at-rest damage to the read path's
# validate_checksum. Wherever a recovery walk finds damage, it truncates only
# when nothing decodable follows and refuses when something does.
#
# The non-fsync verification pass therefore reads and checksums every log byte
# at boot, although it keeps a clean index in place. A byte-0 repair walk also
# rebuilds a missing or contradicted index durably, adding two fsyncs per
# repaired segment. Either path can take minutes over thousands of segments.
# A restore that dropped every .index repairs every segment; an ordinary
# fsynced crash usually repairs only its tail segment. A slow boot is therefore
# not by itself evidence of a stall. Before, the repair shapes refused boot.
#
# Either walk refuses a verifying batch whose base offset does not continue
# the chain, or whose partition_id stamp is not this partition's. The one
# exception is the first batch at an index-derived anchor: the offset it
# fails to match is the ENTRY's, not the log's, so a mismatch there breaks
# the walk as a stale entry, leaving the verdict to the byte-0 rebuild. A
# batch that fails its own checksum is damage rather than data
# whatever its fields claim: the walk stops there and the residue goes
# through the damage probe like any other torn tail, so a bit flip in a tail
# batch's offset truncates instead of refusing the partition, regardless of
# the flip's direction.
#
# Damage in the middle of a segment is never silently truncated: the
# partition is refused, as is trailing residue whose verification cost
# exhausts the probe's residue-derived work budgets while walked batches
# exist (residue after a walk that proved nothing recovers the segment as
# empty instead, fencing the file pair aside whole). With peer replicas the
# refused files are moved to a .fenced.N directory beside the partition,
# which is rebuilt empty and refilled from a peer. With replica_count = 1
# there is no peer, so nothing is moved and nothing is rebuilt: the refused
# files stay at their original paths, the partition is tombstoned (unrouted,
# clients get the retriable transient status, never an empty poll that would
# read as a healthy empty partition), and the same refusal is re-derived and
# re-logged on every boot. The tombstone lifts in exactly two ways: a
# chain-shape refusal whose planned chain provably holds ZERO recoverable
# bytes rebuilds through .fenced.N at boot, and deleting the
# stream/topic/partition removes the refused files and clears the fence, so
# a recreate of the same ids starts clean. One refusal is not durable: a
# length divergence found when a writer reopens a file recovery just
# truncated clears on the next boot, so that tombstone lasts only for the
# life of the process.
#
# .fenced.N has a second producer, on a partition that recovers FINE: a tail
# segment holding bytes that decode to nothing anywhere is moved there and
# the segment recovered empty, so the partition serves without those bytes.
# One fresh directory per such segment, so .fenced.0, .fenced.1, ... can
# accumulate under a healthy partition. .fenced.N alone therefore does not
# mean a partition was refused -- grep the boot log for "refusing the
# recovered segment chain", which names the directory and the reason.
#
# The metadata WAL truncates genuinely torn tails too; interior WAL damage
# or oversized trailing bytes refuse boot instead.

# Memory pool configuration
[system.memory_pool]
# Enables or disables the memory pool (boolean).
# `true` enables the memory pool.
# `false` disables the memory pool.
enabled = true

# Size of the memory pool (string).
# Example: "512 MiB" or "1 GiB".
# This defines the maximum, total memory allocated for the memory pool.
# Note: This number has to be multiplication of 4096 (default linux page size).
# Minimum size is 512 MiB due to internal implementation details.
size = "4 GiB"

# Maximum number of buffers in each bucket (u32).
# There are 32 buckets in the memory pool. Each bucket can hold up to this number of buffers
# and holds different buffer sizes, from 256 B to 512 MiB.
# Note: This number has to be a power of 2. Minimum value is 128 due to internal implementation details.
bucket_capacity = 8192

# Cluster configuration
[cluster]
# Enables or disables cluster mode (boolean).
# When enabled, this node will participate in the cluster and coordinate with other nodes.
enabled = false

# Unique cluster name (string).
# All nodes in the same cluster must share the same name.
# This prevents accidental cross-cluster communication.
# Permanent on-disk identity: the name is hashed into the cluster id stamped
# into every metadata and partition superblock on first boot. Changing it later
# makes the server refuse to start against existing data, since the recovered id
# no longer matches the configured one. Renaming means starting from an empty
# data directory.
name = "iggy-cluster"

# Consensus runs on a fixed 10ms tick, so every duration below is converted to
# whole ticks: values are rounded down to a multiple of 10ms, and anything under
# 10ms is raised to a single tick rather than firing sooner.

# Backup-side liveness window for a consensus plane's primary (duration).
# A replica that sees no primary traffic for this long starts a view change.
# Raise it on oversubscribed hosts where scheduling stalls fake primary
# death. Must be at least "2s" and at least 4x commit_broadcast_interval: the
# primary signals liveness through its commit broadcast, and the window must
# span several broadcasts so one delayed broadcast never trips an election.
heartbeat_timeout = "5s"

# How often the primary broadcasts its commit point to every backup (duration).
# This is the cluster's liveness signal: each broadcast resets every backup's
# heartbeat_timeout window and carries the latest commit point forward. Must be
# nonzero and, with heartbeat_timeout, satisfy heartbeat_timeout >= 4x this
# value. Drives the consensus CommitMessage timer.
commit_broadcast_interval = "500ms"

# How often the primary retransmits prepares that backups have not yet acked
# (duration). Lower values recover faster from a dropped prepare at the cost of
# more replica traffic; must be nonzero. Drives the consensus Prepare timer.
prepare_retransmit_interval = "250ms"

# How often a replica retransmits its StartViewChange / DoViewChange while a
# view change is in progress (duration). Lower values converge a healthy
# election faster at the cost of more replica traffic; must be nonzero. Drives
# both consensus view-change retransmit timers.
view_change_retransmit_interval = "500ms"

# Backstop for a stalled view change (duration): one that does not conclude
# within this window escalates to a fresh cluster-wide election. Must be nonzero
# and at least 4x view_change_retransmit_interval, so a few dropped view-change
# messages retransmit rather than prematurely escalate.
view_change_status_timeout = "5s"

# How often a recovering or view-change backup re-requests the current view's
# StartView from its primary (duration); must be nonzero. Drives the consensus
# RequestStartView timer.
request_start_view_retransmit_interval = "1s"

# How many consecutive unanswered RequestStartView probes a recovering replica
# tolerates before falling back to an election (integer). A full-cluster restart
# leaves nobody settled to answer, so the replica elects on its recovered log.
# Must be between 1 and 100.
view_probe_attempts_max = 5

# How long a stalled journal-repair stream waits before re-requesting its
# remaining window from the serving peer (duration). Repair frames are
# fire-and-forget over the lossy bus, so a session with no retry wedges forever
# on a single dropped frame. Paces both the metadata and partition repair loops;
# must be nonzero.
repair_retry_interval = "1s"

# Prepares a peer serves per repair round before the requester walks to the next
# chunk (integer). Each frame rides the per-peer message-bus queue, so this must
# stay strictly below message_bus.peer_queue_capacity or a full round overruns
# the queue and drops frames. Must be > 0 and <= 1024.
repair_chunk_max = 128

# How long the metadata superblock may stay unwritable before the replica
# fail-stops (duration). A replica that cannot persist its view is already
# fenced quorum-invisible and retries with capped backoff; past this window the
# process exits with a distinct status so a supervisor restarts or replaces it
# instead of an operator finding the wedge in logs. "0" disables the fail-stop
# and leaves the replica fenced indefinitely. Nonzero values must be at least
# 30s so a transient disk hiccup cannot kill the process.
superblock_wedged_fatal_timeout = "2m"

# Replica-to-replica authentication (PSK + BLAKE3 keyed-MAC handshake).
[cluster.auth]
# When true, every replica peer must complete the authenticated handshake or be
# rejected, and shared_secret becomes mandatory. Off by default = legacy
# unauthenticated replica traffic. Enabling it is a coordinated-restart change.
# With http enabled and no http.jwt secrets configured, the PSK also becomes
# the JWT key source, making bearers valid cluster-wide and activating
# follower-to-primary HTTP forwarding.
enabled = false

# Cluster-wide pre-shared key, >= 32 bytes of CSPRNG output, byte-identical on
# every node. Prefer the IGGY_CLUSTER_AUTH_SHARED_SECRET env var (masked in
# logs, never persisted) over storing it on disk. Ignored when enabled = false.
shared_secret = ""

# Retiring pre-shared key, accepted for verification only during a rolling key
# rotation (this node keeps signing with shared_secret). Rotate in three rolls:
# 1) shared_secret = old + previous_shared_secret = new on every node,
# 2) shared_secret = new + previous_shared_secret = old on every node,
# 3) shared_secret = new alone. Leave empty outside a rotation. Same length
# floor and env-var preference as shared_secret
# (IGGY_CLUSTER_AUTH_PREVIOUS_SHARED_SECRET).
previous_shared_secret = ""

# Replica-to-replica TLS for the consensus (tcp_replica) port.
[cluster.tls]
# When true every replica connection is wrapped in TLS 1.3 (ALPN
# "iggy-replica") before the replica handshake runs. Requires
# cluster.auth.enabled: TLS carries no client certificates, so it
# authenticates the acceptor only; the PSK handshake authenticates the
# peer, TLS supplies confidentiality. Off by default = plaintext replica
# traffic. Enabling it is a coordinated-restart change: a TLS dialer
# cannot talk to a plaintext acceptor or vice versa.
enabled = false

# When true the node auto-generates a self-signed certificate at boot and
# the dialer accepts ANY peer certificate. When false (default),
# cert_file / key_file / ca_file are all required.
self_signed = false

# PEM certificate chain presented by this node's acceptor side.
cert_file = ""

# PEM private key matching cert_file.
key_file = ""

# PEM trust anchor(s) the dialer verifies peer certificates against.
# Unused when self_signed = true.
# The dialer verifies each peer against the 'ip' string from that peer's
# cluster.nodes entry, and that string is the TLS server name as well as a
# literal IP, so every peer certificate needs a matching IP SAN. A
# certificate carrying only DNS SANs fails verification.
ca_file = ""

# Shard-0 coordinator placement.
[cluster.coordinator]
# When the server runs more than one shard, exclude shard 0 from replica
# placement. Shard 0 already hosts the coordinator, the metadata writer, and
# both listeners; replica connections are long-lived steady flows, so they are
# offloaded to peer shards by default.
skip_shard_zero_for_replicas = true

# When the server runs more than one shard, exclude shard 0 from client
# placement. Off by default: client connections are short-lived and benefit
# from shard-0 parallelism more than replicas do.
skip_shard_zero_for_clients = false

# Full roster of cluster members. Byte-identical on every node. The running
# node's identity is resolved at launch from the '--replica-id <N>' CLI
# flag, which selects the entry in this list that describes the current
# node. All other entries are remote peers.
#
# 'ip' is the node's roster address. Replica-to-replica traffic and
# follower-to-primary HTTP forwarding use it. It is not the bind interface for
# tcp/quic/http/websocket, which comes from each transport's own 'address'
# setting above; the roster supplies those transports their port only. A
# cluster spread across hosts therefore needs each transport's 'address' set to
# '0.0.0.0' or the routable NIC; the defaults below listen on loopback only,
# and a bind that cannot serve the advertised 'ip' is warned about at startup.
#
# Each node may also set 'advertised_address': the client-facing address
# handed out in cluster metadata and leader redirects. Set it when 'ip' is
# a private replica-network address unreachable by clients (Docker,
# Kubernetes, NAT). Accepts a literal IPv4/IPv6 address or a DNS hostname
# (RFC 1123: ASCII letters, digits, '-' and '.'; no port, no trailing dot).
# When unset, clients receive 'ip'.
#
# When different client networks need different addresses (a public
# 'advertised_address' would route in-VPC clients out through the public
# side), add per-network 'advertised_addresses' selectors: clients whose
# peer IP falls inside 'client_cidr' are handed 'address' instead of the
# catch-all. 'address' takes the same forms as 'advertised_address'
# (literal IP or RFC 1123 hostname, never a port - ports always come from
# 'ports'). At most 16 selectors per node; boot also rejects duplicate
# 'client_cidr' entries on one node (compared truncated, so '10.0.1.0/16'
# duplicates '10.0.0.0/16') and any two nodes advertising one host:port
# to overlapping client sets - reusing a host:port across nodes is legal
# only when no client would resolve both nodes to it.
#
# The longest matching prefix wins; clients matching no selector fall
# back to 'advertised_address', then 'ip'. Matching is per address
# family: '0.0.0.0/0' matches no IPv6 client and '::/0' matches no IPv4
# client, so covering both families takes one selector per family (or the
# catch-all). IPv4-mapped IPv6 CIDRs ('::ffff:10.0.0.0/104') match like
# their IPv4 form only at prefix length 96 or longer; shorter ones match
# native IPv6 clients only. Matching sees the transport-level peer
# address, so clients behind a proxy or load balancer match the proxy's
# network, not their own.
#
# Every 'address' must be routable from inside its own 'client_cidr':
# leader-aware SDK clients redial whatever address metadata advertises,
# so a selector pointing at a host its own clients cannot reach strands
# them mid-redirect. Prefer literal IPs over hostnames - the SDKs differ
# in how they compare an advertised hostname against the address they
# dialed, and a mismatch costs a reconnect on every fresh connect.
#
# Note for rolling upgrades: older server binaries reject a TOML config
# containing 'advertised_addresses' but silently ignore the equivalent
# 'IGGY_CLUSTER_NODES_*_ADVERTISED_ADDRESSES_*' env vars; either way,
# upgrade every binary first, then add selectors. Mid-upgrade, an env-var
# roster would serve selector addresses from upgraded nodes and the
# catch-all from the rest.
#
#   [[cluster.nodes]]
#   name = "iggy-node-1"
#   ip = "10.0.1.5"                        # replica plane, literal IP only
#   advertised_address = "203.0.113.10"    # catch-all for unmatched clients
#   replica_id = 0
#   ports = { tcp = 8090, http = 3000, tcp_replica = 9090 }
#
#   [[cluster.nodes.advertised_addresses]]
#   client_cidr = "10.0.0.0/16"            # in-VPC clients stay private
#   address = "10.0.1.5"
#
# In cluster mode, 'ports' is the single source of listener ports: every
# enabled transport needs an explicit per-node port, otherwise the server
# refuses to start.
[[cluster.nodes]]
name = "iggy-node-1"
ip = "127.0.0.1"
replica_id = 0
ports = { tcp = 8090, quic = 8080, http = 3000, websocket = 8092, tcp_replica = 9090 }

[[cluster.nodes]]
name = "iggy-node-2"
ip = "127.0.0.1"
replica_id = 1
ports = { tcp = 8091, quic = 8081, http = 3001, websocket = 8093, tcp_replica = 9091 }

# Example additional node (commented out). tcp skips 8092-8094: those are the
# websocket ports of the three nodes, which collide once nodes share a host.
# [[cluster.nodes]]
# name = "iggy-node-3"
# ip = "192.168.1.100"
# advertised_address = "iggy-node-3.example.com"
# replica_id = 2
# ports = { tcp = 8095, quic = 8082, http = 3002, websocket = 8094, tcp_replica = 9092 }

# Sharding configuration
[system.sharding]
# CPU allocation - controls the number of shards and their CPU affinity.
# Possible values:
# - "all": Use all available CPU cores (default)
# - numeric value (e.g. 4): Use 4 shards (4 threads pinned to cores 0, 1, 2, 3)
# - range (e.g. "5..8"): Use 3 shards with affinity to cores 5, 6, 7
# - numa settings:
#     + "numa:auto": Use all available numa node, cores
#     + "numa:nodes=0,1;cores=4;no_ht=true": Use NUMA node 0 and 1, each nodes use 4 cores, and no hyperthreads
cpu_allocation = "numa:auto"

# Whether shard threads are pinned to dedicated CPU cores (default: true).
# Pinned cores are drawn from the process's allowed CPU set (affinity/cpuset
# mask), so the server cooperates with systemd `AllowedCPUs=` and container
# cpusets. Set to false when the server shares cores with other workloads
# (e.g. a multi-tenant host slicing CPU via cgroup quotas): unpinned shards
# let the kernel scheduler place threads freely instead of piling every
# process onto the same low-numbered cores.
pin_cores = true

# Per-shard inter-shard inbox capacity (the main lane: consensus frames,
# connection setup, reconcile wakes). Bounded by design; consensus-frame
# drops recover via VSR retransmit. Size for the consensus working set:
# ~ the prepare queue depth of the planes the shard hosts ([metadata] on
# shard 0, [partition] elsewhere) times replica_count times directions.
# Both depths are tunable, so raising either raises the capacity needed here.
inbox_capacity = 1024

# Capacity of the reply lane: a separate bounded channel for cross-shard
# client-reply forwards, whose drops are terminal (no in-protocol
# retransmit - the client never receives the reply). Split from
# inbox_capacity so a consensus burst cannot evict reply forwards; size for
# peak client-reply fan-out per shard.
reply_inbox_capacity = 1024

# Wall-clock budget for a single shard's bus drain on shutdown. Drives
# the per-shard watchdog and the parallel-join survivor path; sized
# larger than typical TCP RTT times in-flight write-batch so writers
# receive their full last `write_vectored_all` budget before the
# connection registry force-tears the bus. Slow-fsync hosts may need
# to extend this past the default.
shutdown_drain_timeout = "10 s"

# Poll cadence for the cross-thread shutdown flag and for the
# metadata-handoff loops. Trades off Ctrl-C latency against idle wakeup
# cost; the default keeps shutdown observably prompt without measurable
# scheduler overhead. Must be less than or equal to shutdown_drain_timeout.
shutdown_poll_interval = "50 ms"

# Hard wall-clock deadline for joining shard threads at process exit. A
# shard whose pump or listener wedges past this budget is abandoned with
# an error log instead of blocking exit forever. Must be at least
# shutdown_drain_timeout, or shards would be abandoned mid-drain.
shutdown_join_timeout = "30 s"

# Safety-tick cadence for the partition reconciliation loop. The reconciler
# also wakes on every metadata commit from shard 0, so this only covers
# dropped wake-ups and the initial post-bootstrap convergence window.
reconcile_periodic_interval = "1 s"

# WebSocket listener configuration. The frame-tuning knobs below are the
# live source for the server's WS / WSS plane; they are folded into a
# compio-ws WebSocketConfig once at bus construction. Each size knob is
# optional: commenting it out keeps the compio-ws (tungstenite) default
# noted next to it. A malformed size string fails config load.
[websocket]
enabled = true
address = "127.0.0.1:8092"

# Target minimum size of the frame read buffer. compio-ws default: "128 KiB".
# read_buffer_size = "128 KiB"

# Target buffer size for batched writes before flush. compio-ws
# default: "128 KiB".
# write_buffer_size = "128 KiB"

# Hard ceiling on the write buffer; writes past it error instead of
# buffering, so it must exceed write_buffer_size by at least one message.
# compio-ws default: unlimited.
# max_write_buffer_size = "128 MiB"

# Hard upper bound on a single inbound WebSocket message
# (post-fragment-reassembly). Must not exceed message_bus.max_message_size.
# compio-ws default: "64 MiB".
# max_message_size = "64 MiB"

# Hard upper bound on a single inbound WebSocket frame
# (pre-fragment-reassembly). Must not exceed max_message_size.
# compio-ws default: "16 MiB".
# max_frame_size = "16 MiB"

# Whether to accept unmasked frames from clients in violation of
# RFC 6455 client-to-server framing rules. Strict (false) by default.
accept_unmasked_frames = false

[websocket.tls]
enabled = false

# Certificate source for the WSS listener.
# `true` generates an ephemeral loopback certificate, but only while cert_file
# does not exist on disk: an existing PEM pair is loaded instead (the server
# logs which way it went).
# `false` loads cert_file / key_file, which must both exist.
self_signed = true

# Loaded when the file exists or self_signed = false.
cert_file = "core/certs/iggy_cert.pem"
key_file = "core/certs/iggy_key.pem"

# Metadata consensus plane tunables (shard 0's VSR replica: users,
# streams, topics, sessions). Size these together: a deeper prepare queue
# admits more concurrent in-flight metadata ops (e.g. login storms), and
# the journal must hold enough slots that a forced checkpoint (triggered
# when remaining slots fall to the checkpoint margin, which itself is
# max(64, prepare_queue_depth)) stays rare. Validation enforces
# journal_slots >= 4 * max(64, prepare_queue_depth).
[metadata]
# Depth of the metadata prepare queue: how many uncommitted metadata ops
# may be in flight at once. Submits beyond it are rejected with the
# transient "metadata prepare queue is full" and retried by the SDK.
# Capped at 127 by the view-change wire format: a DoViewChange describes the
# uncommitted suffix with one nack bit and one present bit per entry in a u128 each,
# so a deeper queue produces entries a view change can neither adopt nor prove dead.
prepare_queue_depth = 32

# Size of the metadata WAL's in-memory index, in slots (one committed but
# not-yet-snapshotted op per slot). Larger values buy more headroom
# between forced checkpoints at the cost of memory and bigger WAL
# rewrites per checkpoint. Reducing this for an existing data directory is
# refused at boot if the live WAL suffix collides in the smaller index;
# restore the previous value to recover.
journal_slots = 1024

# Slot count of the VSR client table: how many distinct clients (TCP/QUIC/WS
# virtual clients and HTTP sessions together) hold live session state at once.
# When full, the client whose last commit is oldest is evicted and its next
# request re-registers. The HTTP session cap tracks this at half, so raising
# it lifts both. Must be between 2 and 65536.
clients_table_max = 8192

# Per-partition consensus plane tunables. Unlike [metadata] (one shard-0
# plane), a pipeline exists per partition, so raising this multiplies pinned
# request-buffer memory by the partition count. Keep it modest.
[partition]
# Depth of a partition's prepare queue: how many uncommitted produce /
# consumer-offset ops may be in flight at once for that partition. Submits past
# it spill into a request queue of twice this depth; once both are full the
# server drops the request without a reply and the client retries on its own
# request timeout. Must be > 0 and <= 127: the ceiling is the view-change wire, not
# memory. A DoViewChange describes the uncommitted suffix with one bit per op in a
# u128 bitset, and this depth bounds that suffix.
prepare_queue_depth = 32

# Distinct clients each partition group tracks request watermarks for, so a
# retried produce or consumer-offset write is answered instead of committing a
# second time. At capacity the client whose newest commit is oldest is evicted;
# that client's next replay re-executes, exactly as it would have before dedup
# existed, so under-sizing degrades rather than breaks. Must be > 0 and <= 65536.
#
# Unlike [metadata] clients_table_max, this budget is PER GROUP, so worst-case
# memory scales with partition count. Size it to the producers one partition
# actually sees, not the node's client total.
dedup_clients_max = 4096

# Entries the evicted ring retains per multi-replica partition for journal
# repair after a peer rejoins. Larger widens the window a restarting peer can be
# served from the ring before falling back to bulk sync, at the cost of pinned
# memory per partition. Must be > 0 and <= 65536. Single-replica partitions
# retain nothing regardless.
evicted_ring_capacity = 4096

# Byte ceiling for the evicted ring per partition; whichever ring cap (this or
# evicted_ring_capacity) trips first evicts. Bounds the ring memory a burst of
# large batches can pin. Must be > 0 and <= "256 MiB".
evicted_ring_bytes_max = "16 MiB"

# Byte budget for segment payloads a SERVING shard keeps resident to answer
# state-transfer chunk requests. PER SHARD, and shard count defaults to core
# count, so the process-wide high-water is this times the core count on top of
# page cache -- keep that product in mind before raising it. The default is a
# FIXED 2176 MiB: two sealed segments at the built-in 1 GiB segment ceiling,
# each of which can close one whole message_bus.max_message_size past its
# target, which is why it is not 2 GiB. That ceiling is a compile-time
# constant: no config key moves it, and a per-topic segment_size is refused
# above it. How many groups this shard serves at once is
# floor(this / max(partition.transfer_artifact_bytes_max, 1 GiB + 64 MiB)),
# minimum one. So raising transfer_artifact_bytes_max without raising this
# lowers concurrency and can take it to one, serialising rejoins; boot warns
# when it drops below two.
# Below one segment a single rejoining node thrashes the cache by itself and
# every miss re-reads and re-hashes a whole segment to serve one 256 KiB chunk.
# Running under the budget costs re-reads, not failures.
# Must be > 0 and <= "64 GiB".
transfer_served_cache_bytes_max = "2176 MiB"

# Alloc ceiling for ONE received state-transfer artifact, per shard. The
# receiver holds it resident through verify, walk and staging write, and up to
# four transfers run at once. MUST cover the largest segment any topic may be
# created with (a fixed 1 GiB, since segment_size is a per-topic option now)
# plus message_bus.max_message_size, because a segment may close one whole batch
# past its cap: under that, a legal segment is refused, the whole manifest with
# it, and the partition livelocks re-requesting it from every peer. Boot
# validates the floor.
#
# The shipped value sits EXACTLY at that floor: 1024 MiB + the shipped 64 MiB
# max_message_size. Raising max_message_size alone therefore refuses boot --
# raise this one by the same amount in the same edit (max_message_size itself
# tops out at the frozen 256 MiB recovery ceiling; see its own note). Raising
# this above the floor for headroom also DIVIDES the serving concurrency
# derived from transfer_served_cache_bytes_max above, so raise that in step.
# Must be > 0 and <= "64 GiB".
transfer_artifact_bytes_max = "1088 MiB"

# Message bus configuration.
# Tunables for the inter-shard / inter-replica internal bus that ships
# consensus traffic between replicas and SDK-client traffic between
# shards. These knobs are consensus-liveness-critical (max_batch gates
# throughput under backpressure). Defaults match
# core::message_bus::config::MessageBusConfig::default().

[message_bus]
# Maximum number of BusMessage entries coalesced into a single writev(2)
# call. Hard upper bound: IOV_MAX/2 = 512 on Linux.
max_batch = 256

# Wire-level cap on a single framed message. Boot refuses a value above the
# frozen 256 MiB ceiling: segment recovery derives fixed scan and allocation
# limits from the widest LEGAL record, so batches admitted above the ceiling
# would be refused as implausible on a later boot. Raising it past 64 MiB
# also breaks Go clients, whose frame cap is a hard 64 MiB constant.
max_message_size = "64 MiB"

# Bound on the per-peer mpsc queue. The writer task drains; the
# send_to_* path enqueues.
peer_queue_capacity = 256

# Interval between outbound reconnect attempts to peers with peer_id > self_id.
reconnect_period = "5 s"

# Timeout for per-peer close drain (flush writer, tear down reader)
# before force-cancellation.
close_peer_timeout = "2 s"

# Wall-clock bound on a single stream.shutdown() / ws.close() in the
# safe-shutdown sequence of the TLS-family transports.
close_grace = "2 s"

# Wall-clock bound on a single connection's handshake phase. Threaded
# into compio::time::timeout(handshake_grace, ...) at each accept site
# (TCP-TLS rustls accept, WS HTTP-Upgrade, WSS combined TLS+WS, QUIC
# connecting.await + accept_bi.await) so a slowloris peer cannot pin
# per-conn channels + registry slot + spawned task indefinitely.
handshake_grace = "10 s"