blob: aefaf5624f57ae5a3c0752de8062c191265821aa [file]
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! Runtime tunables for the message bus.
//!
//! Single source of truth for these knobs is the on-disk schema
//! [`configs::server::ServerConfig`]. The bus consumes that
//! schema at construction (see [`crate::IggyMessageBus::with_config`])
//! and converts the schema-typed fields
//! ([`iggy_common::IggyDuration`] / [`iggy_common::IggyByteSize`])
//! into runtime types ([`Duration`] / `usize`) once, so hot paths read
//! fields directly without per-call conversion.
//!
//! The WebSocket frame-layer config the bus consumes lives under the
//! schema's `[websocket]` block (buffer sizes, message / frame
//! ceilings, unmasked-frame acceptance): the bus IS the server's
//! WS / WSS install path, so the listener section carries the frame
//! tuning (see `configs::websocket`).
//! [`From<&ServerConfig> for MessageBusConfig`](MessageBusConfig)
//! folds that section into [`WebSocketConfig`] once at boot.
//!
//! Liveness detection is NOT done via TCP keepalive on the bus: SDK
//! clients manage their own keepalive policy at the application layer,
//! and replica<->replica liveness is observed by VSR heartbeats rather
//! than by `SO_KEEPALIVE`.
//!
//! Neither plane is authenticated at the bus layer: identity and
//! credential checks belong to the caller (`core/server`) via
//! `LOGIN_*` commands. This struct therefore carries no secret /
//! token-source state.
pub use compio::ws::tungstenite::protocol::WebSocketConfig;
use configs::server::ServerConfig;
use std::time::Duration;
/// Pre-converted QUIC transport tuning derived from
/// [`ServerConfig::quic`](configs::quic::QuicConfig).
///
/// Threaded into [`crate::transports::quic::transport_config_from`] at
/// every bind site so the schema's `[quic]` block actually drives
/// `quinn-proto`'s `TransportConfig`. Hot paths read these fields
/// directly without per-bind `IggyDuration` / `IggyByteSize`
/// conversion.
///
/// `keep_alive_interval` and `max_idle_timeout` follow the legacy
/// QUIC server's convention: a zero `Duration` means *disabled* and
/// the corresponding quinn knob is left unset.
///
/// Hardcoded knobs the bus does NOT expose: `max_concurrent_uni_streams = 0`
/// and the CUBIC congestion controller. Both are architectural
/// invariants of the SDK-client plane (no datagram or unidirectional
/// traffic).
#[derive(Debug, Clone)]
pub struct QuicTuning {
/// Maximum number of concurrent bidirectional streams per
/// connection. Each SDK command opens a fresh bidi stream, so
/// this caps how many commands one client connection may have
/// in flight; 1 disables pipelining.
pub max_concurrent_bidi_streams: u32,
/// Initial path MTU advertised to the peer, in bytes.
pub initial_mtu: u16,
/// Send-flow control window per connection, in bytes.
pub send_window: u64,
/// Receive-flow control window per connection, in bytes.
pub receive_window: u32,
/// Receive-flow control window per stream, in bytes. Never above
/// [`Self::receive_window`], and equal to it under the shipped
/// single-stream default. Setting it strictly below is what keeps
/// one unread stream from pinning the whole connection window, so
/// it only buys anything once several streams share a connection.
pub stream_receive_window: u32,
/// Interval between QUIC keep-alive PINGs. `Duration::ZERO`
/// disables keep-alive; the connection then relies entirely on
/// [`Self::max_idle_timeout`] for liveness.
pub keep_alive_interval: Duration,
/// Idle timeout after which quinn closes the connection.
/// `Duration::ZERO` disables the timer (not recommended).
pub max_idle_timeout: Duration,
}
/// Hard upper bound on `max_batch`, in iovecs.
///
/// Linux's `IOV_MAX` is 1024 (`/usr/include/bits/uio_lim.h`). Future WS
/// transports emit one iovec for the header and one for the body, so a
/// batch of N messages costs `2 * N` iovecs; cap `max_batch` at
/// `IOV_MAX / 2 = 512` to keep that worst case below the syscall limit.
/// Bus construction asserts this in [`crate::IggyMessageBus::with_config`];
/// breaching it at boot panics rather than silently delivering writev
/// `EMSGSIZE` errors on every batch.
pub const IOV_MAX_LIMIT: usize = 512;
/// Pre-converted runtime tunables in effect on a `IggyMessageBus`
/// instance.
///
/// Built from a fully-validated [`ServerConfig`] via
/// [`From<&ServerConfig>`] at boot. All fields are runtime-typed
/// (`Duration`, `usize`, `tungstenite::WebSocketConfig`) so hot paths
/// read them directly without `.get_duration()` / `.as_bytes_u64()`
/// conversion.
///
/// Test code that wants to override a single field can use the
/// struct-update syntax:
/// ```ignore
/// let t = MessageBusConfig {
/// peer_queue_capacity: 8,
/// ..MessageBusConfig::default()
/// };
/// ```
#[derive(Debug, Clone)]
pub struct MessageBusConfig {
/// Maximum number of `BusMessage` entries coalesced into a single
/// `writev(2)` call by the writer task. Higher values improve
/// syscall amortization at the cost of tail latency.
pub max_batch: usize,
/// Wire-level cap on a single framed message, in bytes. Read-side
/// validator; undersize or oversize frames are rejected.
pub max_message_size: usize,
/// Read-ahead buffer per plaintext replica link, in bytes. Threaded
/// into `TcpTransportConn::with_replica_read` by the replica
/// installer; zero selects the unbuffered read path. The client
/// plane and the TLS-family transports ignore it.
pub replica_read_buffer_size: usize,
/// Bound on each replica peer's mpsc queue. The writer task drains; the
/// `send_to_*` path enqueues. Too small drops under burst; too
/// large delays backpressure signalling.
pub peer_queue_capacity: usize,
/// Bound on each SDK connection's inbound and outbound queues.
pub client_queue_capacity: usize,
/// Interval between outbound reconnect attempts to peers with
/// `peer_id > self_id`.
pub reconnect_period: Duration,
/// Number of peer replicas this node expects in its mesh
/// (`cluster.nodes.len() - 1`, or 0 with clustering disabled).
/// Used only for mesh-formation observability: an info log fires
/// when the owner table first holds this many peers.
pub mesh_expected_peers: usize,
/// Timeout for per-peer close drain (flush writer, tear down
/// reader) before force-cancellation.
pub close_peer_timeout: Duration,
/// Wall-clock bound on a single `stream.shutdown()` (or `ws.close()`)
/// invocation in the safe-shutdown sequence. Threaded into the
/// single-task close path of the TLS-family transports
/// (`transports::tcp_tls`, `transports::wss`) and consumed inside
/// `compio::time::timeout(close_grace, ...)`.
///
/// Independent of [`Self::close_peer_timeout`] (which bounds the
/// registry-level drain over both reader and writer joins).
pub close_grace: Duration,
/// Wall-clock bound on a single connection's handshake phase: the
/// rustls accept (TCP-TLS), the WS HTTP-Upgrade (plain WS), the
/// combined TLS + WS handshakes (WSS, sharing one budget end-to-end),
/// and the QUIC `connecting.await` + first `accept_bi.await` pair.
/// Threaded into `compio::time::timeout(handshake_grace, ...)` at
/// each handshake site so a slowloris peer cannot pin per-conn
/// channels + registry slot + spawned task indefinitely.
pub handshake_grace: Duration,
/// WebSocket frame-layer tunables (read/write buffer sizes, max
/// frame size, max message size, accept-unmasked-frames flag).
/// Threaded into `compio_ws::accept_async_with_config` on the WS
/// install path and into `WssTransportConn::ws_handshake` for WSS.
/// Built once at boot by `build_ws_config` (see the
/// [`From<&ServerConfig> for MessageBusConfig`](MessageBusConfig) impl below)
/// from the schema's `[websocket]` section, the live frame-tuning
/// source for the server's WS plane.
///
/// The [`WebSocketConfig`] type is re-exported from `compio_ws`'s
/// vendored `tungstenite` so callers do not need a direct dep on
/// `compio_ws` to construct or pattern-match this field.
pub ws_config: WebSocketConfig,
/// QUIC transport tuning, pre-converted from
/// [`ServerConfig::quic`](configs::quic::QuicConfig) at boot.
pub quic: QuicTuning,
}
impl From<&ServerConfig> for MessageBusConfig {
fn from(cfg: &ServerConfig) -> Self {
let bus = &cfg.message_bus;
// Production load goes through `ServerConfig::validate()`, which
// already exercises `bus.validate()`. This debug-assert catches
// direct callers (tests, simulators) that build a `ServerConfig`
// by hand and forget to validate before converting.
debug_assert!(
<configs::message_bus::MessageBusConfig as iggy_common::Validatable<
configs::ConfigurationError,
>>::validate(bus)
.is_ok(),
"MessageBusConfig::from(&ServerConfig) called on an unvalidated bus config",
);
Self {
max_batch: bus.max_batch,
max_message_size: usize::try_from(bus.max_message_size.as_bytes_u64())
.expect("message_bus.max_message_size fits usize on supported targets"),
replica_read_buffer_size: usize::try_from(bus.replica_read_buffer_size.as_bytes_u64())
.expect("message_bus.replica_read_buffer_size fits usize on supported targets"),
peer_queue_capacity: bus.peer_queue_capacity,
client_queue_capacity: bus.client_queue_capacity,
reconnect_period: bus.reconnect_period.get_duration(),
mesh_expected_peers: if cfg.cluster.enabled {
cfg.cluster.nodes.len().saturating_sub(1)
} else {
0
},
close_peer_timeout: bus.close_peer_timeout.get_duration(),
close_grace: bus.close_grace.get_duration(),
handshake_grace: bus.handshake_grace.get_duration(),
ws_config: build_ws_config(&cfg.websocket),
quic: build_quic_tuning(&cfg.quic),
}
}
}
/// Convert the schema's [`configs::quic::QuicConfig`]
/// (`IggyByteSize` / `IggyDuration` typed) into the runtime
/// [`QuicTuning`] (plain integer / `Duration` fields).
///
/// Range invariants (each numeric field fits its target type, MTU at
/// least 1200, `receive_window` within `u32::MAX`, `send_window` within
/// quinn's `VarInt` cap) are enforced by `QuicConfig::validate()`. The
/// `unwrap_or` arms below are still bounded saturations that keep
/// the build unconditionally infallible if a future caller skips
/// validation in dev / test code.
fn build_quic_tuning(quic: &configs::quic::QuicConfig) -> QuicTuning {
QuicTuning {
max_concurrent_bidi_streams: u32::try_from(quic.max_concurrent_bidi_streams)
.unwrap_or(u32::MAX),
initial_mtu: u16::try_from(quic.initial_mtu.as_bytes_u64()).unwrap_or(u16::MAX),
send_window: quic.send_window.as_bytes_u64(),
receive_window: u32::try_from(quic.receive_window.as_bytes_u64()).unwrap_or(u32::MAX),
stream_receive_window: u32::try_from(quic.stream_receive_window.as_bytes_u64())
.unwrap_or(u32::MAX),
keep_alive_interval: quic.keep_alive_interval.get_duration(),
max_idle_timeout: quic.max_idle_timeout.get_duration(),
}
}
impl Default for QuicTuning {
/// Mirrors the `[quic]` defaults in
/// `core/server/config.toml`: 64 MiB send/receive windows, a per-stream
/// window equal to the connection receive window, 30 s idle timeout, 10 s
/// keep-alive, 1200 B initial MTU, one in-flight command per connection.
///
/// Intended for tests and direct callers; production builds
/// derive the field from [`ServerConfig`] so the values stay in
/// lock-step with the on-disk schema.
fn default() -> Self {
Self {
max_concurrent_bidi_streams: 1,
initial_mtu: 1200,
send_window: 64 * 1024 * 1024,
receive_window: 64 * 1024 * 1024,
stream_receive_window: 64 * 1024 * 1024,
keep_alive_interval: Duration::from_secs(10),
max_idle_timeout: Duration::from_secs(30),
}
}
}
/// Fold the schema's `[websocket]` frame-tuning knobs into a single
/// [`tungstenite::WebSocketConfig`].
///
/// The standalone `tungstenite` crate may be a different major version
/// than the one re-exported by `compio_ws`, so the conversion lives in
/// this crate (next to the `compio_ws` dependency) and constructs the
/// config through the re-export to guarantee type compatibility.
///
/// Each `Some` overrides the compio-ws default; `None` keeps it.
/// Conversion to `usize` saturates on platforms where `IggyByteSize`
/// would overflow, but on supported targets `usize` is at least 32
/// bits, so saturation is unreachable in practice.
fn build_ws_config(websocket: &configs::websocket::WebSocketConfig) -> WebSocketConfig {
let mut ws = WebSocketConfig::default();
if let Some(sz) = websocket.read_buffer_size {
ws = ws.read_buffer_size(byte_size_to_usize(sz));
}
if let Some(sz) = websocket.write_buffer_size {
ws = ws.write_buffer_size(byte_size_to_usize(sz));
}
if let Some(sz) = websocket.max_write_buffer_size {
ws = ws.max_write_buffer_size(byte_size_to_usize(sz));
}
if let Some(sz) = websocket.max_message_size {
ws = ws.max_message_size(Some(byte_size_to_usize(sz)));
}
if let Some(sz) = websocket.max_frame_size {
ws = ws.max_frame_size(Some(byte_size_to_usize(sz)));
}
ws.accept_unmasked_frames(websocket.accept_unmasked_frames)
}
#[allow(clippy::cast_possible_truncation)]
fn byte_size_to_usize(sz: iggy_common::IggyByteSize) -> usize {
let bytes = sz.as_bytes_u64();
usize::try_from(bytes).unwrap_or(usize::MAX)
}
impl Default for MessageBusConfig {
fn default() -> Self {
Self::from(&ServerConfig::default())
}
}
#[cfg(test)]
mod tests {
use super::*;
/// `QuicTuning::default()` carries hand-coded literals that must
/// match the schema-derived path through
/// `From<&ServerConfig> for MessageBusConfig`. If the embedded
/// TOML or the literals drift, every test that uses
/// `QuicTuning::default()` (e.g. `quic_client_roundtrip`) silently
/// observes different bytes than production. Pin both sides here.
#[test]
fn quic_tuning_default_matches_schema() {
let schema_quic = MessageBusConfig::from(&ServerConfig::default()).quic;
let literal = QuicTuning::default();
assert_eq!(
schema_quic.max_concurrent_bidi_streams,
literal.max_concurrent_bidi_streams
);
assert_eq!(
schema_quic.initial_mtu, literal.initial_mtu,
"schema initial_mtu {} bytes vs literal {} bytes",
schema_quic.initial_mtu, literal.initial_mtu
);
assert_eq!(schema_quic.send_window, literal.send_window);
assert_eq!(schema_quic.receive_window, literal.receive_window);
assert_eq!(
schema_quic.stream_receive_window,
literal.stream_receive_window
);
assert_eq!(schema_quic.keep_alive_interval, literal.keep_alive_interval);
assert_eq!(schema_quic.max_idle_timeout, literal.max_idle_timeout);
}
}