blob: 9781520e8a86b8dde2be9f8f9232f6ee8d91d211 [file]
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! A deterministic, tick-based timeout mechanism for VSR consensus.
//! This module provides deterministic timeouts driven by logical ticks rather than
//! wall-clock time, enabling:
//! - Deterministic testing
//! - Predictable behavior in consensus
//! - Per-replica PRNG seeding for jitter
//!
//! Two-phase tick model:
//!
//! 1. Phase 1 (tick): All timeouts are advanced by one tick
//! 2. Phase 2 (check & handle): Check which timeouts fired and invoke handlers
/// A deterministic, tick-based timeout.
///
/// Timeouts count down from an initial duration (`after`) and fire when
/// reaching zero. They support exponential backoff with jitter for retries.
use std::time::Duration;
use rand::RngExt;
use rand_xoshiro::Xoshiro256Plus;
use rand_xoshiro::rand_core::SeedableRng;
/// Wall-clock length of one consensus tick.
///
/// Every tick count in this module is a multiple of this unit: a timeout of N
/// ticks fires after N * `TICK_INTERVAL`, provided the owner drives
/// [`TimeoutManager::tick`] at this cadence.
pub const TICK_INTERVAL: Duration = Duration::from_millis(10);
#[derive(Debug, Clone)]
#[allow(unused)]
pub struct Timeout {
pub id: u128,
after: u64,
ticks_remaining: u64,
pub ticking: bool,
pub attempts: u32,
}
impl Timeout {
#[must_use]
pub const fn new(id: u128, after: u64) -> Self {
Self {
id,
after,
ticks_remaining: 0,
ticking: false,
attempts: 0,
}
}
pub const fn start(&mut self) {
self.ticks_remaining = self.after;
self.ticking = true;
self.attempts = 0;
}
pub const fn stop(&mut self) {
self.ticking = false;
self.ticks_remaining = 0;
self.attempts = 0;
}
/// Restart a running timer's interval, clearing any accumulated backoff.
///
/// # Panics
/// In debug builds, if the timer is not ticking. `reset` leaves `ticking` alone,
/// so on a stopped timer it yields an armed-but-dead state that never fires and
/// reads as healthy. Callers that cannot prove the timer is running want
/// [`Self::start`], identical to this on a running one.
pub const fn reset(&mut self) {
debug_assert!(
self.ticking,
"reset on a stopped timeout leaves it armed-but-dead; use start"
);
self.ticks_remaining = self.after;
self.attempts = 0;
}
pub const fn tick(&mut self) {
if self.ticking {
self.ticks_remaining = self.ticks_remaining.saturating_sub(1);
}
}
#[must_use]
pub const fn fired(&self) -> bool {
self.ticking && self.ticks_remaining == 0
}
pub fn backoff(&mut self, prng: &mut Xoshiro256Plus) {
self.attempts = self.attempts.wrapping_add(1);
let max_backoff = self.after.saturating_mul(16);
let shift = self.attempts.min(4);
let backoff = self.after.saturating_mul(1 << shift).min(max_backoff);
let jitter = prng.random_range(0..=backoff);
self.ticks_remaining = backoff.saturating_add(jitter);
}
}
/// Timeout types in VSR.
#[allow(unused)]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum TimeoutKind {
Prepare,
CommitMessage,
NormalHeartbeat,
StartViewChangeMessage,
ViewChangeStatus,
DoViewChangeMessage,
/// Backup re-requesting the current view's `StartView` from its
/// primary: fired while a recovering replica probes for the view.
RequestStartViewMessage,
}
/// Manager for all VSR timeouts of a replica.
#[allow(unused)]
#[derive(Debug)]
pub struct TimeoutManager {
prepare: Timeout,
commit_message: Timeout,
normal_heartbeat: Timeout,
start_view_change_message: Timeout,
do_view_change_message: Timeout,
view_change_status: Timeout,
request_start_view_message: Timeout,
prng: Xoshiro256Plus,
}
#[allow(unused)]
impl TimeoutManager {
// Timeout durations in ticks of [`TICK_INTERVAL`].
/// Public so the runtime can pin its `[cluster] prepare_retransmit_interval`
/// config default against this built-in.
pub const PREPARE_TICKS: u64 = 25;
/// Public so the runtime can pin its `[cluster] commit_broadcast_interval`
/// config default against this built-in.
pub const COMMIT_MESSAGE_TICKS: u64 = 50;
/// Public so the runtime can pin its config default (`[cluster]
/// heartbeat_timeout`) against this built-in with a static assert.
pub const NORMAL_HEARTBEAT_TICKS: u64 = 500;
/// Public so the runtime can pin its `[cluster]
/// view_change_retransmit_interval` config default against this built-in.
/// Deliberately equal to [`Self::DO_VIEW_CHANGE_MESSAGE_TICKS`]: one config
/// knob drives both retransmit timers.
pub const START_VIEW_CHANGE_MESSAGE_TICKS: u64 = 50;
/// Public so the runtime can pin its `[cluster] view_change_status_timeout`
/// config default against this built-in.
pub const VIEW_CHANGE_STATUS_TICKS: u64 = 500;
/// Public so the runtime can pin its `[cluster]
/// view_change_retransmit_interval` config default against this built-in.
/// Deliberately equal to [`Self::START_VIEW_CHANGE_MESSAGE_TICKS`]: one
/// config knob drives both retransmit timers.
pub const DO_VIEW_CHANGE_MESSAGE_TICKS: u64 = 50;
/// Public so the runtime can pin its `[cluster]
/// request_start_view_retransmit_interval` config default against this
/// built-in.
pub const REQUEST_START_VIEW_MESSAGE_TICKS: u64 = 100;
#[must_use]
pub fn new(replica_id: u128) -> Self {
Self {
prepare: Timeout::new(replica_id, Self::PREPARE_TICKS),
commit_message: Timeout::new(replica_id, Self::COMMIT_MESSAGE_TICKS),
normal_heartbeat: Timeout::new(replica_id, Self::NORMAL_HEARTBEAT_TICKS),
start_view_change_message: Timeout::new(
replica_id,
Self::START_VIEW_CHANGE_MESSAGE_TICKS,
),
do_view_change_message: Timeout::new(replica_id, Self::DO_VIEW_CHANGE_MESSAGE_TICKS),
view_change_status: Timeout::new(replica_id, Self::VIEW_CHANGE_STATUS_TICKS),
request_start_view_message: Timeout::new(
replica_id,
Self::REQUEST_START_VIEW_MESSAGE_TICKS,
),
// Upper bits of replica_id are not critical for PRNG seed diversity.
#[allow(clippy::cast_possible_truncation)]
prng: Xoshiro256Plus::seed_from_u64(replica_id as u64),
}
}
/// Override the backup's primary-liveness window (`[cluster]
/// heartbeat_timeout`). Replaces the timeout object, so any countdown
/// already in flight is discarded: call before the replica starts
/// ticking (i.e. before `init` / `init_as_backup`).
pub const fn set_normal_heartbeat_ticks(&mut self, ticks: u64) {
self.normal_heartbeat = Timeout::new(self.normal_heartbeat.id, ticks);
}
/// Override the primary's commit-broadcast interval (`[cluster]
/// commit_broadcast_interval`). Replaces the timeout object, so any
/// countdown already in flight is discarded: call before the replica
/// starts ticking (i.e. before `init`).
pub const fn set_commit_message_ticks(&mut self, ticks: u64) {
self.commit_message = Timeout::new(self.commit_message.id, ticks);
}
/// Override the primary's prepare-retransmit interval (`[cluster]
/// prepare_retransmit_interval`). Replaces the timeout object, so any
/// countdown already in flight is discarded: call before the replica
/// starts ticking (i.e. before `init`).
pub const fn set_prepare_ticks(&mut self, ticks: u64) {
self.prepare = Timeout::new(self.prepare.id, ticks);
}
/// Override the view-change retransmit interval (`[cluster]
/// view_change_retransmit_interval`), in consensus ticks. Drives BOTH the
/// `StartViewChange` and `DoViewChange` retransmit timers - kept equal by
/// design so a view change retransmits both messages at one cadence.
/// Replaces the timeout objects, so any countdown already in flight is
/// discarded: call before the replica starts ticking (i.e. before `init`).
pub const fn set_view_change_retransmit_ticks(&mut self, ticks: u64) {
self.start_view_change_message = Timeout::new(self.start_view_change_message.id, ticks);
self.do_view_change_message = Timeout::new(self.do_view_change_message.id, ticks);
}
/// Override the view-change status backstop (`[cluster]
/// view_change_status_timeout`), in consensus ticks. A view change stalled
/// this long escalates to a fresh cluster-wide election. Replaces the
/// timeout object, so any countdown already in flight is discarded: call
/// before the replica starts ticking (i.e. before `init`).
pub const fn set_view_change_status_ticks(&mut self, ticks: u64) {
self.view_change_status = Timeout::new(self.view_change_status.id, ticks);
}
/// Override the request-start-view retransmit interval (`[cluster]
/// request_start_view_retransmit_interval`), in consensus ticks: how often
/// a recovering or view-change backup re-requests the current view's
/// `StartView`. Replaces the timeout object, so any countdown already in
/// flight is discarded: call before the replica starts ticking (i.e. before
/// `init`).
pub const fn set_request_start_view_ticks(&mut self, ticks: u64) {
self.request_start_view_message = Timeout::new(self.request_start_view_message.id, ticks);
}
/// Tick all timeouts
/// This is the first phase of the two-phase tick-based timeout mechanism.
/// 2nd phase is checking which timeouts have fired and calling the appropriate handlers.
pub const fn tick(&mut self) {
self.prepare.tick();
self.commit_message.tick();
self.normal_heartbeat.tick();
self.start_view_change_message.tick();
self.do_view_change_message.tick();
self.request_start_view_message.tick();
self.view_change_status.tick();
}
#[must_use]
pub const fn fired(&self, kind: TimeoutKind) -> bool {
self.get(kind).fired()
}
#[must_use]
pub const fn get(&self, kind: TimeoutKind) -> &Timeout {
match kind {
TimeoutKind::Prepare => &self.prepare,
TimeoutKind::CommitMessage => &self.commit_message,
TimeoutKind::NormalHeartbeat => &self.normal_heartbeat,
TimeoutKind::StartViewChangeMessage => &self.start_view_change_message,
TimeoutKind::ViewChangeStatus => &self.view_change_status,
TimeoutKind::DoViewChangeMessage => &self.do_view_change_message,
TimeoutKind::RequestStartViewMessage => &self.request_start_view_message,
}
}
pub const fn get_mut(&mut self, kind: TimeoutKind) -> &mut Timeout {
match kind {
TimeoutKind::Prepare => &mut self.prepare,
TimeoutKind::CommitMessage => &mut self.commit_message,
TimeoutKind::NormalHeartbeat => &mut self.normal_heartbeat,
TimeoutKind::StartViewChangeMessage => &mut self.start_view_change_message,
TimeoutKind::ViewChangeStatus => &mut self.view_change_status,
TimeoutKind::DoViewChangeMessage => &mut self.do_view_change_message,
TimeoutKind::RequestStartViewMessage => &mut self.request_start_view_message,
}
}
pub const fn start(&mut self, kind: TimeoutKind) {
self.get_mut(kind).start();
}
pub const fn stop(&mut self, kind: TimeoutKind) {
self.get_mut(kind).stop();
}
pub const fn reset(&mut self, kind: TimeoutKind) {
self.get_mut(kind).reset();
}
pub fn backoff(&mut self, kind: TimeoutKind) {
let timeout = match kind {
TimeoutKind::Prepare => &mut self.prepare,
TimeoutKind::CommitMessage => &mut self.commit_message,
TimeoutKind::NormalHeartbeat => &mut self.normal_heartbeat,
TimeoutKind::StartViewChangeMessage => &mut self.start_view_change_message,
TimeoutKind::ViewChangeStatus => &mut self.view_change_status,
TimeoutKind::DoViewChangeMessage => &mut self.do_view_change_message,
TimeoutKind::RequestStartViewMessage => &mut self.request_start_view_message,
};
timeout.backoff(&mut self.prng);
}
#[must_use]
pub const fn is_ticking(&self, kind: TimeoutKind) -> bool {
self.get(kind).ticking
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_timeout_lifecycle() {
let mut timeout = Timeout::new(0, 10);
assert!(!timeout.ticking);
assert!(!timeout.fired());
timeout.start();
assert!(timeout.ticking);
for _ in 0..9 {
timeout.tick();
assert!(!timeout.fired());
}
timeout.tick();
assert!(timeout.fired());
}
#[test]
fn test_timeout_reset() {
let mut timeout = Timeout::new(0, 10);
timeout.start();
for _ in 0..5 {
timeout.tick();
}
timeout.reset();
assert!(timeout.ticking);
for _ in 0..9 {
timeout.tick();
assert!(!timeout.fired());
}
timeout.tick();
assert!(timeout.fired());
}
#[test]
fn test_timeout_stop() {
let mut timeout = Timeout::new(0, 10);
timeout.start();
for _ in 0..5 {
timeout.tick();
}
timeout.stop();
assert!(!timeout.ticking);
for _ in 0..10 {
timeout.tick();
}
assert!(!timeout.fired());
}
#[test]
fn test_backoff_increases() {
let mut timeout = Timeout::new(0, 10);
let mut prng = Xoshiro256Plus::seed_from_u64(42);
timeout.start();
let initial = timeout.ticks_remaining;
timeout.backoff(&mut prng);
// After backoff, ticks_remaining should be larger than initial with some jitter.
assert!(timeout.ticks_remaining >= initial);
}
}