blob: 3023d3df13a82f0c49d6de6e5bf44fd071a7e3b1 [file]
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
//! The durable VSR state: the consensus numbers a replica must recover from its
//! own disk after a crash, rather than infer from the WAL or relearn from a
//! peer.
//!
//! The payload `journal::superblock` persists. Consensus owns its meaning and
//! byte layout; the superblock adds framing, versioning, and integrity.
//!
//! Fixed little-endian, so it is byte-identical across replicas and stable
//! across restarts. Field order is load-bearing: reorders and width changes are
//! breaking and must ride a superblock version bump.
use std::fmt;
/// Number of bytes [`VsrState::to_bytes`] produces: `cluster`(16) +
/// `replica_id`(1) + `replica_count`(1) + `view`(4) + `log_view`(4) +
/// `commit_max`(8) + `checkpoint_op`(8) + `checkpoint_checksum`(16) +
/// `offset_frontier`(8) + `offset_reserved`(8).
///
/// Growing this is one-way: a record of this length is [`VsrStateError::WrongLength`]
/// to every build that predates the field, so a ROLLBACK needs the data directory
/// wiped even though the upgrade does not. Stated for operators beside
/// `partition.offset_reservation_lease` in `config.toml`.
pub const ENCODED_LEN: usize = 74;
/// The layout before `offset_reserved` was appended.
///
/// [`VsrState::try_from`] accepts records of this length. Without it every
/// superblock already on disk decodes as [`VsrStateError::WrongLength`], which
/// the metadata plane treats as a durability violation and refuses the whole
/// node's boot on.
///
/// "Already on disk" is not a clustering concern. `PingPongSuperblock::open`
/// runs unconditionally in metadata recovery, so EVERY server writes one of
/// these into `metadata/superblock.a`, clustered or not, on every view change
/// and checkpoint. Every release from `server-0.9.0-edge.2` (the first that
/// carries this module at all) through `edge.6` wrote exactly 66 bytes,
/// single-node deployments that never enabled clustering included.
///
/// The 58-byte layout that preceded it is deliberately NOT accepted: it left
/// trunk before any release carried it -- `server-0.8.2-edge.1` has no
/// `vsr_state.rs`, and `edge.2` already wrote 66 -- so a record that short is
/// corruption, not history.
///
/// A version bump instead of this would not help on its own: `classify` compares
/// the version for exact equality, so a v2 build turns every v1 record into
/// `Unreadable`, which is the same refusal wearing a different name -- and it
/// would make this tolerance unreachable, refusing even records that decode.
pub const ENCODED_LEN_WITHOUT_RESERVATION: usize = 66;
/// The durable consensus state of one replica for one consensus group.
///
/// A view a replica acted in must survive a crash, or it can re-participate in
/// an old view and split the log.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct VsrState {
/// Cluster this superblock belongs to. Recovery rejects a mismatch: a copied or
/// misplaced data directory. Catches misplacement only, never staleness -- an
/// older backup of this replica's own directory carries the right identity and
/// passes.
pub cluster: u128,
/// Replica index this superblock belongs to. Recovery rejects a mismatch, with
/// the same misplacement-only scope as `cluster`.
pub replica_id: u8,
/// Cluster size at write time. Recovery rejects a mismatch: quorum size and the
/// `view % replica_count` primary mapping both derive from it, so booting a
/// resized cluster without reconfiguration splits the log.
pub replica_count: u8,
/// Current view.
pub view: u32,
/// Latest view in which this replica changed its head (adopted a `StartView`
/// as a backup, or completed a DVC quorum as primary). `log_view <= view`.
pub log_view: u32,
/// Highest op known committed by the cluster at write time.
///
/// A recovery lower bound and nothing more: recovery reports a gap against what the
/// WAL can prove, but cannot act on one, since closing it needs state transfer.
/// Kept in the durable record because that is what state transfer will read, and
/// adding it later would mean a version bump that invalidates every record.
pub commit_max: u64,
/// Op of the paired checkpoint (the metadata snapshot's sequence number).
pub checkpoint_op: u64,
/// Integrity tag of the paired checkpoint, detecting a torn
/// snapshot/superblock pairing across a crash.
pub checkpoint_checksum: u128,
/// PARTITION plane: the next message offset this replica will mint, or `0`
/// for a group whose offset space is still empty.
///
/// A durable LOWER BOUND, not a completeness claim: boot takes the max of
/// this and whatever the recovered segments prove. It exists because
/// nothing else durably names the frontier once the segments that carried
/// it are gone -- a state-transfer install of an all-GC'd origin, a crash
/// inside the install's swap window, and the fence-and-rebuild path all
/// leave a replica whose counter would otherwise restart at 0 while the
/// group is at N. That is not a lag: replicas re-stamp `base_offset` from
/// this counter and recompute `batch_checksum` over it, so the next
/// replicated prepare would persist different bytes here than on every
/// peer, silently.
///
/// Always `0` on the metadata plane, which mints no message offsets.
pub offset_frontier: u64,
/// PARTITION plane: a monotone CEILING on the offsets this replica may
/// already have minted, claimed ahead of the counter in blocks so an append
/// pays one superblock write per block instead of one per batch.
///
/// Never folded into [`Self::offset_frontier`]. The frontier is a claim
/// about DATA, which state transfer's rewind guard refuses to destroy; a
/// reservation names no bytes, only "an offset up to here may have reached
/// a client". Comparing an offer against it refuses every legitimate offer
/// below the lease headroom, and the replica cycles transfer -> refusal ->
/// backoff forever. Boot seeds the mint counter from both; the rewind guard
/// reads the frontier alone.
///
/// Always `0` on the metadata plane, which mints no message offsets.
pub offset_reserved: u64,
}
impl VsrState {
/// Encode to the fixed little-endian on-disk layout.
#[must_use]
pub fn to_bytes(&self) -> [u8; ENCODED_LEN] {
let mut out = [0u8; ENCODED_LEN];
out[0..16].copy_from_slice(&self.cluster.to_le_bytes());
out[16] = self.replica_id;
out[17] = self.replica_count;
out[18..22].copy_from_slice(&self.view.to_le_bytes());
out[22..26].copy_from_slice(&self.log_view.to_le_bytes());
out[26..34].copy_from_slice(&self.commit_max.to_le_bytes());
out[34..42].copy_from_slice(&self.checkpoint_op.to_le_bytes());
out[42..58].copy_from_slice(&self.checkpoint_checksum.to_le_bytes());
out[58..66].copy_from_slice(&self.offset_frontier.to_le_bytes());
out[66..74].copy_from_slice(&self.offset_reserved.to_le_bytes());
out
}
}
impl TryFrom<&[u8]> for VsrState {
type Error = VsrStateError;
fn try_from(bytes: &[u8]) -> Result<Self, Self::Error> {
// Length-tolerant for ONE legacy layout, the pre-`offset_reserved` record
// every tagged release wrote. The one length check then puts every field
// slice below in bounds by construction, so the `try_into`s cannot fail.
//
// `offset_reserved` is filled from `offset_frontier`, not zeroed. The two
// agree on a record written before the reservation existed: the frontier
// is what that build's data proved, and the write side clamps the
// reservation up to it anyway, so this is the same value a first write
// under this build would record. A 0 would instead claim "nothing
// reserved" for offsets the frontier says exist.
//
// It cannot recover what the old build never wrote down -- offsets acked
// out of RAM above the frontier are gone with the process either way --
// but refusing the record recovers nothing and costs the node its boot.
let mut padded = [0u8; ENCODED_LEN];
match bytes.len() {
ENCODED_LEN => padded.copy_from_slice(bytes),
ENCODED_LEN_WITHOUT_RESERVATION => {
padded[..ENCODED_LEN_WITHOUT_RESERVATION].copy_from_slice(bytes);
padded[ENCODED_LEN_WITHOUT_RESERVATION..ENCODED_LEN]
.copy_from_slice(&bytes[58..66]);
}
actual => {
return Err(VsrStateError::WrongLength {
expected: ENCODED_LEN,
actual,
});
}
}
let bytes = &padded;
let state = Self {
cluster: u128::from_le_bytes(field(bytes, 0)),
replica_id: bytes[16],
replica_count: bytes[17],
view: u32::from_le_bytes(field(bytes, 18)),
log_view: u32::from_le_bytes(field(bytes, 22)),
commit_max: u64::from_le_bytes(field(bytes, 26)),
checkpoint_op: u64::from_le_bytes(field(bytes, 34)),
checkpoint_checksum: u128::from_le_bytes(field(bytes, 42)),
offset_frontier: u64::from_le_bytes(field(bytes, 58)),
offset_reserved: u64::from_le_bytes(field(bytes, 66)),
};
// A record violating `log_view <= view` decodes into a replica that looks
// healthy locally while `DoViewChangeHeader::validate` makes every peer drop
// its DVCs, so it can never conclude a view change. Length validation alone
// would let corruption inside the checksummed region through as a live
// consensus state.
if state.log_view > state.view {
return Err(VsrStateError::LogViewAheadOfView {
view: state.view,
log_view: state.log_view,
});
}
Ok(state)
}
}
/// Copy a fixed-width field out of the length-validated record. Always in bounds
/// for a `[u8; ENCODED_LEN]`, so the slice conversion is infallible.
fn field<const N: usize>(bytes: &[u8; ENCODED_LEN], start: usize) -> [u8; N] {
let mut out = [0u8; N];
out.copy_from_slice(&bytes[start..start + N]);
out
}
/// Failure decoding a [`VsrState`] from bytes.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum VsrStateError {
/// The byte slice was not exactly `ENCODED_LEN` long.
WrongLength { expected: usize, actual: usize },
/// The record violates `log_view <= view`, so it cannot be a state any replica
/// reached.
LogViewAheadOfView { view: u32, log_view: u32 },
}
impl fmt::Display for VsrStateError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::WrongLength { expected, actual } => {
write!(
f,
"VsrState needs {expected} bytes (or {ENCODED_LEN_WITHOUT_RESERVATION}, \
the layout before the offset reservation), got {actual}"
)
}
Self::LogViewAheadOfView { view, log_view } => write!(
f,
"VsrState log_view {log_view} exceeds view {view}, which no replica can reach"
),
}
}
}
impl std::error::Error for VsrStateError {}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn given_distinct_fields_when_to_bytes_should_pin_offsets_and_round_trip() {
// Guards the on-disk layout: a reorder or width change trips this, forcing a
// deliberate superblock version bump. Distinct per-field values make a swap
// between two same-width fields observable.
let state = VsrState {
cluster: 1,
replica_id: 2,
replica_count: 5,
// Distinct per-field values make a swap between two same-width fields
// observable, and `view > log_view` keeps the record decodable.
view: 4,
log_view: 3,
commit_max: 6,
checkpoint_op: 7,
checkpoint_checksum: 8,
offset_frontier: 9,
offset_reserved: 10,
};
let bytes = state.to_bytes();
assert_eq!(bytes.len(), ENCODED_LEN);
assert_eq!(bytes[0], 1, "cluster low byte");
assert_eq!(bytes[16], 2, "replica_id");
assert_eq!(bytes[17], 5, "replica_count");
assert_eq!(bytes[18], 4, "view low byte");
assert_eq!(bytes[22], 3, "log_view low byte");
assert_eq!(bytes[26], 6, "commit_max low byte");
assert_eq!(bytes[34], 7, "checkpoint_op low byte");
assert_eq!(bytes[42], 8, "checkpoint_checksum low byte");
assert_eq!(bytes[58], 9, "offset_frontier low byte");
assert_eq!(bytes[66], 10, "offset_reserved low byte");
assert_eq!(VsrState::try_from(&bytes[..]).unwrap(), state);
assert!(VsrState::try_from(&bytes[..ENCODED_LEN - 1]).is_err());
}
/// A superblock written before `offset_reserved` existed must still decode:
/// every release from `server-0.9.0-edge.2` to `edge.6` wrote that layout on
/// both planes -- the metadata one unconditionally, so single-node
/// deployments that never enabled clustering have one -- and the metadata
/// plane refuses the whole node's boot on a record it cannot decode.
#[test]
fn given_pre_reservation_record_when_decoded_should_fill_from_the_frontier() {
let full = VsrState {
cluster: 3,
replica_id: 1,
replica_count: 3,
view: 9,
log_view: 8,
commit_max: 41,
checkpoint_op: 7,
checkpoint_checksum: 5,
offset_frontier: 77,
offset_reserved: 88,
}
.to_bytes();
let legacy = &full[..ENCODED_LEN_WITHOUT_RESERVATION];
let decoded = VsrState::try_from(legacy).expect("a pre-reservation record must decode");
assert_eq!(decoded.offset_frontier, 77);
assert_eq!(
decoded.offset_reserved, 77,
"the reservation fills from the frontier, not from zero: a 0 would claim \
nothing was reserved for offsets the frontier says exist"
);
assert_eq!(decoded.view, 9);
assert_eq!(decoded.log_view, 8);
assert_eq!(decoded.commit_max, 41);
assert_eq!(decoded.checkpoint_op, 7);
assert_eq!(decoded.checkpoint_checksum, 5);
// Anything that is neither layout is still refused, the 58-byte
// pre-frontier layout included: `server-0.8.2-edge.1` has no
// `vsr_state.rs` and `edge.2` already wrote 66, so no release ever put a
// 58-byte record on a disk and one that short is corruption, not
// history.
for short in [40, 58] {
assert!(
matches!(
VsrState::try_from(&full[..short]),
Err(VsrStateError::WrongLength { .. })
),
"a {short}-byte record must not decode"
);
}
}
/// A zero frontier is the shape a record written before either offset field
/// existed decodes into, and the fill must not turn that into a claim.
#[test]
fn given_pre_reservation_record_with_no_frontier_when_decoded_should_reserve_nothing() {
let full = VsrState {
cluster: 3,
replica_id: 1,
replica_count: 3,
view: 2,
log_view: 2,
commit_max: 0,
checkpoint_op: 0,
checkpoint_checksum: 0,
offset_frontier: 0,
offset_reserved: 0,
}
.to_bytes();
let decoded = VsrState::try_from(&full[..ENCODED_LEN_WITHOUT_RESERVATION])
.expect("a pre-reservation record must decode");
assert_eq!(decoded.offset_frontier, 0);
assert_eq!(decoded.offset_reserved, 0);
}
#[test]
fn given_log_view_past_view_when_decoded_should_reject() {
// Corruption inside the checksummed region can produce a length-valid record
// that violates `log_view <= view`. Decoding it yields a replica that looks
// healthy locally while every peer drops its DoViewChange messages
// (`DoViewChangeHeader::validate`), so it can never conclude a view change.
let mut bytes = VsrState {
cluster: 1,
replica_id: 0,
replica_count: 3,
view: 4,
log_view: 4,
commit_max: 0,
checkpoint_op: 0,
checkpoint_checksum: 0,
// Distinct and nonzero: with 0 here a transposed write over the
// trailing field would still satisfy every assertion below.
offset_frontier: 9,
offset_reserved: 11,
}
.to_bytes();
assert_eq!(bytes[58], 9, "offset_frontier must occupy bytes 58..66");
assert_eq!(bytes[66], 11, "offset_reserved must occupy bytes 66..74");
bytes[22] = 5; // log_view = 5, view stays 4
assert_eq!(
VsrState::try_from(&bytes[..]),
Err(VsrStateError::LogViewAheadOfView {
view: 4,
log_view: 5
})
);
}
}