feat(cluster)!: add per-topic replication factor (#4692) Closes #4663
diff --git a/Cargo.lock b/Cargo.lock index 333ee7a..231d6a0 100644 --- a/Cargo.lock +++ b/Cargo.lock
@@ -7091,7 +7091,7 @@ [[package]] name = "iggy" -version = "0.11.1-edge.6" +version = "0.11.1-edge.7" dependencies = [ "async-broadcast", "async-dropper", @@ -7129,7 +7129,7 @@ [[package]] name = "iggy-bench" -version = "0.6.1-edge.5" +version = "0.6.1-edge.6" dependencies = [ "async-trait", "bench-report", @@ -7188,7 +7188,7 @@ [[package]] name = "iggy-cli" -version = "0.14.1-edge.5" +version = "0.14.1-edge.6" dependencies = [ "anyhow", "apple-native-keyring-store", @@ -7222,7 +7222,7 @@ [[package]] name = "iggy-connectors" -version = "0.5.1-edge.5" +version = "0.5.1-edge.6" dependencies = [ "apache-avro 0.22.0", "async-trait", @@ -7282,7 +7282,7 @@ [[package]] name = "iggy-gateway-kafka" -version = "0.1.0-edge.2" +version = "0.1.0-edge.3" dependencies = [ "arc-swap", "async-trait", @@ -7319,7 +7319,7 @@ [[package]] name = "iggy-mcp" -version = "0.5.1-edge.5" +version = "0.5.1-edge.6" dependencies = [ "axum", "axum-server", @@ -7354,7 +7354,7 @@ [[package]] name = "iggy_binary_protocol" -version = "0.11.1-edge.5" +version = "0.11.1-edge.7" dependencies = [ "aligned-vec", "bytemuck", @@ -7369,7 +7369,7 @@ [[package]] name = "iggy_common" -version = "0.11.1-edge.6" +version = "0.11.1-edge.8" dependencies = [ "aes-gcm 0.11.1", "aes-gcm-siv", @@ -12999,7 +12999,7 @@ [[package]] name = "server" -version = "0.9.1-edge.5" +version = "0.9.1-edge.7" dependencies = [ "ahash", "arc-swap",
diff --git a/Cargo.toml b/Cargo.toml index 6cdab18..a8515ae 100644 --- a/Cargo.toml +++ b/Cargo.toml
@@ -214,11 +214,11 @@ iceberg = "0.10.1" iceberg-catalog-rest = "0.10.1" iceberg-storage-opendal = "0.10.1" -iggy = { path = "core/sdk", version = "0.11.1-edge.6" } -iggy-cli = { path = "core/cli", version = "0.14.1-edge.5" } +iggy = { path = "core/sdk", version = "0.11.1-edge.7" } +iggy-cli = { path = "core/cli", version = "0.14.1-edge.6" } iggy-gateway-kafka = { path = "gateways/kafka" } -iggy_binary_protocol = { path = "core/binary_protocol", version = "0.11.1-edge.5" } -iggy_common = { path = "core/common", version = "0.11.1-edge.6" } +iggy_binary_protocol = { path = "core/binary_protocol", version = "0.11.1-edge.7" } +iggy_common = { path = "core/common", version = "0.11.1-edge.8" } iggy_connector_doris_sink = { path = "core/connectors/sinks/doris_sink" } iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.5.1-edge.2" } iggy_kafka_wire = { path = "core/kafka_wire" }
diff --git a/bdd/go/tests/tcp_test/consumers_feature_create.go b/bdd/go/tests/tcp_test/consumers_feature_create.go index 162e19b..e77c560 100644 --- a/bdd/go/tests/tcp_test/consumers_feature_create.go +++ b/bdd/go/tests/tcp_test/consumers_feature_create.go
@@ -51,7 +51,7 @@ client := createAuthorizedConnection() _, err := client.CreateConsumerGroup( context.Background(), - randomU32Identifier(), + nonExistingStreamIdentifier(client), randomU32Identifier(), createRandomString(16), )
diff --git a/bdd/go/tests/tcp_test/consumers_feature_delete.go b/bdd/go/tests/tcp_test/consumers_feature_delete.go index e3a09f1..c3d1f8b 100644 --- a/bdd/go/tests/tcp_test/consumers_feature_delete.go +++ b/bdd/go/tests/tcp_test/consumers_feature_delete.go
@@ -87,7 +87,7 @@ client := createAuthorizedConnection() err := client.DeleteConsumerGroup( context.Background(), - randomU32Identifier(), + nonExistingStreamIdentifier(client), randomU32Identifier(), randomU32Identifier(), )
diff --git a/bdd/go/tests/tcp_test/consumers_feature_get_by_id.go b/bdd/go/tests/tcp_test/consumers_feature_get_by_id.go index a5f422c..558c2e9 100644 --- a/bdd/go/tests/tcp_test/consumers_feature_get_by_id.go +++ b/bdd/go/tests/tcp_test/consumers_feature_get_by_id.go
@@ -48,7 +48,7 @@ _, err := client.GetConsumerGroup( context.Background(), - randomU32Identifier(), + nonExistingStreamIdentifier(client), randomU32Identifier(), randomU32Identifier(), )
diff --git a/bdd/go/tests/tcp_test/consumers_feature_join.go b/bdd/go/tests/tcp_test/consumers_feature_join.go index 00ad8f2..939022d 100644 --- a/bdd/go/tests/tcp_test/consumers_feature_join.go +++ b/bdd/go/tests/tcp_test/consumers_feature_join.go
@@ -84,7 +84,7 @@ client := createAuthorizedConnection() err := client.JoinConsumerGroup( context.Background(), - randomU32Identifier(), + nonExistingStreamIdentifier(client), randomU32Identifier(), randomU32Identifier(), )
diff --git a/bdd/go/tests/tcp_test/consumers_feature_leave.go b/bdd/go/tests/tcp_test/consumers_feature_leave.go index 0a9997d..85f44c6 100644 --- a/bdd/go/tests/tcp_test/consumers_feature_leave.go +++ b/bdd/go/tests/tcp_test/consumers_feature_leave.go
@@ -87,7 +87,7 @@ err := client.LeaveConsumerGroup( context.Background(), - randomU32Identifier(), + nonExistingStreamIdentifier(client), randomU32Identifier(), randomU32Identifier(), )
diff --git a/bdd/go/tests/tcp_test/messages_feature_send.go b/bdd/go/tests/tcp_test/messages_feature_send.go index 413ca87..defc931 100644 --- a/bdd/go/tests/tcp_test/messages_feature_send.go +++ b/bdd/go/tests/tcp_test/messages_feature_send.go
@@ -68,7 +68,7 @@ messages := createDefaultMessages() _, err := client.SendMessages( context.Background(), - randomU32Identifier(), + nonExistingStreamIdentifier(client), randomU32Identifier(), iggcon.None(), messages,
diff --git a/bdd/go/tests/tcp_test/offset_feature_delete.go b/bdd/go/tests/tcp_test/offset_feature_delete.go index 76c844b..404f7dc 100644 --- a/bdd/go/tests/tcp_test/offset_feature_delete.go +++ b/bdd/go/tests/tcp_test/offset_feature_delete.go
@@ -156,7 +156,7 @@ err := client.DeleteConsumerOffset( context.Background(), consumer, - randomU32Identifier(), + nonExistingStreamIdentifier(client), randomU32Identifier(), &partitionId) itShouldReturnSpecificError(err, ierror.ErrStreamIdNotFound)
diff --git a/bdd/go/tests/tcp_test/offset_feature_deserialize.go b/bdd/go/tests/tcp_test/offset_feature_deserialize.go index 229d200..153d48f 100644 --- a/bdd/go/tests/tcp_test/offset_feature_deserialize.go +++ b/bdd/go/tests/tcp_test/offset_feature_deserialize.go
@@ -182,7 +182,7 @@ _, err := client.GetConsumerOffset( context.Background(), consumer, - randomU32Identifier(), + nonExistingStreamIdentifier(client), randomU32Identifier(), &partitionId, )
diff --git a/bdd/go/tests/tcp_test/partitions_feature_create.go b/bdd/go/tests/tcp_test/partitions_feature_create.go index d989057..4f6f6ad 100644 --- a/bdd/go/tests/tcp_test/partitions_feature_create.go +++ b/bdd/go/tests/tcp_test/partitions_feature_create.go
@@ -51,7 +51,7 @@ client := createAuthorizedConnection() err := client.CreatePartitions( context.Background(), - randomU32Identifier(), + nonExistingStreamIdentifier(client), randomU32Identifier(), 10, )
diff --git a/bdd/go/tests/tcp_test/partitions_feature_delete.go b/bdd/go/tests/tcp_test/partitions_feature_delete.go index 8816e79..bc1eee3 100644 --- a/bdd/go/tests/tcp_test/partitions_feature_delete.go +++ b/bdd/go/tests/tcp_test/partitions_feature_delete.go
@@ -51,7 +51,7 @@ client := createAuthorizedConnection() err := client.DeletePartitions( context.Background(), - randomU32Identifier(), + nonExistingStreamIdentifier(client), randomU32Identifier(), 10, )
diff --git a/bdd/go/tests/tcp_test/segments_feature_delete.go b/bdd/go/tests/tcp_test/segments_feature_delete.go index 79f45ce..d9582ef 100644 --- a/bdd/go/tests/tcp_test/segments_feature_delete.go +++ b/bdd/go/tests/tcp_test/segments_feature_delete.go
@@ -51,7 +51,7 @@ client := createAuthorizedConnection() err := client.DeleteSegments( context.Background(), - randomU32Identifier(), + nonExistingStreamIdentifier(client), randomU32Identifier(), 0, 1,
diff --git a/bdd/go/tests/tcp_test/stream_feature_delete.go b/bdd/go/tests/tcp_test/stream_feature_delete.go index 2acd9e9..83ad49e 100644 --- a/bdd/go/tests/tcp_test/stream_feature_delete.go +++ b/bdd/go/tests/tcp_test/stream_feature_delete.go
@@ -41,7 +41,7 @@ ginkgo.Context("and tries to delete non-existing stream", func() { client := createAuthorizedConnection() - err := client.DeleteStream(context.Background(), randomU32Identifier()) + err := client.DeleteStream(context.Background(), nonExistingStreamIdentifier(client)) itShouldReturnSpecificError(err, ierror.ErrStreamIdNotFound) })
diff --git a/bdd/go/tests/tcp_test/stream_feature_get_by_id.go b/bdd/go/tests/tcp_test/stream_feature_get_by_id.go index f4a001b..696b850 100644 --- a/bdd/go/tests/tcp_test/stream_feature_get_by_id.go +++ b/bdd/go/tests/tcp_test/stream_feature_get_by_id.go
@@ -44,7 +44,7 @@ ginkgo.Context("and tries to get non-existing stream", func() { client := createAuthorizedConnection() - _, err := client.GetStream(context.Background(), randomU32Identifier()) + _, err := client.GetStream(context.Background(), nonExistingStreamIdentifier(client)) itShouldReturnSpecificError(err, ierror.ErrStreamIdNotFound) })
diff --git a/bdd/go/tests/tcp_test/stream_feature_update.go b/bdd/go/tests/tcp_test/stream_feature_update.go index c20b379..6a609a4 100644 --- a/bdd/go/tests/tcp_test/stream_feature_update.go +++ b/bdd/go/tests/tcp_test/stream_feature_update.go
@@ -55,7 +55,7 @@ ginkgo.Context("and tries to update non-existing stream", func() { client := createAuthorizedConnection() name := createRandomString(128) - err := client.UpdateStream(context.Background(), randomU32Identifier(), iggcon.StreamUpdate{Name: &name}) + err := client.UpdateStream(context.Background(), nonExistingStreamIdentifier(client), iggcon.StreamUpdate{Name: &name}) itShouldReturnSpecificError(err, ierror.ErrStreamIdNotFound) })
diff --git a/bdd/go/tests/tcp_test/test_helpers.go b/bdd/go/tests/tcp_test/test_helpers.go index b122b8c..93ec1d7 100644 --- a/bdd/go/tests/tcp_test/test_helpers.go +++ b/bdd/go/tests/tcp_test/test_helpers.go
@@ -19,6 +19,7 @@ import ( "context" + "errors" "math/rand" "strings" "time" @@ -26,6 +27,7 @@ "github.com/apache/iggy/bdd/go/tests/env" "github.com/apache/iggy/foreign/go/client" iggcon "github.com/apache/iggy/foreign/go/contracts" + ierror "github.com/apache/iggy/foreign/go/errors" "github.com/apache/iggy/foreign/go/client/tcp" ) @@ -76,6 +78,20 @@ return id } +// nonExistingStreamIdentifier returns an id that no stream holds. Earlier +// suites leave streams on the shared server, so a random id can resolve and +// turn a stream-not-found answer into a topic-not-found one. +func nonExistingStreamIdentifier(cli iggcon.Client) iggcon.Identifier { + const attempts = 64 + for range attempts { + id := randomU32Identifier() + if _, err := cli.GetStream(context.Background(), id); errors.Is(err, ierror.ErrStreamIdNotFound) { + return id + } + } + panic("no free stream id found") +} + func createRandomString(length int) string { // Define the character set from which to create the random string const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
diff --git a/bdd/go/tests/tcp_test/topic_feature_create.go b/bdd/go/tests/tcp_test/topic_feature_create.go index 07b4beb..116192e 100644 --- a/bdd/go/tests/tcp_test/topic_feature_create.go +++ b/bdd/go/tests/tcp_test/topic_feature_create.go
@@ -49,9 +49,8 @@ ginkgo.Context("and tries to create topic for a non existing stream", func() { client := createAuthorizedConnection() - streamId := createRandomUInt32() name := createRandomString(32) - streamIdentifier, _ := iggcon.NewIdentifier(streamId) + streamIdentifier := nonExistingStreamIdentifier(client) partitions := uint32(2) expiry := iggcon.Millisecond maxTopicSize := uint64(math.MaxUint64)
diff --git a/bdd/go/tests/tcp_test/topic_feature_delete.go b/bdd/go/tests/tcp_test/topic_feature_delete.go index b8be81a..4e65378 100644 --- a/bdd/go/tests/tcp_test/topic_feature_delete.go +++ b/bdd/go/tests/tcp_test/topic_feature_delete.go
@@ -54,7 +54,7 @@ ginkgo.Context("and tries to delete non-existing topic and stream", func() { client := createAuthorizedConnection() - err := client.DeleteTopic(context.Background(), randomU32Identifier(), randomU32Identifier()) + err := client.DeleteTopic(context.Background(), nonExistingStreamIdentifier(client), randomU32Identifier()) itShouldReturnSpecificError(err, ierror.ErrStreamIdNotFound) })
diff --git a/bdd/go/tests/tcp_test/topic_feature_update.go b/bdd/go/tests/tcp_test/topic_feature_update.go index d375d11..9a4e96c 100644 --- a/bdd/go/tests/tcp_test/topic_feature_update.go +++ b/bdd/go/tests/tcp_test/topic_feature_update.go
@@ -72,7 +72,7 @@ maxTopicSize := uint64(math.MaxUint64) err := client.UpdateTopic( context.Background(), - randomU32Identifier(), + nonExistingStreamIdentifier(client), randomU32Identifier(), iggcon.TopicUpdate{Name: &name, MaxTopicSize: &maxTopicSize})
diff --git a/bdd/python/uv.lock b/bdd/python/uv.lock index a9760ae..f9cc365 100644 --- a/bdd/python/uv.lock +++ b/bdd/python/uv.lock
@@ -8,7 +8,7 @@ [[package]] name = "apache-iggy" -version = "0.9.1.dev6" +version = "0.9.1.dev8" source = { directory = "../../foreign/python" } [package.metadata]
diff --git a/core/ai/mcp/Cargo.toml b/core/ai/mcp/Cargo.toml index 100f671..e6b9438 100644 --- a/core/ai/mcp/Cargo.toml +++ b/core/ai/mcp/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy-mcp" -version = "0.5.1-edge.5" +version = "0.5.1-edge.6" description = "MCP Server for Iggy message streaming platform" edition = "2024" license = "Apache-2.0"
diff --git a/core/bench/Cargo.toml b/core/bench/Cargo.toml index 7370975..2417771 100644 --- a/core/bench/Cargo.toml +++ b/core/bench/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy-bench" -version = "0.6.1-edge.5" +version = "0.6.1-edge.6" edition = "2024" license = "Apache-2.0" repository = "https://github.com/apache/iggy"
diff --git a/core/binary_protocol/Cargo.toml b/core/binary_protocol/Cargo.toml index 40a64f4..7c9ce5f 100644 --- a/core/binary_protocol/Cargo.toml +++ b/core/binary_protocol/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy_binary_protocol" -version = "0.11.1-edge.5" +version = "0.11.1-edge.7" description = "Wire protocol types and codec for the Iggy binary protocol. Shared between server and SDK." edition = "2024" rust-version.workspace = true
diff --git a/core/binary_protocol/src/consensus/primary_rebalance.rs b/core/binary_protocol/src/consensus/primary_rebalance.rs index 2af0ee5..e6f59cc 100644 --- a/core/binary_protocol/src/consensus/primary_rebalance.rs +++ b/core/binary_protocol/src/consensus/primary_rebalance.rs
@@ -16,6 +16,7 @@ // under the License. use super::{Command, ConsensusError, ConsensusHeader, HEADER_SIZE, Operation}; +use crate::primitives::partition_status::PartitionStatusEntry; use bytemuck::{CheckedBitPattern, NoUninit}; #[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] @@ -32,6 +33,9 @@ /// Same-release rebalance exchange, always routed through the metadata owner. /// +/// A metadata probe or reply may carry a body of [`PartitionStatusEntry`] +/// records, covered by `checksum_body`; every other kind is header-only. +/// /// `view` identifies the metadata coordinator's view, while `partition_view` /// belongs to the probed partition. A ready reply only supplies evidence to the /// coordinator. A `PrimaryTransfer` from the metadata primary of `view` grants @@ -105,13 +109,20 @@ found: self.command, }); } - if self.size as usize != HEADER_SIZE { + let carries_status = matches!( + self.kind, + PrimaryRebalanceKind::MetadataProbe | PrimaryRebalanceKind::MetadataReply + ); + let body_len = (self.size as usize).checked_sub(HEADER_SIZE); + if !body_len.is_some_and(|len| { + len == 0 || (carries_status && len.is_multiple_of(PartitionStatusEntry::ENCODED_SIZE)) + }) { return Err(ConsensusError::InvalidSize { expected: HEADER_SIZE as u32, found: self.size, }); } - if self.checksum_body != 0 || self.release != 0 { + if (body_len == Some(0) && self.checksum_body != 0) || self.release != 0 { return Err(ConsensusError::InvalidField( "primary rebalance body checksum and release must be zero".into(), )); @@ -221,6 +232,40 @@ } #[test] + fn given_a_status_body_when_validating_should_admit_it_only_on_metadata_exchanges() { + let with_body = |kind, body: usize| { + let mut header = header(kind); + header.size = u32::try_from(HEADER_SIZE + body).unwrap(); + header.checksum_body = u128::from(body != 0); + header + }; + let entry = PartitionStatusEntry::ENCODED_SIZE; + for kind in [ + PrimaryRebalanceKind::MetadataProbe, + PrimaryRebalanceKind::MetadataReply, + ] { + assert!(with_body(kind, 0).validate().is_ok(), "{kind:?}"); + assert!(with_body(kind, 3 * entry).validate().is_ok(), "{kind:?}"); + assert!(with_body(kind, entry + 1).validate().is_err(), "{kind:?}"); + } + for kind in [ + PrimaryRebalanceKind::PartitionProbe, + PrimaryRebalanceKind::PartitionReply, + PrimaryRebalanceKind::SourceProbe, + PrimaryRebalanceKind::SourceReply, + PrimaryRebalanceKind::PrimaryTransfer, + ] { + assert!(with_body(kind, entry).validate().is_err(), "{kind:?}"); + } + let mut empty = header(PrimaryRebalanceKind::MetadataReply); + empty.checksum_body = 1; + assert!(empty.validate().is_err(), "no body, no body checksum"); + let mut short = header(PrimaryRebalanceKind::MetadataReply); + short.size = u32::try_from(HEADER_SIZE - 1).unwrap(); + assert!(short.validate().is_err()); + } + + #[test] fn given_rebalance_phases_when_encoded_should_preserve_the_fixed_layout_and_seal() { assert_eq!(offset_of!(PrimaryRebalanceHeader, nonce), 128); assert_eq!(offset_of!(PrimaryRebalanceHeader, namespace), 144);
diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs index 4705ddc..1c662fc 100644 --- a/core/binary_protocol/src/lib.rs +++ b/core/binary_protocol/src/lib.rs
@@ -109,7 +109,7 @@ }; pub use version::{ ClientVersionInfo, IGGY_PROTOCOL_VERSION, IGGY_PROTOCOL_VERSION_MIN, ProtocolVersion, - is_protocol_compatible, + REPLICA_WIRE_REVISION, is_protocol_compatible, }; /// Maximum number of partitions allowed in a single create/delete request.
diff --git a/core/binary_protocol/src/primitives/mod.rs b/core/binary_protocol/src/primitives/mod.rs index 7a031d9..e152999 100644 --- a/core/binary_protocol/src/primitives/mod.rs +++ b/core/binary_protocol/src/primitives/mod.rs
@@ -24,6 +24,7 @@ pub mod options; pub mod partition_assignment; pub mod partition_history; +pub mod partition_status; pub mod partitioning; pub mod permissions; pub mod polling_strategy;
diff --git a/core/binary_protocol/src/primitives/partition_assignment.rs b/core/binary_protocol/src/primitives/partition_assignment.rs index 646c425..d391a0f 100644 --- a/core/binary_protocol/src/primitives/partition_assignment.rs +++ b/core/binary_protocol/src/primitives/partition_assignment.rs
@@ -16,7 +16,7 @@ // under the License. use crate::WireError; -use crate::codec::{WireDecode, WireEncode, read_u32_le, read_u64_le}; +use crate::codec::{WireDecode, WireEncode, read_u8, read_u32_le, read_u64_le}; use bytes::{BufMut, BytesMut}; #[derive(Debug, Clone, PartialEq, Eq)] @@ -26,6 +26,11 @@ /// Committed placement seed, sealed by the request body checksum and never /// restamped when a later metadata view retransmits the create. pub initial_view: u32, + /// Node ids storing the partition, in member order. Empty means every + /// node, the only layout before replication factors. Encoded by the + /// enclosing request through [`encode_members`], not in + /// [`Self::ENCODED_SIZE`]. + pub members: Vec<u8>, } impl CreatedPartitionAssignment { @@ -33,6 +38,72 @@ pub const ENCODED_SIZE: usize = 16; } +/// Bytes [`encode_members`] writes for `partitions`. +#[must_use] +pub const fn members_encoded_size(partitions: &[CreatedPartitionAssignment]) -> usize { + match partitions.first() { + Some(first) if !first.members.is_empty() => 1 + partitions.len() * first.members.len(), + _ => 0, + } +} + +/// Append the member lists after the fixed assignment records. +/// +/// The group size, then that many node ids per partition. Nothing when the +/// members are empty, which keeps a create from before replication factors +/// byte-identical. +/// +/// # Panics +/// If the partitions do not all carry the same number of members, or more +/// than `u8::MAX`. +pub fn encode_members(partitions: &[CreatedPartitionAssignment], buf: &mut BytesMut) { + let Some(first) = partitions.first().filter(|first| !first.members.is_empty()) else { + return; + }; + let size = u8::try_from(first.members.len()).expect("member count exceeds u8"); + buf.put_u8(size); + for partition in partitions { + assert_eq!( + partition.members.len(), + usize::from(size), + "every partition of one create carries the same member count" + ); + buf.put_slice(&partition.members); + } +} + +/// Read what [`encode_members`] wrote at `offset` into `partitions`, returning +/// the bytes consumed. An empty tail leaves every member list empty. +/// +/// # Errors +/// A zero group size or a section cut short. +pub fn decode_members( + buf: &[u8], + offset: usize, + partitions: &mut [CreatedPartitionAssignment], +) -> Result<usize, WireError> { + if offset >= buf.len() || partitions.is_empty() { + return Ok(0); + } + let size = usize::from(read_u8(buf, offset)?); + if size == 0 { + return Err(WireError::Validation("empty partition member list".into())); + } + let start = offset + 1; + let need = partitions.len() * size; + let section = buf + .get(start..start + need) + .ok_or_else(|| WireError::UnexpectedEof { + offset: start, + need, + have: buf.len().saturating_sub(start), + })?; + for (partition, members) in partitions.iter_mut().zip(section.chunks_exact(size)) { + partition.members = members.to_vec(); + } + Ok(1 + need) +} + impl WireEncode for CreatedPartitionAssignment { fn encoded_size(&self) -> usize { Self::ENCODED_SIZE @@ -60,6 +131,7 @@ partition_id: read_u32_le(buf, 0)?, consensus_group_id: read_u64_le(buf, 4)?, initial_view: read_u32_le(buf, 12)?, + members: Vec::new(), }, Self::ENCODED_SIZE, )) @@ -77,6 +149,7 @@ partition_id: 0x0102_0304, consensus_group_id: 0x0506_0708_090A_0B0C, initial_view: 0x0D0E_0F10, + members: Vec::new(), }; let bytes = request.to_bytes(); let (decoded, consumed) = CreatedPartitionAssignment::decode(&bytes).unwrap();
diff --git a/core/binary_protocol/src/primitives/partition_status.rs b/core/binary_protocol/src/primitives/partition_status.rs new file mode 100644 index 0000000..2121f6e --- /dev/null +++ b/core/binary_protocol/src/primitives/partition_status.rs
@@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::WireError; +use crate::codec::{WireDecode, WireEncode, read_u32_le, read_u64_le}; +use bytes::{BufMut, BytesMut}; + +/// One partition as its primary last reported it: the view it leads, the op +/// it had committed, and its counters. The body of a metadata probe or reply +/// is a run of these. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PartitionStatusEntry { + pub namespace: u64, + pub created_revision: u64, + pub view: u32, + pub commit: u64, + pub current_offset: u64, + pub messages_count: u64, + pub size_bytes: u64, + pub segments_count: u32, + /// Nodes the primary counts in sync, bit `n` for node `n`, itself included. + pub in_sync_nodes: u32, +} + +impl PartitionStatusEntry { + pub const ENCODED_SIZE: usize = 60; +} + +impl WireEncode for PartitionStatusEntry { + fn encoded_size(&self) -> usize { + Self::ENCODED_SIZE + } + + fn encode(&self, buf: &mut BytesMut) { + buf.put_u64_le(self.namespace); + buf.put_u64_le(self.created_revision); + buf.put_u32_le(self.view); + buf.put_u64_le(self.commit); + buf.put_u64_le(self.current_offset); + buf.put_u64_le(self.messages_count); + buf.put_u64_le(self.size_bytes); + buf.put_u32_le(self.segments_count); + buf.put_u32_le(self.in_sync_nodes); + } +} + +impl WireDecode for PartitionStatusEntry { + fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { + if buf.len() < Self::ENCODED_SIZE { + return Err(WireError::UnexpectedEof { + offset: 0, + need: Self::ENCODED_SIZE, + have: buf.len(), + }); + } + Ok(( + Self { + namespace: read_u64_le(buf, 0)?, + created_revision: read_u64_le(buf, 8)?, + view: read_u32_le(buf, 16)?, + commit: read_u64_le(buf, 20)?, + current_offset: read_u64_le(buf, 28)?, + messages_count: read_u64_le(buf, 36)?, + size_bytes: read_u64_le(buf, 44)?, + segments_count: read_u32_le(buf, 52)?, + in_sync_nodes: read_u32_le(buf, 56)?, + }, + Self::ENCODED_SIZE, + )) + } +} + +/// Encode `entries` as one body, in one allocation. +#[must_use] +pub fn encode_status_entries(entries: &[PartitionStatusEntry]) -> bytes::Bytes { + let mut body = BytesMut::with_capacity(entries.len() * PartitionStatusEntry::ENCODED_SIZE); + for entry in entries { + entry.encode(&mut body); + } + body.freeze() +} + +/// Decode a body that is nothing but entries. +/// +/// # Errors +/// A body whose length is not a whole number of entries. +pub fn decode_status_entries(body: &[u8]) -> Result<Vec<PartitionStatusEntry>, WireError> { + if !body + .len() + .is_multiple_of(PartitionStatusEntry::ENCODED_SIZE) + { + return Err(WireError::Validation( + "partition status body is not a whole number of entries".into(), + )); + } + body.as_chunks::<{ PartitionStatusEntry::ENCODED_SIZE }>() + .0 + .iter() + .map(|chunk| PartitionStatusEntry::decode(chunk).map(|(entry, _)| entry)) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn given_entries_when_encoding_should_roundtrip_and_refuse_partial_ones() { + let entries = [ + PartitionStatusEntry { + namespace: 0x0102_0304_0506_0708, + created_revision: 9, + view: 10, + commit: 15, + current_offset: 11, + messages_count: 12, + size_bytes: 13, + segments_count: 14, + in_sync_nodes: 0b101, + }, + PartitionStatusEntry { + namespace: 1, + created_revision: u64::MAX, + view: u32::MAX, + commit: u64::MAX, + current_offset: u64::MAX, + messages_count: 0, + size_bytes: 0, + segments_count: u32::MAX, + in_sync_nodes: u32::MAX, + }, + ]; + let body = encode_status_entries(&entries); + assert_eq!(body.len(), 2 * PartitionStatusEntry::ENCODED_SIZE); + assert_eq!(decode_status_entries(&body).unwrap(), entries); + assert!(decode_status_entries(&body[..body.len() - 1]).is_err()); + assert_eq!(decode_status_entries(&[]).unwrap(), []); + } +}
diff --git a/core/binary_protocol/src/requests/partitions/create_partitions_with_assignments.rs b/core/binary_protocol/src/requests/partitions/create_partitions_with_assignments.rs index d035906..d23c901 100644 --- a/core/binary_protocol/src/requests/partitions/create_partitions_with_assignments.rs +++ b/core/binary_protocol/src/requests/partitions/create_partitions_with_assignments.rs
@@ -17,7 +17,9 @@ use crate::WireError; use crate::codec::{WireDecode, WireEncode, read_u32_le}; -use crate::primitives::partition_assignment::CreatedPartitionAssignment; +use crate::primitives::partition_assignment::{ + CreatedPartitionAssignment, decode_members, encode_members, members_encoded_size, +}; use crate::requests::partitions::CreatePartitionsRequest; use bytes::{BufMut, BytesMut}; @@ -40,6 +42,7 @@ .iter() .map(WireEncode::encoded_size) .sum::<usize>() + + members_encoded_size(&self.partitions) } fn encode(&self, buf: &mut BytesMut) { @@ -55,6 +58,7 @@ for partition in &self.partitions { partition.encode(buf); } + encode_members(&self.partitions, buf); } } @@ -112,6 +116,7 @@ offset += consumed; partitions.push(partition); } + offset += decode_members(buf, offset, &mut partitions)?; Ok(( Self { @@ -144,11 +149,13 @@ initial_view: 9, partition_id: 3, consensus_group_id: 11, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 10, partition_id: 4, consensus_group_id: 12, + members: Vec::new(), }, ], }; @@ -171,6 +178,7 @@ initial_view: 9, partition_id: 3, consensus_group_id: 11, + members: Vec::new(), }], }; let bytes = request.to_bytes();
diff --git a/core/binary_protocol/src/requests/system/finalize_session.rs b/core/binary_protocol/src/requests/system/finalize_session.rs index 8d84d25..e888e28 100644 --- a/core/binary_protocol/src/requests/system/finalize_session.rs +++ b/core/binary_protocol/src/requests/system/finalize_session.rs
@@ -18,25 +18,53 @@ use bytes::{BufMut, BytesMut}; use crate::WireError; -use crate::codec::{WireDecode, WireEncode, read_u64_le}; +use crate::codec::{WireDecode, WireEncode, read_u32_le, read_u64_le}; use crate::requests::system::SessionIdentity; +/// A partition incarnation that could not prove the session retired. Its next +/// primary retires the session before serving session-bound requests. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct RetirementDebt { + pub namespace: u64, + pub created_revision: u64, +} + +impl RetirementDebt { + pub const ENCODED_SIZE: usize = 2 * size_of::<u64>(); +} + /// Retirement evidence covers only the partition set at `namespace_revision`. /// Ordered metadata apply must recheck that revision before reclaiming the session. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +/// +/// `debt` rides as a trailing section that is absent when empty. +#[derive(Debug, Clone, PartialEq, Eq)] pub struct FinalizeSessionRequest { pub identity: SessionIdentity, pub namespace_revision: u64, + pub debt: Vec<RetirementDebt>, } impl WireEncode for FinalizeSessionRequest { fn encoded_size(&self) -> usize { - self.identity.encoded_size() + size_of::<u64>() + let debt = if self.debt.is_empty() { + 0 + } else { + size_of::<u32>() + self.debt.len() * RetirementDebt::ENCODED_SIZE + }; + self.identity.encoded_size() + size_of::<u64>() + debt } fn encode(&self, buf: &mut BytesMut) { self.identity.encode(buf); buf.put_u64_le(self.namespace_revision); + if self.debt.is_empty() { + return; + } + buf.put_u32_le(u32::try_from(self.debt.len()).expect("debt list fits a message")); + for debt in &self.debt { + buf.put_u64_le(debt.namespace); + buf.put_u64_le(debt.created_revision); + } } } @@ -52,14 +80,34 @@ } fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { - let (identity, consumed) = SessionIdentity::decode(buf)?; + let (identity, mut consumed) = SessionIdentity::decode(buf)?; let namespace_revision = read_u64_le(buf, consumed)?; + consumed += size_of::<u64>(); + let mut debt = Vec::new(); + if consumed < buf.len() { + let count = read_u32_le(buf, consumed)? as usize; + consumed += size_of::<u32>(); + if count == 0 || buf.len() - consumed < count * RetirementDebt::ENCODED_SIZE { + return Err(WireError::Validation( + "invalid session finalization debt section".into(), + )); + } + debt.reserve_exact(count); + for _ in 0..count { + debt.push(RetirementDebt { + namespace: read_u64_le(buf, consumed)?, + created_revision: read_u64_le(buf, consumed + size_of::<u64>())?, + }); + consumed += RetirementDebt::ENCODED_SIZE; + } + } Ok(( Self { identity, namespace_revision, + debt, }, - consumed + size_of::<u64>(), + consumed, )) } } @@ -77,6 +125,7 @@ metadata_watermark: 101, }, namespace_revision: 2, + debt: Vec::new(), }; let bytes = request.to_bytes(); assert_eq!( @@ -93,4 +142,50 @@ trailing.push(0); assert!(FinalizeSessionRequest::decode_from(&trailing).is_err()); } + + #[test] + fn given_retirement_debt_when_decoding_should_round_trip_and_refuse_a_damaged_section() { + let identity = SessionIdentity { + client_id: 3, + session: 7, + metadata_watermark: 9, + }; + let without = FinalizeSessionRequest { + identity, + namespace_revision: 4, + debt: Vec::new(), + }; + let request = FinalizeSessionRequest { + debt: vec![ + RetirementDebt { + namespace: 11, + created_revision: 2, + }, + RetirementDebt { + namespace: 12, + created_revision: 3, + }, + ], + ..without + }; + let bytes = request.to_bytes(); + assert_eq!(bytes.len(), request.encoded_size()); + assert_eq!( + bytes.len(), + without.to_bytes().len() + 4 + 2 * RetirementDebt::ENCODED_SIZE + ); + assert_eq!( + FinalizeSessionRequest::decode_from(&bytes).unwrap(), + request + ); + for end in without.to_bytes().len() + 1..bytes.len() { + assert!( + FinalizeSessionRequest::decode_from(&bytes[..end]).is_err(), + "accepted truncation at {end}" + ); + } + let mut empty_section = without.to_bytes().to_vec(); + empty_section.extend_from_slice(&0_u32.to_le_bytes()); + assert!(FinalizeSessionRequest::decode_from(&empty_section).is_err()); + } }
diff --git a/core/binary_protocol/src/requests/system/mod.rs b/core/binary_protocol/src/requests/system/mod.rs index e59a75d..3a26ef9 100644 --- a/core/binary_protocol/src/requests/system/mod.rs +++ b/core/binary_protocol/src/requests/system/mod.rs
@@ -33,7 +33,7 @@ pub use describe_options::{ DescribeOptionsRequest, OPTIONS_SCOPE_STREAM, OPTIONS_SCOPE_TOPIC, OPTIONS_SCOPE_USER, }; -pub use finalize_session::FinalizeSessionRequest; +pub use finalize_session::{FinalizeSessionRequest, RetirementDebt}; pub use get_client::GetClientRequest; pub use get_clients::GetClientsRequest; pub use get_cluster_metadata::GetClusterMetadataRequest;
diff --git a/core/binary_protocol/src/requests/system/retire_sessions.rs b/core/binary_protocol/src/requests/system/retire_sessions.rs index 6ab770a..b69d552 100644 --- a/core/binary_protocol/src/requests/system/retire_sessions.rs +++ b/core/binary_protocol/src/requests/system/retire_sessions.rs
@@ -15,8 +15,9 @@ // specific language governing permissions and limitations // under the License. -use bytes::BytesMut; +use bytes::{BufMut, BytesMut}; +use crate::codec::read_u64_le; use crate::requests::system::SessionIdentity; use crate::{WireDecode, WireEncode, WireError}; @@ -24,27 +25,50 @@ /// Server-originated partition barrier. Every identity is already ended in /// committed metadata. Admission excludes pending writes for the whole batch. +/// +/// A nonzero `paid_through` marks the last batch of a retirement debt payment: +/// every session the metadata op `paid_through` had finalized is retired once +/// it commits. It rides as a trailing `u64`, absent when zero. #[derive(Debug, Clone, PartialEq, Eq)] pub struct RetireSessionsRequest { pub identities: Vec<SessionIdentity>, + pub paid_through: u64, } impl WireEncode for RetireSessionsRequest { fn encoded_size(&self) -> usize { self.identities.len() * SessionIdentity::ENCODED_SIZE + + if self.paid_through == 0 { + 0 + } else { + size_of::<u64>() + } } fn encode(&self, buf: &mut BytesMut) { for identity in &self.identities { identity.encode(buf); } + if self.paid_through != 0 { + buf.put_u64_le(self.paid_through); + } } } impl WireDecode for RetireSessionsRequest { fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { let (chunks, remainder) = buf.as_chunks::<{ SessionIdentity::ENCODED_SIZE }>(); - if chunks.is_empty() || chunks.len() > MAX_SESSIONS_PER_RETIREMENT || !remainder.is_empty() + let paid_through = match remainder.len() { + 0 => 0, + 8 => read_u64_le(remainder, 0)?, + _ => { + return Err(WireError::Validation( + "invalid session retirement batch length".into(), + )); + } + }; + if (paid_through == 0 && (chunks.is_empty() || !remainder.is_empty())) + || chunks.len() > MAX_SESSIONS_PER_RETIREMENT { return Err(WireError::Validation( "invalid session retirement batch length".into(), @@ -65,7 +89,13 @@ Ok(identity) }) .collect::<Result<Vec<_>, _>>()?; - Ok((Self { identities }, buf.len())) + Ok(( + Self { + identities, + paid_through, + }, + buf.len(), + )) } } @@ -83,6 +113,7 @@ for count in [1, MAX_SESSIONS_PER_RETIREMENT] { let request = RetireSessionsRequest { identities: vec![identity; count], + paid_through: 0, }; let bytes = request.to_bytes(); assert_eq!(RetireSessionsRequest::decode_from(&bytes).unwrap(), request); @@ -93,6 +124,7 @@ RetireSessionsRequest::decode_from( &RetireSessionsRequest { identities: vec![identity; MAX_SESSIONS_PER_RETIREMENT + 1], + paid_through: 0, } .to_bytes() ) @@ -108,10 +140,31 @@ ..identity } ], + paid_through: 0, } .to_bytes() ) .is_err() ); } + + #[test] + fn given_a_debt_payment_when_decoding_should_carry_paid_through_even_without_identities() { + let identity = SessionIdentity { + client_id: 1, + session: 2, + metadata_watermark: 3, + }; + for identities in [Vec::new(), vec![identity; MAX_SESSIONS_PER_RETIREMENT]] { + let request = RetireSessionsRequest { + identities, + paid_through: 17, + }; + let bytes = request.to_bytes(); + assert_eq!(bytes.len(), request.encoded_size()); + assert_eq!(RetireSessionsRequest::decode_from(&bytes).unwrap(), request); + assert!(RetireSessionsRequest::decode_from(&bytes[..bytes.len() - 1]).is_err()); + } + assert!(RetireSessionsRequest::decode_from(&0_u64.to_le_bytes()).is_err()); + } }
diff --git a/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs b/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs index a43e06a..0abf052 100644 --- a/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs +++ b/core/binary_protocol/src/requests/topics/create_topic_with_assignments.rs
@@ -18,7 +18,9 @@ use crate::WireError; use crate::codec::{WireDecode, WireEncode, read_u32_le}; use crate::primitives::options::{WireOptions, decode_options_prefixed}; -use crate::primitives::partition_assignment::CreatedPartitionAssignment; +use crate::primitives::partition_assignment::{ + CreatedPartitionAssignment, decode_members, encode_members, members_encoded_size, +}; use crate::requests::topics::CreateTopicRequest; use bytes::{BufMut, BytesMut}; @@ -51,6 +53,7 @@ .iter() .map(WireEncode::encoded_size) .sum::<usize>() + + members_encoded_size(&self.partitions) } fn encode(&self, buf: &mut BytesMut) { @@ -71,6 +74,7 @@ for partition in &self.partitions { partition.encode(buf); } + encode_members(&self.partitions, buf); } } @@ -135,6 +139,7 @@ offset += consumed; partitions.push(partition); } + offset += decode_members(buf, offset, &mut partitions)?; Ok(( Self { @@ -180,11 +185,13 @@ initial_view: 7, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 8, partition_id: 1, consensus_group_id: 2, + members: Vec::new(), }, ], }; @@ -209,6 +216,7 @@ initial_view: 7, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], }; let bytes = request.to_bytes(); @@ -250,6 +258,49 @@ } #[test] + fn given_member_lists_when_encoding_should_roundtrip_and_stay_optional() { + let assignment = |partition_id, members: &[u8]| CreatedPartitionAssignment { + initial_view: 1, + partition_id, + consensus_group_id: u64::from(partition_id) + 1, + members: members.to_vec(), + }; + let mut request = CreateTopicWithAssignmentsRequest { + request: CreateTopicRequest { + stream_id: WireIdentifier::numeric(1), + partitions_count: 2, + name: WireName::new("events").unwrap(), + options: WireOptions::empty(), + }, + derived_options: WireOptions::empty(), + partitions: vec![assignment(0, &[4, 1, 3]), assignment(1, &[1, 3, 0])], + }; + let bytes = request.to_bytes(); + assert_eq!(bytes.len(), request.encoded_size()); + assert_eq!(&bytes[bytes.len() - 7..], &[3, 4, 1, 3, 1, 3, 0]); + assert_eq!( + CreateTopicWithAssignmentsRequest::decode_from(&bytes).unwrap(), + request + ); + for truncated in bytes.len() - 6..bytes.len() { + assert!(matches!( + CreateTopicWithAssignmentsRequest::decode_from(&bytes[..truncated]), + Err(WireError::UnexpectedEof { .. }) + )); + } + + for partition in &mut request.partitions { + partition.members.clear(); + } + let legacy = request.to_bytes(); + assert_eq!(legacy.len(), bytes.len() - 7); + assert_eq!( + CreateTopicWithAssignmentsRequest::decode_from(&legacy).unwrap(), + request + ); + } + + #[test] fn roundtrip_with_empty_blocks() { let request = CreateTopicWithAssignmentsRequest { request: CreateTopicRequest {
diff --git a/core/binary_protocol/src/responses/topics/get_topic.rs b/core/binary_protocol/src/responses/topics/get_topic.rs index 96cd8e8..b936456 100644 --- a/core/binary_protocol/src/responses/topics/get_topic.rs +++ b/core/binary_protocol/src/responses/topics/get_topic.rs
@@ -16,17 +16,19 @@ // under the License. use crate::WireError; -use crate::codec::{WireDecode, WireEncode, read_u32_le, read_u64_le}; +use crate::codec::{WireDecode, WireEncode, read_u8, read_u32_le, read_u64_le}; use crate::primitives::partition_history::PartitionContext; use crate::responses::streams::get_stream::TopicHeader; use bytes::{BufMut, BytesMut}; /// Partition details within a `GetTopic` response. /// -/// Wire format (64 bytes fixed): +/// Wire format, 66 bytes plus one per listed node: /// ```text /// [id:4][created_at:8][segments_count:4][current_offset:8][size_bytes:8][messages_count:8][context:24] +/// [members_count:1][members:members_count][in_sync_count:1][in_sync:in_sync_count] /// ``` +/// Each node list is a `u8` count followed by that many `u8` node ids. #[derive(Debug, Clone, PartialEq, Eq)] pub struct PartitionResponse { pub id: u32, @@ -36,17 +38,23 @@ pub size_bytes: u64, pub messages_count: u64, pub context: PartitionContext, + /// Node ids that store the partition, in member order. + pub members: Vec<u8>, + /// Node ids among `members` that the partition's primary last reported + /// in sync and that are up. Empty until the primary first reports. + pub in_sync_members: Vec<u8>, } impl PartitionResponse { - const FIXED_SIZE: usize = 4 + 8 + 4 + 8 + 8 + 8 + PartitionContext::ENCODED_SIZE; + const COUNTERS_SIZE: usize = 4 + 8 + 4 + 8 + 8 + 8 + PartitionContext::ENCODED_SIZE; + /// Smallest encoding: the counters and two empty node lists. + const FIXED_SIZE: usize = Self::COUNTERS_SIZE + 2; } -// Wire pin: SDK decoders read fixed 64-byte partition records. -const _: () = assert!(PartitionResponse::FIXED_SIZE == 64); +const _: () = assert!(PartitionResponse::COUNTERS_SIZE == 64); impl WireEncode for PartitionResponse { fn encoded_size(&self) -> usize { - Self::FIXED_SIZE + Self::FIXED_SIZE + self.members.len() + self.in_sync_members.len() } fn encode(&self, buf: &mut BytesMut) { @@ -57,9 +65,32 @@ buf.put_u64_le(self.size_bytes); buf.put_u64_le(self.messages_count); self.context.encode(buf); + encode_nodes(buf, &self.members); + encode_nodes(buf, &self.in_sync_members); } } +/// A node list as a `u8` count then the ids. A partition has at most 32 +/// members, so the count always fits. +fn encode_nodes(buf: &mut BytesMut, nodes: &[u8]) { + let listed = &nodes[..nodes.len().min(usize::from(u8::MAX))]; + buf.put_u8(u8::try_from(listed.len()).unwrap_or(u8::MAX)); + buf.put_slice(listed); +} + +fn decode_nodes(buf: &[u8], offset: usize) -> Result<(Vec<u8>, usize), WireError> { + let count = usize::from(read_u8(buf, offset)?); + let start = offset + 1; + let nodes = buf + .get(start..start + count) + .ok_or_else(|| WireError::UnexpectedEof { + offset: start, + need: count, + have: buf.len().saturating_sub(start), + })?; + Ok((nodes.to_vec(), start + count)) +} + impl WireDecode for PartitionResponse { fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { let id = read_u32_le(buf, 0)?; @@ -69,6 +100,8 @@ let size_bytes = read_u64_le(buf, 24)?; let messages_count = read_u64_le(buf, 32)?; let (context, _) = PartitionContext::decode(&buf[40..])?; + let (members, pos) = decode_nodes(buf, Self::COUNTERS_SIZE)?; + let (in_sync_members, pos) = decode_nodes(buf, pos)?; Ok(( Self { @@ -79,8 +112,10 @@ size_bytes, messages_count, context, + members, + in_sync_members, }, - Self::FIXED_SIZE, + pos, )) } } @@ -141,6 +176,7 @@ mod tests { use super::*; use crate::{WireName, WireOptions}; + use std::fmt::Write; fn sample_topic(partitions_count: u32) -> TopicHeader { TopicHeader { @@ -168,6 +204,8 @@ owner_generation: 0, metadata_op: 101, }, + members: vec![1, 3, 4], + in_sync_members: vec![1, 4], } } @@ -175,12 +213,38 @@ fn partition_roundtrip() { let p = sample_partition(1); let bytes = p.to_bytes(); - assert_eq!(bytes.len(), PartitionResponse::FIXED_SIZE); + assert_eq!(bytes.len(), PartitionResponse::FIXED_SIZE + 5); + assert_eq!(&bytes[64..], &[3, 1, 3, 4, 2, 1, 4]); let (decoded, consumed) = PartitionResponse::decode(&bytes).unwrap(); - assert_eq!(consumed, PartitionResponse::FIXED_SIZE); + assert_eq!(consumed, bytes.len()); assert_eq!(decoded, p); } + /// Shared with the SDK decoder tests, so every SDK reads the same bytes. + #[test] + fn given_sample_partition_when_encoding_should_match_the_pinned_bytes() { + let hex = "01000000004cf1238e01000002000000630000000000000000040000000000006400000000000000\ + 17000000000000000000000000000000650000000000000003010304020104"; + let bytes = sample_partition(1).to_bytes(); + let encoded = bytes.iter().fold(String::new(), |mut hex, byte| { + let _ = write!(hex, "{byte:02x}"); + hex + }); + assert_eq!(encoded, hex); + } + + #[test] + fn given_empty_node_lists_when_encoding_should_take_one_count_byte_each() { + let p = PartitionResponse { + members: Vec::new(), + in_sync_members: Vec::new(), + ..sample_partition(1) + }; + let bytes = p.to_bytes(); + assert_eq!(bytes.len(), PartitionResponse::FIXED_SIZE); + assert_eq!(PartitionResponse::decode(&bytes).unwrap(), (p, bytes.len())); + } + #[test] fn partition_truncated_returns_error() { let p = sample_partition(1);
diff --git a/core/binary_protocol/src/version.rs b/core/binary_protocol/src/version.rs index 20e46ce..67f0e0e 100644 --- a/core/binary_protocol/src/version.rs +++ b/core/binary_protocol/src/version.rs
@@ -108,10 +108,21 @@ /// option blocks), adds derived option blocks to stream and user responses and /// a flags byte to each `DescribeOptions` descriptor. /// Version 0.12.1 replaces `WaitForMessages` (107) with `FetchMessages` (109). +/// Version 0.12.2 gives each `GetTopic` partition its `members` and +/// `in_sync_members` node lists, so a partition record is no longer 64 bytes. /// Finalize this number and the minimum during stable release preparation. /// Released servers using protocol 0.11.0 accept every 0.11.x patch, so an /// incompatible release needs a new minor version for those servers to reject it. -pub const IGGY_PROTOCOL_VERSION: u32 = pack_protocol_version(0, 12, 1); +pub const IGGY_PROTOCOL_VERSION: u32 = pack_protocol_version(0, 12, 2); + +/// Revision of what replicas send each other, which clients never see. +/// +/// Peers refuse each other at the replica handshake unless it matches, so bump +/// it whenever a build stops decoding what another build sends. Revision 1 adds +/// partition member lists, partition status bodies on metadata probes and +/// replies, retirement debt on session finalization and the paid-through mark +/// on session retirement. +pub const REPLICA_WIRE_REVISION: u32 = 1; /// Oldest protocol version this build accepts at login. /// @@ -122,8 +133,9 @@ /// Keep this equal to the current protocol unless compatibility with older /// released protocols has been verified. Decide the range per stable release. /// -/// Version 0.12.0 clients can send the retired `WaitForMessages` (107) command. -pub const IGGY_PROTOCOL_VERSION_MIN: u32 = pack_protocol_version(0, 12, 1); +/// Version 0.12.0 clients can send the retired `WaitForMessages` (107) command, +/// and 0.12.1 clients read every `GetTopic` partition as a 64-byte record. +pub const IGGY_PROTOCOL_VERSION_MIN: u32 = pack_protocol_version(0, 12, 2); const _: () = assert!(IGGY_PROTOCOL_VERSION_MIN > 0 && IGGY_PROTOCOL_VERSION_MIN <= IGGY_PROTOCOL_VERSION); @@ -270,8 +282,9 @@ assert!(!is_protocol_compatible(pack_protocol_version(0, 11, 0))); assert!(!is_protocol_compatible(pack_protocol_version(0, 11, 1))); assert!(!is_protocol_compatible(pack_protocol_version(0, 12, 0))); - assert!(is_protocol_compatible(pack_protocol_version(0, 12, 1))); - assert!(!is_protocol_compatible(pack_protocol_version(0, 12, 2))); + assert!(!is_protocol_compatible(pack_protocol_version(0, 12, 1))); + assert!(is_protocol_compatible(pack_protocol_version(0, 12, 2))); + assert!(!is_protocol_compatible(pack_protocol_version(0, 12, 3))); } #[test] @@ -281,8 +294,8 @@ } #[test] - fn min_requires_the_fetch_protocol() { - assert_eq!(IGGY_PROTOCOL_VERSION_MIN, pack_protocol_version(0, 12, 1)); + fn min_requires_the_partition_member_protocol() { + assert_eq!(IGGY_PROTOCOL_VERSION_MIN, pack_protocol_version(0, 12, 2)); } #[test]
diff --git a/core/cli/Cargo.toml b/core/cli/Cargo.toml index 54b011e..88e1cfd 100644 --- a/core/cli/Cargo.toml +++ b/core/cli/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy-cli" -version = "0.14.1-edge.5" +version = "0.14.1-edge.6" edition = "2024" rust-version.workspace = true authors = ["bartosz.ciesla@gmail.com"]
diff --git a/core/cli/src/commands/binary_topics/get_topic.rs b/core/cli/src/commands/binary_topics/get_topic.rs index 717cff1..30d48b2 100644 --- a/core/cli/src/commands/binary_topics/get_topic.rs +++ b/core/cli/src/commands/binary_topics/get_topic.rs
@@ -121,6 +121,8 @@ "Current offset", "Size", "Messages count", + "Members", + "In sync", ]); for partition in &topic.partitions { @@ -131,6 +133,8 @@ partition.current_offset.to_string(), partition.size.to_string(), partition.messages_count.to_string(), + node_list(&partition.members), + node_list(&partition.in_sync_members), ]); } table.add_row(vec!["Partitions", partitions_table.to_string().as_str()]); @@ -158,6 +162,18 @@ table } +/// Node ids as `0,2,4`, or `-` for none. +fn node_list(nodes: &[u8]) -> String { + if nodes.is_empty() { + return "-".to_owned(); + } + nodes + .iter() + .map(u8::to_string) + .collect::<Vec<_>>() + .join(",") +} + #[cfg(test)] mod tests { use super::topic_table; @@ -202,6 +218,8 @@ size: IggyByteSize::from(1230_u64), messages_count: 5, context: PartitionContext::default(), + members: vec![0, 2, 4], + in_sync_members: vec![0, 4], }, Partition { id: 2, @@ -211,6 +229,8 @@ size: IggyByteSize::from(0_u64), messages_count: 13, context: PartitionContext::default(), + members: vec![1], + in_sync_members: Vec::new(), }, ]; let output = topic_table(&topic).to_string(); @@ -220,12 +240,14 @@ .collect::<Vec<_>>(); assert!( rows.contains(&format!( - "| | 9 | 1970-01-01 00:00:00 | 3 | {} | 1.23 KB | 5 |", + "| | 9 | 1970-01-01 00:00:00 | 3 | {} | 1.23 KB | 5 | 0,2,4 | 0,4 |", u64::MAX )), "{output}" ); - assert!(rows.contains(&"| | 2 | 1970-01-01 00:00:01 | 7 | 11 | 0 B | 13 |".to_owned())); + assert!( + rows.contains(&"| | 2 | 1970-01-01 00:00:01 | 7 | 11 | 0 B | 13 | 1 | - |".to_owned()) + ); } #[test]
diff --git a/core/common/Cargo.toml b/core/common/Cargo.toml index ebb201d..e90f9e7 100644 --- a/core/common/Cargo.toml +++ b/core/common/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy_common" -version = "0.11.1-edge.6" +version = "0.11.1-edge.8" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2024" rust-version.workspace = true
diff --git a/core/common/examples/resource_options_fixtures.rs b/core/common/examples/resource_options_fixtures.rs index 9017f82..ddd7d55 100644 --- a/core/common/examples/resource_options_fixtures.rs +++ b/core/common/examples/resource_options_fixtures.rs
@@ -147,7 +147,7 @@ SCOPES .into_iter() .map(|scope| { - let bytes = describe_options_response(scope)?.to_bytes(); + let bytes = describe_options_response(scope, &[])?.to_bytes(); let entries = DescribeOptionsResponse::decode_from(&bytes)? .entries .iter()
diff --git a/core/common/fixtures/resource_options.json b/core/common/fixtures/resource_options.json index 202888f..3911499 100644 --- a/core/common/fixtures/resource_options.json +++ b/core/common/fixtures/resource_options.json
@@ -16,18 +16,18 @@ }, "constants": { "protocol_version": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "protocol_version_min": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "command_codes": { "create_stream": { @@ -343,6 +343,21 @@ "default": "false" }, { + "key": "replication_factor", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": null + }, + { "key": "encryption_key", "scopes": [ "stream" @@ -439,7 +454,7 @@ { "scope": "topic", "request_hex": "01", - "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "response_hex": "0c000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465127265706c69636174696f6e5f666163746f720b00000000007b0100004e6f64657320746861742073746f7265206561636820706172746974696f6e2c20312e2e3d333220616e64206174206d6f73742074686520636c75737465722773206e6f646520636f756e742e204f6d69747465642c20746865207365727665722773205b636c75737465725d2064656661756c745f7265706c69636174696f6e5f666163746f72206170706c6965733a206576657279206e6f646520756e6c65737320616e206f70657261746f72207365742061206e756d6265722e204120777269746520636f6d6d697473206f6e20612071756f72756d206f66207468657365206e6f6465732c20736f2032206b656570732074776f20636f70696573206275742073746f70732077726974696e67207768696c652065697468657220697320646f776e2c20616e642031206b65657073206f6e6520636f70792074686174207265706c696361746564206475726162696c6974792061636b6e6f776c6564676573206265666f72652069742072656163686573206469736b", "entries": [ { "key": "name", @@ -506,6 +521,12 @@ "kind": "bool", "flags": 0, "default": "false" + }, + { + "key": "replication_factor", + "kind": "uint32", + "flags": 0, + "default": null } ] },
diff --git a/core/common/src/consumer_group_client_state.rs b/core/common/src/consumer_group_client_state.rs index ec9dbfd..ba9180b 100644 --- a/core/common/src/consumer_group_client_state.rs +++ b/core/common/src/consumer_group_client_state.rs
@@ -373,6 +373,8 @@ size: Default::default(), messages_count: 0, context: PartitionContext::default(), + members: Vec::new(), + in_sync_members: Vec::new(), }) .collect() }
diff --git a/core/common/src/types/options/catalog.rs b/core/common/src/types/options/catalog.rs index d6edbba..cfe3cf5 100644 --- a/core/common/src/types/options/catalog.rs +++ b/core/common/src/types/options/catalog.rs
@@ -21,8 +21,9 @@ DEFAULT_MAX_TOPIC_SIZE, DEFAULT_MESSAGE_EXPIRY, DEFAULT_MESSAGES_REQUIRED_TO_SAVE, DEFAULT_PREALLOCATE_SEGMENTS, DEFAULT_SEGMENT_SIZE, DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, Durability, MAX_MESSAGES_REQUIRED_TO_SAVE, MAX_PREALLOCATED_TOPIC_BYTES, - MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, MAX_TOPIC_SEGMENT_SIZE, MIN_TOPIC_SEGMENT_SIZE, - OptionValue, OptionsScope, PartitionResizePolicy, ResourceOptions, erase_options, + MAX_REPLICATION_FACTOR, MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, MAX_TOPIC_SEGMENT_SIZE, + MIN_TOPIC_SEGMENT_SIZE, OptionValue, OptionsScope, PartitionResizePolicy, ResourceOptions, + erase_options, }; use crate::http::users::defaults::{MAX_USERNAME_LENGTH, MIN_USERNAME_LENGTH}; use crate::types::compression::compression_algorithm::CompressionAlgorithm; @@ -67,7 +68,7 @@ const USER_ONLY: &[OptionsScope] = &[OptionsScope::User]; /// Every option key, indexed by its `OptionKey` discriminant. -pub static CATALOG: [OptionDescriptor; 19] = [ +pub static CATALOG: [OptionDescriptor; 20] = [ OptionDescriptor { key: OptionKey::Name, scopes: STREAM_AND_TOPIC, @@ -279,6 +280,27 @@ }, }, OptionDescriptor { + key: OptionKey::ReplicationFactor, + scopes: TOPIC_ONLY, + kind: HeaderKind::Uint32, + storage: OptionStorage::Map, + mutability: OptionMutability::CreateOnly, + // A server setting, so the server reports it rather than the catalog. + default: OptionDefault::None, + visibility: OptionVisibility::Public, + critical: false, + description: || { + format!( + "Nodes that store each partition, 1..={MAX_REPLICATION_FACTOR} and at most the \ + cluster's node count. Omitted, the server's [cluster] \ + default_replication_factor applies: every node unless an operator set a \ + number. A write commits on a quorum of these nodes, so 2 keeps two copies but \ + stops writing while either is down, and 1 keeps one copy that replicated \ + durability acknowledges before it reaches disk" + ) + }, + }, + OptionDescriptor { key: OptionKey::EncryptionKey, scopes: STREAM_ONLY, kind: HeaderKind::Raw, @@ -626,7 +648,9 @@ /// 4. `OptionNotUpdatable`: a create-only key in an update's set or reset. /// 5. `OptionNotResettable`: a reset of a column or of a key without default. /// 6. Value errors: a column inside a create block is `InvalidOptionValue`, -/// column values keep their own errors, other keys `InvalidOptionValue`. +/// column values keep their own errors, other keys `InvalidOptionValue`, +/// except a well-formed `replication_factor` out of range, which is +/// `InvalidReplicationFactor`. /// 7. `InvalidCommand`: an update that changes nothing once its zero /// sentinels are dropped. /// 8. `OptionsBlockTooLarge`: the resulting map exceeds the block limits. @@ -780,7 +804,9 @@ Durability::decode(key, entry)?.encode()? } OptionKey::PartitionResizePolicy => PartitionResizePolicy::decode(key, entry)?.encode()?, - OptionKey::MessagesRequiredToSave => u32::decode(key, entry)?.encode()?, + OptionKey::MessagesRequiredToSave | OptionKey::ReplicationFactor => { + u32::decode(key, entry)?.encode()? + } OptionKey::PreallocateSegments => bool::decode(key, entry)?.encode()?, OptionKey::EncryptionKey => { HeaderValue::try_from(stream_key_from_entry(entry)?.as_bytes().as_slice())? @@ -1143,6 +1169,10 @@ topic_option_keys::PREALLOCATE_SEGMENTS, OptionKey::PreallocateSegments, ), + ( + topic_option_keys::REPLICATION_FACTOR, + OptionKey::ReplicationFactor, + ), ] { assert_eq!(constant, key.to_string()); } @@ -1268,6 +1298,40 @@ } #[test] + fn given_replication_factor_when_decoding_should_accept_only_supported_group_sizes() { + let key = OptionKey::ReplicationFactor; + for (input, expected) in [ + (HeaderValue::from(1u32), 1u32), + (HeaderValue::try_from("3").unwrap(), 3), + ( + HeaderValue::from(MAX_REPLICATION_FACTOR), + MAX_REPLICATION_FACTOR, + ), + ] { + assert_eq!( + canonical_value(key, &entry(key, &input)), + Ok(Some(HeaderValue::from(expected))), + "{input:?}" + ); + } + for input in [ + HeaderValue::from(0u32), + HeaderValue::from(MAX_REPLICATION_FACTOR + 1), + HeaderValue::try_from("0").unwrap(), + ] { + assert_eq!( + canonical_value(key, &entry(key, &input)), + Err(IggyError::InvalidReplicationFactor), + "{input:?}" + ); + } + assert_eq!( + canonical_value(key, &entry(key, &HeaderValue::try_from("three").unwrap())), + Err(IggyError::InvalidOptionValue(key.to_string())) + ); + } + + #[test] fn given_create_only_option_when_resetting_should_reject_not_updatable() { for key in [OptionKey::SegmentSize, OptionKey::Durability] { assert_eq!(
diff --git a/core/common/src/types/options/codec.rs b/core/common/src/types/options/codec.rs index 897dd68..24b2180 100644 --- a/core/common/src/types/options/codec.rs +++ b/core/common/src/types/options/codec.rs
@@ -17,8 +17,8 @@ use super::key::OptionKey; use super::{ - Durability, MAX_MESSAGES_REQUIRED_TO_SAVE, MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, - PartitionResizePolicy, + Durability, MAX_MESSAGES_REQUIRED_TO_SAVE, MAX_REPLICATION_FACTOR, + MAX_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE, PartitionResizePolicy, }; use crate::types::compression::compression_algorithm::CompressionAlgorithm; use crate::types::message::{HeaderKind, HeaderValue}; @@ -218,6 +218,14 @@ { return Err(invalid(key)); } + if key == OptionKey::ReplicationFactor && !(1..=MAX_REPLICATION_FACTOR).contains(&value) { + tracing::debug!( + replication_factor = value, + max = MAX_REPLICATION_FACTOR, + "refusing a replication factor outside the supported range" + ); + return Err(IggyError::InvalidReplicationFactor); + } Ok(value) }
diff --git a/core/common/src/types/options/key.rs b/core/common/src/types/options/key.rs index fe99018..be981d4 100644 --- a/core/common/src/types/options/key.rs +++ b/core/common/src/types/options/key.rs
@@ -52,6 +52,7 @@ MessagesRequiredToSave, SizeOfMessagesRequiredToSave, PreallocateSegments, + ReplicationFactor, EncryptionKey, EncryptionKeyWrapped, EncryptionKeySource,
diff --git a/core/common/src/types/options/mod.rs b/core/common/src/types/options/mod.rs index 284dd33..6a50f9b 100644 --- a/core/common/src/types/options/mod.rs +++ b/core/common/src/types/options/mod.rs
@@ -340,6 +340,9 @@ /// [`SEGMENT_SIZE`] -- preallocation reserves exactly that much, so /// the two only make sense decided together. pub const PREALLOCATE_SEGMENTS: &str = "preallocate_segments"; + /// Nodes that store each partition: `Uint32`, create-only. Bounded by + /// [`super::MAX_REPLICATION_FACTOR`] and the cluster's node count. + pub const REPLICATION_FACTOR: &str = "replication_factor"; } /// Values an absent topic option resolves to at admission. @@ -366,6 +369,9 @@ /// 1 MiB (was `[partition] size_of_messages_required_to_save`). pub const DEFAULT_SIZE_OF_MESSAGES_REQUIRED_TO_SAVE: u64 = 1024 * 1024; +/// Largest partition group consensus supports. +pub const MAX_REPLICATION_FACTOR: u32 = 32; + /// Ceiling for `size_of_messages_required_to_save`. /// /// A threshold above the largest a segment may be can never trip, so every @@ -553,8 +559,9 @@ #[serde(default)] pub critical: bool, /// The key's default in `kind`'s encoding; empty when the key has no - /// default. A build constant rather than a per-node value: these defaults - /// stopped being config-derived when the `[system.*]` keys became options. + /// default. A build constant for every key but `replication_factor`, which + /// reports what the answering node's `[cluster] default_replication_factor` + /// resolves to. #[serde(default)] pub default_value: Vec<u8>, /// Human-readable description. @@ -1194,6 +1201,10 @@ /// Whether this topic's segments reserve their bytes on open; `None` /// resolves against [`DEFAULT_PREALLOCATE_SEGMENTS`]. pub preallocate_segments: Option<bool>, + /// Nodes that store each partition; `None` resolves at admission to the + /// server's `[cluster] default_replication_factor`, every node unless an + /// operator set a number. + pub replication_factor: Option<u32>, /// String-valued keys with no typed field above, parsed server-side /// through each key's `FromStr`. Lets a client reach a key added to the /// server catalog after this build shipped. Outbound only: a typed field @@ -1311,6 +1322,9 @@ OptionKey::PreallocateSegments => { self.preallocate_segments = Some(bool::decode(option, entry)?); } + OptionKey::ReplicationFactor => { + self.replication_factor = Some(u32::decode(option, entry)?); + } OptionKey::Name | OptionKey::Username | OptionKey::Status @@ -1354,6 +1368,7 @@ .size_of_messages_required_to_save .or(defaults.size_of_messages_required_to_save), preallocate_segments: self.preallocate_segments.or(defaults.preallocate_segments), + replication_factor: self.replication_factor.or(defaults.replication_factor), raw: BTreeMap::new(), } } @@ -1461,6 +1476,13 @@ OptionValue::explicit(HeaderValue::from(preallocate_segments)), ); } + if let Some(replication_factor) = self.replication_factor { + options.insert( + HeaderKey::from_str(topic_option_keys::REPLICATION_FACTOR) + .expect("catalog key is a valid header key"), + OptionValue::explicit(HeaderValue::from(replication_factor)), + ); + } Ok(options) } @@ -1648,10 +1670,13 @@ && descriptor.visibility == OptionVisibility::Public }) { let key = HeaderKey::from(descriptor.key); - let default = descriptor - .default_value() - .expect("default encodes") - .expect("every topic map key has a default"); + // A server setting decides the replication factor's default, so the + // catalog derives none and admission stores the resolved value. + let Some(default) = descriptor.default_value().expect("default encodes") else { + assert_eq!(descriptor.key, OptionKey::ReplicationFactor); + assert!(!derived.contains_key(&key)); + continue; + }; let stored = ResourceOptions::from([(key.clone(), OptionValue::explicit(default.clone()))]); let block = @@ -1732,6 +1757,7 @@ messages_required_to_save: Some(500), size_of_messages_required_to_save: Some(IggyByteSize::from(2_097_152u64)), preallocate_segments: Some(false), + replication_factor: Some(2), partitions_count: None, consumer_offset_durability: Durability::Replicated, partition_resize_policy: Some(PartitionResizePolicy::Fixed),
diff --git a/core/common/src/types/partition/mod.rs b/core/common/src/types/partition/mod.rs index b00e77a..4b8e964 100644 --- a/core/common/src/types/partition/mod.rs +++ b/core/common/src/types/partition/mod.rs
@@ -30,6 +30,8 @@ /// - `size_bytes`: the size of the partition in bytes. /// - `messages_count`: the number of messages in the partition. /// - `context`: the partition incarnation and the metadata operation the details were read at. +/// - `members`: the nodes that store the partition. +/// - `in_sync_members`: the members its primary last reported in sync. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Partition { /// Unique identifier of the partition. @@ -47,4 +49,13 @@ /// The partition incarnation and the metadata operation the details were read at. The /// owner generation is zero because the details belong to no consumer group. pub context: PartitionContext, + /// Node ids that store the partition, in member order. While one is down, a write + /// still commits if a quorum of the others is in sync. + #[serde(default)] + pub members: Vec<u8>, + /// Node ids among `members` that the partition's primary last reported in sync and + /// that are up. Empty until the primary first reports, and while no member that is + /// up can lead it. + #[serde(default)] + pub in_sync_members: Vec<u8>, }
diff --git a/core/common/src/types/streaming_stats.rs b/core/common/src/types/streaming_stats.rs index a98e87a..c642434 100644 --- a/core/common/src/types/streaming_stats.rs +++ b/core/common/src/types/streaming_stats.rs
@@ -415,6 +415,36 @@ self.segments_count.load(Ordering::Relaxed) } + /// Move every counter to the values another node reported, carrying the + /// difference into the parents. + pub fn store_reported( + &self, + size_bytes: u64, + messages_count: u64, + segments_count: u32, + current_offset: u64, + ) { + let size = self.size_bytes.load(Ordering::Relaxed); + if size_bytes >= size { + self.increment_size_bytes(size_bytes - size); + } else { + self.decrement_size_bytes(size - size_bytes); + } + let messages = self.messages_count.load(Ordering::Relaxed); + if messages_count >= messages { + self.increment_messages_count(messages_count - messages); + } else { + self.decrement_messages_count(messages - messages_count); + } + let segments = self.segments_count.load(Ordering::Relaxed); + if segments_count >= segments { + self.increment_segments_count(segments_count - segments); + } else { + self.decrement_segments_count(segments - segments_count); + } + self.set_current_offset(current_offset); + } + pub fn current_offset(&self) -> u64 { self.current_offset.load(Ordering::Relaxed) }
diff --git a/core/common/src/wire_conversions.rs b/core/common/src/wire_conversions.rs index 1932229..965fc47 100644 --- a/core/common/src/wire_conversions.rs +++ b/core/common/src/wire_conversions.rs
@@ -27,10 +27,10 @@ ConsumerGroupInfo, ConsumerGroupMember, ConsumerOffsetInfo, GlobalPermissions, HeaderKey, HeaderKind, HeaderValue, IdKind, Identifier, IdentityInfo, IggyByteSize, IggyError, OPTION_FLAG_CRITICAL, OPTION_FLAG_READ_ONLY, OPTION_FLAG_RESETTABLE, OPTION_FLAG_SECRET, - OPTION_FLAG_UPDATABLE, OptionSpec, OptionValue, OptionsProvenance, OptionsScope, Partition, - Permissions, PersonalAccessTokenInfo, RawPersonalAccessToken, ResourceOptions, Stats, Stream, - StreamDetails, StreamPermissions, Topic, TopicDetails, TopicPermissions, TransportEndpoints, - UserInfo, UserInfoDetails, UserStatus, descriptors_for, + OPTION_FLAG_UPDATABLE, OptionKey, OptionSpec, OptionValue, OptionsProvenance, OptionsScope, + Partition, Permissions, PersonalAccessTokenInfo, RawPersonalAccessToken, ResourceOptions, + Stats, Stream, StreamDetails, StreamPermissions, Topic, TopicDetails, TopicPermissions, + TransportEndpoints, UserInfo, UserInfoDetails, UserStatus, descriptors_for, }; use bytes::{BufMut, BytesMut}; use iggy_binary_protocol::primitives::options::{MAX_OPTIONS, MAX_OPTIONS_BYTES, WireOptions}; @@ -157,6 +157,8 @@ size: IggyByteSize::from(w.size_bytes), messages_count: w.messages_count, context: w.context, + members: w.members, + in_sync_members: w.in_sync_members, } } } @@ -871,7 +873,8 @@ /// The `DescribeOptions` response for `scope`: every catalog entry a client /// may see, with its kind, flags, default and description. Internal keys stay -/// out. +/// out. `server_defaults` fills in the defaults a server setting decides, +/// such as `replication_factor`. /// /// # Errors /// @@ -879,17 +882,22 @@ /// catalog tests rule out. pub fn describe_options_response( scope: OptionsScope, + server_defaults: &[(OptionKey, HeaderValue)], ) -> Result<iggy_binary_protocol::responses::system::DescribeOptionsResponse, IggyError> { let entries = descriptors_for(scope) .filter(|descriptor| descriptor.is_described()) .map(|descriptor| { + let server_default = server_defaults + .iter() + .find(|(key, _)| *key == descriptor.key) + .map(|(_, value)| value.clone()); Ok(iggy_binary_protocol::responses::system::OptionDescriptor { key: iggy_binary_protocol::WireName::new(descriptor.key.to_string()) .map_err(|_| IggyError::InvalidFormat)?, kind: descriptor.kind.as_code(), flags: descriptor.flags(), - default_value: descriptor - .default_value()? + default_value: server_default + .map_or_else(|| descriptor.default_value(), |value| Ok(Some(value)))? .map(|value| bytes::Bytes::copy_from_slice(value.as_bytes())) .unwrap_or_default(), description: (descriptor.description)(),
diff --git a/core/common/tests/resource_options_fixtures.rs b/core/common/tests/resource_options_fixtures.rs index dcb6fbc..2089f4b 100644 --- a/core/common/tests/resource_options_fixtures.rs +++ b/core/common/tests/resource_options_fixtures.rs
@@ -65,9 +65,9 @@ ); assert_eq!( constants.protocol_version.packed, - pack_protocol_version(0, 12, 1) + pack_protocol_version(0, 12, 2) ); - assert_eq!(constants.protocol_version.text, "0.12.1"); + assert_eq!(constants.protocol_version.text, "0.12.2"); assert_eq!( constants.command_codes["update_resource"], CommandCode { @@ -189,7 +189,7 @@ .expect("entries decode"); assert_eq!(entries, vector.entries, "{}", vector.scope); - let rebuilt = describe_options_response(vector.scope).expect("server response builds"); + let rebuilt = describe_options_response(vector.scope, &[]).expect("server response builds"); assert_eq!( rebuilt.to_bytes().as_ref(), bytes.as_slice(),
diff --git a/core/configs/src/server_config/cluster.rs b/core/configs/src/server_config/cluster.rs index 63020d9..bf0c2cf 100644 --- a/core/configs/src/server_config/cluster.rs +++ b/core/configs/src/server_config/cluster.rs
@@ -170,6 +170,16 @@ SERVER_CONFIG.cluster.repair_retry_interval.parse().unwrap() } +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server/config.toml` like every other default. +fn default_replication_factor() -> ReplicationFactorDefault { + SERVER_CONFIG + .cluster + .default_replication_factor + .parse() + .unwrap() +} + fn default_repair_gap_debounce_interval() -> IggyDuration { SERVER_CONFIG .cluster @@ -321,6 +331,13 @@ #[serde_as(as = "DisplayFromStr")] #[config_env(leaf)] pub superblock_wedged_fatal_timeout: IggyDuration, + /// Replication factor of a topic created without one. The metadata primary + /// resolves it at admission and the topic stores the result, so nodes + /// configured differently only differ in what a create admitted through + /// them gets. + #[serde(default = "default_replication_factor")] + #[config_env(leaf)] + pub default_replication_factor: ReplicationFactorDefault, /// Full roster of cluster members. Intended to be byte-identical across /// every node so operators ship one config. The running node's identity /// is supplied out-of-band via the `--replica-id` CLI flag, which @@ -491,6 +508,11 @@ /// `nodes.len()`. Validated by [`ClusterConfig::validate`]. pub replica_id: u8, pub ports: TransportPorts, + /// Failure domain, such as a rack or an availability zone. Placement puts + /// a partition's copies in as many as it can. A node without one shares a + /// domain with no other node. + #[serde(default)] + pub rack: Option<String>, } /// One client-network-scoped advertised address: clients whose IP falls @@ -806,6 +828,128 @@ } } +/// What a topic created without a replication factor gets. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReplicationFactorDefault { + /// Every node, the layout from before per-topic replication factors. + AllNodes, + /// This many nodes, lowered to the node count when larger. + Nodes(u8), +} + +impl ReplicationFactorDefault { + /// Upper bound of [`Self::Nodes`], the most members a partition can have. + pub const MAX: u8 = 32; + const ALL_NODES: &'static str = "all"; + + /// The configured number of nodes, `None` for every node. + #[must_use] + pub const fn factor(self) -> Option<u8> { + match self { + Self::AllNodes => None, + Self::Nodes(factor) => Some(factor), + } + } + + fn from_number(value: u64) -> Result<Self, ReplicationFactorDefaultError> { + u8::try_from(value) + .ok() + .filter(|factor| (1..=Self::MAX).contains(factor)) + .map(Self::Nodes) + .ok_or_else(|| ReplicationFactorDefaultError { + value: value.to_string(), + }) + } +} + +impl FromStr for ReplicationFactorDefault { + type Err = ReplicationFactorDefaultError; + + fn from_str(value: &str) -> Result<Self, Self::Err> { + if value.eq_ignore_ascii_case(Self::ALL_NODES) { + return Ok(Self::AllNodes); + } + value + .parse::<u64>() + .map_err(|_| ReplicationFactorDefaultError { + value: value.to_owned(), + }) + .and_then(Self::from_number) + } +} + +impl Serialize for ReplicationFactorDefault { + fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> { + serializer.collect_str(self) + } +} + +/// Accepts a TOML integer as well as a string, because the environment +/// provider turns a numeric `IGGY_CLUSTER_DEFAULT_REPLICATION_FACTOR` into one. +impl<'de> Deserialize<'de> for ReplicationFactorDefault { + fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> { + deserializer.deserialize_any(ReplicationFactorDefaultVisitor) + } +} + +struct ReplicationFactorDefaultVisitor; + +impl serde::de::Visitor<'_> for ReplicationFactorDefaultVisitor { + type Value = ReplicationFactorDefault; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "\"{}\" or an integer from 1 to {}", + ReplicationFactorDefault::ALL_NODES, + ReplicationFactorDefault::MAX + ) + } + + fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<Self::Value, E> { + value.parse().map_err(E::custom) + } + + fn visit_u64<E: serde::de::Error>(self, value: u64) -> Result<Self::Value, E> { + ReplicationFactorDefault::from_number(value).map_err(E::custom) + } + + fn visit_i64<E: serde::de::Error>(self, value: i64) -> Result<Self::Value, E> { + u64::try_from(value) + .map_err(|_| E::custom(format!("invalid default_replication_factor {value}"))) + .and_then(|value| self.visit_u64(value)) + } +} + +impl fmt::Display for ReplicationFactorDefault { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AllNodes => formatter.write_str(Self::ALL_NODES), + Self::Nodes(factor) => write!(formatter, "{factor}"), + } + } +} + +/// A `default_replication_factor` that is neither `all` nor 1 to 32. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReplicationFactorDefaultError { + value: String, +} + +impl fmt::Display for ReplicationFactorDefaultError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "invalid default_replication_factor {:?}: expected \"{}\" or 1 to {}", + self.value, + ReplicationFactorDefault::ALL_NODES, + ReplicationFactorDefault::MAX + ) + } +} + +impl std::error::Error for ReplicationFactorDefaultError {} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum AdvertisedAddressError { Empty, @@ -1093,6 +1237,28 @@ ConfigurationError::InvalidConfigurationValue })?; + // Placement treats a node without a rack as alone in one, so a single + // unlabeled node would take a copy of nearly every partition. Kafka + // likewise refuses a rack-aware assignment when only some brokers set + // `broker.rack`. + let racked = self.nodes.iter().filter(|node| node.rack.is_some()).count(); + if racked != 0 && racked != self.nodes.len() { + eprintln!( + "Invalid cluster configuration: {racked} of {} nodes set 'rack'; set it on every \ + node or on none", + self.nodes.len() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.nodes.iter().any(|node| { + node.rack + .as_deref() + .is_some_and(|rack| rack.trim().is_empty()) + }) { + eprintln!("Invalid cluster configuration: a node's 'rack' cannot be empty"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + let mut seen_ids = std::collections::HashSet::new(); let mut seen_names = std::collections::HashSet::new(); let mut used_endpoints = std::collections::HashSet::new(); @@ -1536,6 +1702,7 @@ request_start_view_retransmit_interval: default_request_start_view_retransmit_interval( ), view_probe_attempts_max: default_view_probe_attempts_max(), + default_replication_factor: default_replication_factor(), repair_retry_interval: default_repair_retry_interval(), repair_gap_debounce_interval: default_repair_gap_debounce_interval(), repair_chunk_max: default_repair_chunk_max(), @@ -1745,6 +1912,7 @@ fn node_with_selectors(selectors: Vec<AdvertisedAddressSelector>) -> ClusterNodeConfig { ClusterNodeConfig { + rack: None, name: "node-0".to_owned(), ip: "10.0.1.5".to_owned(), advertised_address: Some("203.0.113.10".to_owned()), @@ -1935,6 +2103,7 @@ fn node(name: &str, id: u8) -> ClusterNodeConfig { ClusterNodeConfig { + rack: None, name: name.to_string(), ip: "127.0.0.1".to_string(), advertised_address: None, @@ -1963,6 +2132,7 @@ request_start_view_retransmit_interval: default_request_start_view_retransmit_interval( ), view_probe_attempts_max: default_view_probe_attempts_max(), + default_replication_factor: default_replication_factor(), repair_retry_interval: default_repair_retry_interval(), repair_gap_debounce_interval: default_repair_gap_debounce_interval(), repair_chunk_max: default_repair_chunk_max(), @@ -2097,6 +2267,50 @@ } #[test] + fn given_default_replication_factor_text_when_parsing_should_accept_all_or_one_to_thirty_two() { + assert_eq!("all".parse(), Ok(ReplicationFactorDefault::AllNodes)); + assert_eq!("ALL".parse(), Ok(ReplicationFactorDefault::AllNodes)); + assert_eq!("3".parse(), Ok(ReplicationFactorDefault::Nodes(3))); + assert_eq!("32".parse(), Ok(ReplicationFactorDefault::Nodes(32))); + for invalid in ["0", "33", "-1", "three", ""] { + assert!( + invalid.parse::<ReplicationFactorDefault>().is_err(), + "{invalid:?}" + ); + } + for value in [ + ReplicationFactorDefault::AllNodes, + ReplicationFactorDefault::Nodes(7), + ] { + assert_eq!(value.to_string().parse(), Ok(value)); + } + assert_eq!( + ClusterConfig::default().default_replication_factor, + ReplicationFactorDefault::AllNodes, + "the shipped default keeps every partition on every node" + ); + } + + #[test] + fn given_default_replication_factor_as_integer_or_string_when_deserializing_should_accept_both() + { + let parse = |text: &str| serde_json::from_str::<ReplicationFactorDefault>(text); + assert_eq!(parse("2").unwrap(), ReplicationFactorDefault::Nodes(2)); + assert_eq!(parse("\"2\"").unwrap(), ReplicationFactorDefault::Nodes(2)); + assert_eq!( + parse("\"all\"").unwrap(), + ReplicationFactorDefault::AllNodes + ); + for invalid in ["0", "-1", "33", "\"none\""] { + assert!(parse(invalid).is_err(), "{invalid}"); + } + assert_eq!( + serde_json::to_string(&ReplicationFactorDefault::Nodes(2)).unwrap(), + "\"2\"" + ); + } + + #[test] fn validate_rejects_zero_view_probe_attempts_max() { let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); c.view_probe_attempts_max = 0; @@ -2172,6 +2386,30 @@ } #[test] + fn validate_requires_a_rack_on_every_node_or_on_none() { + let racked = |name: &str, id: u8, rack: Option<&str>| ClusterNodeConfig { + rack: rack.map(str::to_owned), + ..node(name, id) + }; + assert!( + cfg(vec![racked("n1", 0, Some("a")), racked("n2", 1, None)]) + .validate() + .is_err() + ); + assert!( + cfg(vec![racked("n1", 0, Some("a")), racked("n2", 1, Some(" "))]) + .validate() + .is_err() + ); + assert!( + cfg(vec![racked("n1", 0, Some("a")), racked("n2", 1, Some("b"))]) + .validate() + .is_ok() + ); + assert!(cfg(vec![node("n1", 0), node("n2", 1)]).validate().is_ok()); + } + + #[test] fn validate_rejects_duplicate_names() { let c = cfg(vec![node("n1", 0), node("n1", 1)]); assert!(c.validate().is_err());
diff --git a/core/configs/src/server_config/defaults.rs b/core/configs/src/server_config/defaults.rs index e4fd326..66024f9 100644 --- a/core/configs/src/server_config/defaults.rs +++ b/core/configs/src/server_config/defaults.rs
@@ -112,6 +112,11 @@ .parse() .unwrap(), view_probe_attempts_max: SERVER_CONFIG.cluster.view_probe_attempts_max as u32, + default_replication_factor: SERVER_CONFIG + .cluster + .default_replication_factor + .parse() + .unwrap(), repair_retry_interval: SERVER_CONFIG .cluster .repair_retry_interval @@ -128,6 +133,7 @@ .nodes .iter() .map(|node| ClusterNodeConfig { + rack: None, name: node.name.parse().unwrap(), ip: node.ip.parse().unwrap(), advertised_address: None,
diff --git a/core/configs/src/server_config/validators.rs b/core/configs/src/server_config/validators.rs index 4fcc7b8..07460e1 100644 --- a/core/configs/src/server_config/validators.rs +++ b/core/configs/src/server_config/validators.rs
@@ -1212,6 +1212,7 @@ fn cluster_node(replica_id: u8, http: Option<u16>) -> ClusterNodeConfig { ClusterNodeConfig { + rack: None, name: format!("node-{replica_id}"), ip: "127.0.0.1".to_string(), advertised_address: None,
diff --git a/core/connectors/runtime/Cargo.toml b/core/connectors/runtime/Cargo.toml index f2defb2..fce5d41 100644 --- a/core/connectors/runtime/Cargo.toml +++ b/core/connectors/runtime/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy-connectors" -version = "0.5.1-edge.5" +version = "0.5.1-edge.6" description = "Connectors runtime for Iggy message streaming platform" edition = "2024" license = "Apache-2.0"
diff --git a/core/consensus/src/client_table.rs b/core/consensus/src/client_table.rs index 38c1311..14e1e54 100644 --- a/core/consensus/src/client_table.rs +++ b/core/consensus/src/client_table.rs
@@ -527,8 +527,9 @@ /// /// Metadata registration stores an immutable principal, epoch and bind /// verifier. Matching login retries resolve that same registration. Explicit -/// logout or lease expiry ends it; ordered retirement in every partition group -/// precedes removal. Capacity pressure never evicts a live registration. +/// logout or lease expiry ends it. Removal follows its retirement in every +/// partition group, or a retirement debt the group pays before it serves +/// sessions again. Capacity pressure never evicts a live registration. /// /// Partition slices retain session-qualified receipts, committed windows /// and exact results. The first committed prepare fixes the group's capacity. @@ -1466,6 +1467,13 @@ .map(|entry| entry.epoch) } + /// Epoch of the session `client_id` holds a slot for, ended or not. + #[must_use] + pub fn session_epoch(&self, client_id: u128) -> Option<u64> { + let &slot_idx = self.index.get(&client_id)?; + self.slots[slot_idx].as_ref().map(|entry| entry.epoch) + } + /// Attach only after the caller has authenticated `user_id` and waited /// for the local metadata frontier to cover the requested session. /// The registered user owns the session, matching authenticated login
diff --git a/core/consensus/src/dvc_merge.rs b/core/consensus/src/dvc_merge.rs index 33d5ac6..54abfd3 100644 --- a/core/consensus/src/dvc_merge.rs +++ b/core/consensus/src/dvc_merge.rs
@@ -58,6 +58,13 @@ /// stay unless an explicit or conflicting-header nack quorum discards them, /// so the merge keeps more history, never less. pub absence_nacks: bool, + /// Empty votes the every-replica merge may take. An empty vote holds none + /// of the ops its sender acknowledged, so a replication quorum of them can + /// hide an op that only those senders acknowledged. Where a restart keeps + /// every acknowledged prepare, only a replica that lost its storage casts + /// one, and fewer than a replication quorum of them leave every committed op + /// in some other sender's log. + pub empty_votes_max: usize, } /// What the collected DVCs say about starting the view. @@ -500,6 +507,9 @@ } else if received < quorums.replica_count { return MergeOutcome::AwaitingQuorum; } + if dvc_iter(quorum).filter(|dvc| dvc.is_empty_vote()).count() > quorums.empty_votes_max { + return MergeOutcome::AwaitingQuorum; + } merge_dvc_quorum( quorum, MergeQuorums { @@ -678,6 +688,7 @@ replica_count: 3, prepare_queue_ceiling: 32, absence_nacks: true, + empty_votes_max: 3, } } @@ -940,6 +951,7 @@ replica_count: 7, prepare_queue_ceiling: 32, absence_nacks: true, + empty_votes_max: 7, }; let mut quorum = dvc_quorum_array_empty(); let mut abstaining = suffix_headers(4, 5, 7); @@ -1238,6 +1250,61 @@ untrusted(dvc(replica, 0, 0, 0, DvcSuffix::empty())) } + /// Every replica reported, so the merge takes every log, but an empty vote + /// holds none of the ops its sender acknowledged. Two of three are empty, a + /// replication quorum, and could be all that acknowledged op 10. + #[test] + fn given_every_replica_reporting_when_empty_votes_reach_a_replication_quorum_should_not_start() + { + let persisted = MergeQuorums { + empty_votes_max: 1, + ..quorums_r3() + }; + let mut quorum = dvc_quorum_array_empty(); + dvc_record( + &mut quorum, + dvc(0, 1, 9, 4, suffix_all_present(suffix_headers(4, 9, 1))), + ); + dvc_record(&mut quorum, empty_vote(1)); + dvc_record(&mut quorum, empty_vote(2)); + assert_eq!( + merge_view_change(&quorum, persisted), + MergeOutcome::AwaitingQuorum + ); + assert!( + matches!( + merge_view_change(&quorum, quorums_r3()), + MergeOutcome::Ready(_) + ), + "a durability that loses acknowledged ops on any restart takes every vote" + ); + } + + /// A two-member group commits on both members, so the member that kept its + /// storage holds every committed op, and its log alone decides the view. + #[test] + fn given_a_two_member_group_when_one_member_lost_its_storage_should_start_on_the_survivor_log() + { + let quorums = MergeQuorums { + view_change: 2, + nack_prepare: 2, + replica_count: 2, + prepare_queue_ceiling: 32, + absence_nacks: true, + empty_votes_max: 1, + }; + let mut quorum = dvc_quorum_array_empty(); + dvc_record( + &mut quorum, + dvc(0, 1, 9, 4, suffix_all_present(suffix_headers(4, 9, 1))), + ); + dvc_record(&mut quorum, empty_vote(1)); + let MergeOutcome::Ready(merged) = merge_view_change(&quorum, quorums) else { + panic!("one empty vote leaves every committed op in the survivor's log"); + }; + assert_eq!(merged.op_head, 9); + } + #[test] fn given_quorum_of_trusted_senders_when_an_untrusted_one_has_a_higher_log_view_should_merge_the_trusted_log() {
diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs index 433943e..abd7e2c 100644 --- a/core/consensus/src/impls.rs +++ b/core/consensus/src/impls.rs
@@ -20,10 +20,10 @@ use crate::{ AckLogEvent, Consensus, ControlActionLogEvent, DvcHeaderKind, DvcQuorumArray, DvcSuffix, HistoryCertificate, HistoryRecovery, IgnoreReason, MergeOutcome, MergeQuorums, MergedLog, - Pipeline, PlaneKind, PrepareLogEvent, Project, RepairEvidence, ReplicaLogContext, SimEventKind, - StoredDvc, ViewChangeLogEvent, ViewChangeReason, VsrState, dvc_count, dvc_header_kind, - dvc_iter, dvc_quorum_array_empty, dvc_record, dvc_reset, dvc_suffix_decode, emit_replica_event, - emit_sim_event, merge_view_change, seal_prepare_checksum, + Pipeline, PlaneKind, PrepareLogEvent, Project, RepairEvidence, ReplicaLogContext, ReplicaSet, + SimEventKind, StoredDvc, ViewChangeLogEvent, ViewChangeReason, VsrState, dvc_count, + dvc_header_kind, dvc_iter, dvc_quorum_array_empty, dvc_record, dvc_reset, dvc_suffix_decode, + emit_replica_event, emit_sim_event, merge_view_change, seal_prepare_checksum, }; use bit_set::BitSet; use clock::{Clock, IggySystemClock}; @@ -36,7 +36,10 @@ use iggy_common::calculate_checksum; use message_bus::IggyMessageBus; use message_bus::MessageBus; +use message_bus::SendError; +use server_common::MESSAGE_ALIGN; use server_common::Message; +use server_common::iobuf::Frozen; use server_common::poll::AutoCommitReservation; use server_common::sharding::{IggyNamespace, METADATA_GROUP}; use std::cell::{Cell, RefCell}; @@ -186,6 +189,41 @@ /// commit path, so wide clusters spend the difference on the view-change quorum. pub const QUORUM_REPLICATION_MAX: usize = 3; +/// How long a member may trail its primary's commit point before the partition +/// counts as under-replicated: Kafka's `replica.lag.time.max.ms` default. +pub const MEMBER_LAG_MAX: Duration = Duration::from_secs(30); + +/// Replicas of a `replica_count` group that must ack before an op is committed. +/// +/// Capped at [`QUORUM_REPLICATION_MAX`] to keep a wide group's commit path +/// cheap; the view-change quorum grows so the two still sum above the count. +#[must_use] +pub const fn quorum_replication(replica_count: u8) -> usize { + if replica_count == 2 { + // =1 would intersect, but =2 keeps a two-replica group durable. + return 2; + } + let half_rounded_up = (replica_count as usize).div_ceil(2); + if half_rounded_up < QUORUM_REPLICATION_MAX { + half_rounded_up + } else { + QUORUM_REPLICATION_MAX + } +} + +/// Replicas of a `replica_count` group that must send a `DoViewChange` before a +/// view can start. +/// +/// Pays for the cheaper replication quorum, which is the far hotter path. +#[must_use] +pub const fn quorum_view_change(replica_count: u8) -> usize { + if replica_count == 2 { + // Avoids a single-replica view change special case. + return 2; + } + replica_count as usize - quorum_replication(replica_count) + 1 +} + /// Headers a `DoViewChange` may carry, and so the widest uncommitted suffix a /// view change can reason about. /// @@ -1250,6 +1288,10 @@ acked: u64, /// Highest commit point sent to the replica in this view. sent: u64, + /// The commit point the replica has not acknowledged, and the monotonic + /// microseconds it has trailed since. Each ack reaching it restarts the + /// clock, so only a replica that stops keeping up ages. + behind: Option<(u64, u64)>, } impl ReplicaCommitProgress { @@ -1273,12 +1315,40 @@ self.replicas.fill(ReplicaCommitProgress::default()); } - fn record_ack(&mut self, replica: u8, op: u64) { + fn record_ack(&mut self, replica: u8, op: u64, commit: u64, now: impl FnOnce() -> u64) { if let Some(progress) = self.replicas.get_mut(usize::from(replica)) { progress.acked = progress.acked.max(op); + if progress + .behind + .is_some_and(|(marker, _)| progress.acked >= marker) + { + progress.behind = (progress.acked < commit).then(|| (commit, now())); + } } } + fn record_commit(&mut self, op: u64, now: impl FnOnce() -> u64) { + let mut now = Some(now); + let mut since = 0; + for progress in &mut self.replicas { + if progress.acked < op && progress.behind.is_none() { + if let Some(now) = now.take() { + since = now(); + } + progress.behind = Some((op, since)); + } + } + } + + /// Whether `replica` has trailed the commit point for at least `lag_max` + /// microseconds at `now`. + fn lagging(&self, replica: u8, now: u64, lag_max: u64) -> bool { + self.replicas + .get(usize::from(replica)) + .and_then(|progress| progress.behind) + .is_some_and(|(_, since)| now.saturating_sub(since) >= lag_max) + } + fn record_broadcast(&mut self, commit: u64) { for progress in &mut self.replicas { progress.sent = progress.sent.max(commit); @@ -1313,8 +1383,11 @@ P: Pipeline, { cluster: u128, + /// This replica's member index, not its node id. replica: u8, replica_count: u8, + /// `members.len() == replica_count`. + members: ReplicaSet, group: u64, /// Metadata creation revision for a partition group; zero for metadata. partition_incarnation: Cell<u64>, @@ -1559,6 +1632,8 @@ /// [`Self::request_eager_commit`]. eager_commit: Cell<bool>, commit_announcements: RefCell<CommitAnnouncements>, + /// Bit per member index: the last frame sent to it found no link. + member_unreachable: Cell<u32>, pub(crate) replication: RefCell<crate::replication::ReplicationFlow>, /// Views whose logs were refused for naming a different prepare at an op /// this replica applied; see [`Self::contradicts_applied_log`]. @@ -1750,24 +1825,24 @@ /// advertise a view older than the recorded one. /// /// # Panics - /// - If `replica >= replica_count`. - /// - If `replica_count < 1`. + /// If `self_node` is not in `members`. pub fn restored( cluster: u128, - replica: u8, - replica_count: u8, + self_node: u8, + members: ReplicaSet, group: u64, message_bus: B, pipeline: P, restore: VsrRestore<'_>, ) -> Self { - let mut consensus = Self::new( + let mut consensus = Self::with_members( cluster, - replica, - replica_count, + self_node, + members, group, message_bus, pipeline, + ConsensusClock::system(), ); let timers = restore.timers; consensus.set_normal_heartbeat_ticks(timers.normal_heartbeat_ticks); @@ -1881,6 +1956,35 @@ "replica index must be < replica_count" ); assert!(replica_count >= 1, "need at least 1 replica"); + Self::with_members( + cluster, + replica, + ReplicaSet::all(replica_count), + group, + message_bus, + pipeline, + clock, + ) + } + + /// A group whose members are the cluster nodes `members` lists, in member + /// order. [`Self::with_clock`] is the case where they are every node. + /// + /// # Panics + /// If `self_node` is not in `members`. + pub fn with_members( + cluster: u128, + self_node: u8, + members: ReplicaSet, + group: u64, + message_bus: B, + pipeline: P, + clock: ConsensusClock, + ) -> Self { + let replica = members + .member(self_node) + .unwrap_or_else(|| panic!("node {self_node} is not a member of group {group:#x}")); + let replica_count = members.len(); // Consensus-control routing distinguishes metadata frames from // partition frames by the group id: metadata uses the sentinel, // partitions use `IggyNamespace::inner()` which lives strictly @@ -1903,6 +2007,7 @@ cluster, replica, replica_count, + members, group, view: Cell::new(0), log_view: Cell::new(0), @@ -1960,6 +2065,7 @@ heartbeat_replies: RefCell::new(BitSet::with_capacity(REPLICAS_MAX)), eager_commit: Cell::new(false), commit_announcements: RefCell::new(CommitAnnouncements::new(replica_count)), + member_unreachable: Cell::new(0), replication: RefCell::new(crate::replication::ReplicationFlow::new(replica_count)), applied_log_contradictions: Cell::new(0), applied_log_contradictions_taken: Cell::new(0), @@ -2140,6 +2246,12 @@ (view % self.replica_count as u32) as u8 } + /// Node id of `view`'s primary, for anything addressed outside the group. + #[must_use] + pub fn primary_node(&self, view: u32) -> u8 { + self.member_node(self.primary_index(view)) + } + #[must_use] pub const fn is_primary(&self) -> bool { self.primary_index(self.view.get()) == self.replica @@ -2152,16 +2264,18 @@ self.is_primary() && self.status.get() == Status::Normal && !self.ceded_primaryship.get() } - /// Ask the partition quorum to elect `target` in the next view assigned to it. - /// Only the primary can: its `StartViewChange` leaves its view, so its backups - /// follow it, while that of a backup is only a vote. Refusal leaves consensus - /// unchanged. + /// Ask the partition quorum to elect node `target_node` in the next view + /// assigned to it. Only the primary can: its `StartViewChange` leaves its + /// view, so its backups follow it, while that of a backup is only a vote. + /// Refusal leaves consensus unchanged. /// The caller fences the incarnation, drains history work, refreshes the DVC /// suffix and persists the new view before dispatching the returned actions. - pub fn request_primary_transfer(&self, expected_view: u32, target: u8) -> Vec<VsrAction> { + pub fn request_primary_transfer(&self, expected_view: u32, target_node: u8) -> Vec<VsrAction> { let view = self.view(); + let Some(target) = self.members.member(target_node) else { + return Vec::new(); + }; if self.group == METADATA_GROUP - || target >= self.replica_count || view != expected_view || !self.is_acting_primary() || self.is_transferring() @@ -2235,6 +2349,54 @@ ); self.commit_min.set(op); assert!(self.commit_max.get() >= self.commit_min.get()); + // Members acked ops up to the view start in an earlier view, and the + // acks this view counts were reset with it. + if self.is_primary() && op > self.view_start_head.get() { + self.commit_announcements + .borrow_mut() + .record_commit(op, || self.message_bus.monotonic_micros()); + } + } + + /// Members, this replica included, that keep up with this primary: a link + /// reaches them and they trail its commit point for less than `lag_max`. + /// `None` unless this replica is the acting primary. + #[must_use] + pub fn in_sync_members(&self, lag_max: Duration) -> Option<u8> { + self.in_sync_member_mask(lag_max) + .and_then(|mask| u8::try_from(mask.count_ones()).ok()) + } + + /// [`Self::in_sync_members`] as a mask of node ids, bit `n` for node `n`. + #[must_use] + pub fn in_sync_nodes(&self, lag_max: Duration) -> Option<u32> { + self.in_sync_member_mask(lag_max).map(|mask| { + (0..self.replica_count) + .filter(|member| mask & (1 << member) != 0) + .filter_map(|member| self.members.node(member)) + .filter_map(|node| 1_u32.checked_shl(u32::from(node))) + .fold(0, |nodes, bit| nodes | bit) + }) + } + + /// The in-sync members as a mask of member indices. + fn in_sync_member_mask(&self, lag_max: Duration) -> Option<u32> { + if !self.is_acting_primary() { + return None; + } + let now = self.message_bus.monotonic_micros(); + let lag_max = u64::try_from(lag_max.as_micros()).unwrap_or(u64::MAX); + let unreachable = self.member_unreachable.get(); + let announcements = self.commit_announcements.borrow(); + let mask = (0..self.replica_count) + .filter(|member| { + *member == self.replica + || !(unreachable & (1 << member) != 0 + || announcements.lagging(*member, now, lag_max)) + }) + .fold(0, |mask, member| mask | (1 << member)); + drop(announcements); + Some(mask) } /// Restore local commit progress from already-applied state during bootstrap. @@ -2271,34 +2433,17 @@ (self.replica_count as usize - 1) / 2 } - /// Replicas that must ack before an op is committed. - /// - /// Capped at [`QUORUM_REPLICATION_MAX`] to keep a wide cluster's commit path - /// cheap; the view-change quorum grows so the two still sum above the count. + /// Replicas that must ack before an op is committed; see [`quorum_replication`]. #[must_use] pub const fn quorum_replication(&self) -> usize { - if self.replica_count == 2 { - // =1 would intersect, but =2 keeps a two-replica cluster durable. - return 2; - } - let half_rounded_up = (self.replica_count as usize).div_ceil(2); - if half_rounded_up < QUORUM_REPLICATION_MAX { - half_rounded_up - } else { - QUORUM_REPLICATION_MAX - } + quorum_replication(self.replica_count) } - /// Replicas that must send a `DoViewChange` before a view can start. - /// - /// Pays for the cheaper replication quorum, which is the far hotter path. + /// Replicas that must send a `DoViewChange` before a view can start; see + /// [`quorum_view_change`]. #[must_use] pub const fn quorum_view_change(&self) -> usize { - if self.replica_count == 2 { - // Avoids a single-replica view change special case. - return 2; - } - self.replica_count as usize - self.quorum_replication() + 1 + quorum_view_change(self.replica_count) } /// Nacks required to prove an op was never committed, so the new primary may @@ -2329,6 +2474,31 @@ self.replica } + /// This replica's cluster node id. + #[must_use] + pub fn node_id(&self) -> u8 { + self.member_node(self.replica) + } + + #[must_use] + pub const fn members(&self) -> &ReplicaSet { + &self.members + } + + /// Node id of `member`, which every caller derives from this group's size. + /// + /// # Panics + /// If `member` is not below [`Self::replica_count`]. + #[must_use] + pub fn member_node(&self, member: u8) -> u8 { + self.members.node(member).unwrap_or_else(|| { + panic!( + "member {member} outside group {:#x} of {}", + self.group, self.replica_count + ) + }) + } + #[must_use] pub const fn sequencer(&self) -> &LocalSequencer { &self.sequencer @@ -2860,7 +3030,7 @@ pub fn vsr_state(&self, checkpoint_op: u64, checkpoint_checksum: u128) -> VsrState { VsrState { cluster: self.cluster, - replica_id: self.replica, + replica_id: self.node_id(), replica_count: self.replica_count, view: self.view.get(), log_view: self.persisted_log_view(), @@ -5170,6 +5340,11 @@ // config says nothing about how deep a peer's pipeline is. prepare_queue_ceiling: PREPARE_QUEUE_CEILING as u64, absence_nacks: true, + empty_votes_max: if self.recovered_log_complete.get() { + self.quorum_replication() - 1 + } else { + self.replica_count as usize + }, } } @@ -5773,7 +5948,9 @@ return; } let mut announcements = self.commit_announcements.borrow_mut(); - announcements.record_ack(replica, op); + announcements.record_ack(replica, op, self.commit_min.get(), || { + self.message_bus.monotonic_micros() + }); if announcements.owes(replica) { self.eager_commit.set(true); } @@ -5844,11 +6021,7 @@ self.push_loopback(message); return true; } - match self - .message_bus - .send_to_replica(target, message.into_frozen()) - .await - { + match self.send_to_member(target, message.into_frozen()).await { Ok(()) => true, Err(error) => { tracing::warn!( @@ -5861,6 +6034,37 @@ } } + /// Send to the node behind `member`. Every frame a group addresses to one + /// of its own replicas goes through here, never straight to the bus. + /// + /// # Errors + /// The bus error, or `ReplicaNotConnected` when `member` is outside the group. + // VsrConsensus uses Cell/RefCell for single-threaded compio shards; futures are intentionally !Send. + #[allow(clippy::future_not_send)] + pub async fn send_to_member( + &self, + member: u8, + message: Frozen<MESSAGE_ALIGN>, + ) -> Result<(), SendError> { + let Some(node) = self.members.node(member) else { + return Err(SendError::ReplicaNotConnected(member)); + }; + let sent = self.message_bus.send_to_replica(node, message).await; + let unreachable = match &sent { + Ok(()) => Some(false), + Err(SendError::ReplicaNotConnected(_) | SendError::ConnectionClosed) => Some(true), + Err(_) => None, + }; + if let Some(unreachable) = unreachable + && let Some(bit) = 1_u32.checked_shl(u32::from(member)) + { + let mask = self.member_unreachable.get(); + self.member_unreachable + .set(if unreachable { mask | bit } else { mask & !bit }); + } + sent + } + #[must_use] pub const fn message_bus(&self) -> &B { &self.message_bus @@ -7400,6 +7604,138 @@ } #[test] + fn given_subset_group_when_requesting_primary_transfer_should_address_target_by_node() { + const MEMBERS: [u8; 3] = [4, 1, 3]; + let members = ReplicaSet::new(&MEMBERS).expect("valid members"); + for (target_node, next_view) in [(3, 2), (1, 1)] { + let consensus = VsrConsensus::with_members( + 1, + 4, + members, + 0, + StageNoopBus, + LocalPipeline::new(), + ConsensusClock::system(), + ); + consensus.mark_superblock_durable(0, 0); + consensus.init(); + assert_eq!(consensus.primary_node(0), 4); + + let actions = consensus.request_primary_transfer(0, target_node); + + assert!( + matches!( + actions.as_slice(), + [VsrAction::SendStartViewChange { view, group: 0 }] if *view == next_view + ), + "target node {target_node}" + ); + assert_eq!(consensus.primary_node(next_view), target_node); + } + let consensus = VsrConsensus::with_members( + 1, + 4, + members, + 0, + StageNoopBus, + LocalPipeline::new(), + ConsensusClock::system(), + ); + consensus.mark_superblock_durable(0, 0); + consensus.init(); + assert!( + consensus.request_primary_transfer(0, 2).is_empty(), + "node 2 is not a member, whatever its id falls inside the group size" + ); + } + + #[test] + fn given_a_member_that_stops_acking_when_ops_commit_should_drop_out_of_sync() { + let consensus = VsrConsensus::new(1, 0, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + assert_eq!(consensus.in_sync_members(Duration::ZERO), Some(3)); + + consensus.note_backup_ack(1, 1); + consensus.advance_commit_max(1); + consensus.advance_commit_min(1); + assert_eq!( + consensus.in_sync_members(Duration::ZERO), + Some(2), + "member 2 trails op 1" + ); + assert_eq!( + consensus.in_sync_members(MEMBER_LAG_MAX), + Some(3), + "within the lag window" + ); + + consensus.note_backup_ack(2, 1); + assert_eq!(consensus.in_sync_members(Duration::ZERO), Some(3)); + + let backup = VsrConsensus::new(1, 1, 3, 0, StageNoopBus, LocalPipeline::new()); + backup.init(); + assert_eq!( + backup.in_sync_members(MEMBER_LAG_MAX), + None, + "only a primary counts" + ); + } + + #[test] + fn given_ops_up_to_the_view_start_when_a_new_primary_commits_them_should_not_mark_members_behind() + { + let consensus = VsrConsensus::new(1, 0, 3, 0, StageNoopBus, LocalPipeline::new()); + consensus.init(); + consensus.view_start_head.set(2); + consensus.advance_commit_max(3); + consensus.advance_commit_min(1); + consensus.advance_commit_min(2); + assert_eq!( + consensus.in_sync_members(Duration::ZERO), + Some(3), + "ops the members acked in an earlier view" + ); + consensus.advance_commit_min(3); + assert_eq!(consensus.in_sync_members(Duration::ZERO), Some(1)); + } + + #[test] + fn given_subset_group_when_snapshotting_state_should_record_node_and_group_size() { + let members = ReplicaSet::new(&[4, 1, 3]).expect("valid members"); + let consensus = VsrConsensus::with_members( + 1, + 3, + members, + 0, + StageNoopBus, + LocalPipeline::new(), + ConsensusClock::system(), + ); + + let state = consensus.vsr_state(0, 0); + + assert_eq!(consensus.replica(), 2); + assert_eq!(consensus.node_id(), 3); + assert_eq!(state.replica_id, 3); + assert_eq!(state.replica_count, 3); + } + + #[test] + #[should_panic(expected = "node 0 is not a member")] + fn given_node_outside_members_when_building_group_should_panic() { + let members = ReplicaSet::new(&[4, 1, 3]).expect("valid members"); + let _ = VsrConsensus::with_members( + 1, + 0, + members, + 0, + StageNoopBus, + LocalPipeline::new(), + ConsensusClock::system(), + ); + } + + #[test] fn given_a_backup_when_requesting_primary_transfer_should_refuse() { let backup = VsrConsensus::new(1, 1, 3, 0, StageNoopBus, LocalPipeline::new()); backup.init();
diff --git a/core/consensus/src/lib.rs b/core/consensus/src/lib.rs index 6df0a8d..4303049 100644 --- a/core/consensus/src/lib.rs +++ b/core/consensus/src/lib.rs
@@ -215,6 +215,8 @@ mod dvc_merge; pub use dvc_merge::*; +mod replica_set; +pub use replica_set::{ReplicaSet, ReplicaSetError}; mod vsr_state; pub use vsr_state::{AdoptedView, HistoryCertificate, HistoryRecovery, VsrState, VsrStateError}; mod vsr_timeout;
diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs index 0fe8620..050fecb 100644 --- a/core/consensus/src/plane_helpers.rs +++ b/core/consensus/src/plane_helpers.rs
@@ -1094,7 +1094,7 @@ #[cfg(test)] mod tests { use super::*; - use crate::{Consensus, LocalPipeline, VsrAction}; + use crate::{Consensus, ConsensusClock, LocalPipeline, ReplicaSet, VsrAction}; use aligned_vec::{AVec, ConstAlign}; use iggy_binary_protocol::{ConsensusHeader, Operation, StartViewChangeHeader}; use iggy_common::calculate_checksum; @@ -2709,6 +2709,101 @@ assert_eq!(sent[0].0, 1); } + const SUBSET_MEMBERS: [u8; 3] = [4, 1, 3]; + + fn subset_consensus(self_node: u8) -> VsrConsensus<SpyBus> { + let consensus = VsrConsensus::with_members( + 1, + self_node, + ReplicaSet::new(&SUBSET_MEMBERS).expect("valid members"), + 0, + SpyBus::new(), + LocalPipeline::new(), + ConsensusClock::system(), + ); + consensus.init(); + consensus + } + + #[test] + fn given_subset_group_when_sending_to_members_should_address_their_nodes() { + let consensus = subset_consensus(1); + let message = || { + Message::<PrepareOkHeader>::new(std::mem::size_of::<PrepareOkHeader>()).into_generic() + }; + + assert!(futures::executor::block_on( + consensus.send_or_loopback(0, message()) + )); + assert!(futures::executor::block_on( + consensus.send_or_loopback(2, message()) + )); + assert!(futures::executor::block_on( + consensus.send_or_loopback(1, message()) + )); + assert!(matches!( + futures::executor::block_on(consensus.send_to_member(3, message().into_frozen())), + Err(SendError::ReplicaNotConnected(3)) + )); + + let mut looped = Vec::new(); + consensus.drain_loopback_into(&mut looped); + assert_eq!(looped.len(), 1, "member 1 is this node"); + assert_eq!(*consensus.message_bus().attempts.borrow(), [4, 3]); + } + + #[test] + fn given_no_link_to_a_member_when_sending_should_count_it_out_of_sync() { + let consensus = subset_consensus(4); + let send = |consensus: &VsrConsensus<SpyBus>| { + futures::executor::block_on( + consensus.send_to_member( + 1, + Message::<PrepareOkHeader>::new(std::mem::size_of::<PrepareOkHeader>()) + .into_generic() + .into_frozen(), + ), + ) + }; + assert_eq!(consensus.in_sync_members(crate::MEMBER_LAG_MAX), Some(3)); + + consensus.message_bus().reject_sends.set(true); + assert!(send(&consensus).is_err()); + assert_eq!(consensus.in_sync_members(crate::MEMBER_LAG_MAX), Some(2)); + + consensus.message_bus().reject_sends.set(false); + send(&consensus).expect("the link is back"); + assert_eq!(consensus.in_sync_members(crate::MEMBER_LAG_MAX), Some(3)); + } + + #[test] + fn given_subset_group_when_replicating_should_send_to_each_member_node() { + let consensus = subset_consensus(4); + futures::executor::block_on(replicate_frozen_prepare( + &consensus, + prepare_message(1, 0, 42).into_frozen(), + )) + .expect("both backups are connected"); + assert_eq!(*consensus.message_bus().attempts.borrow(), [1, 3]); + + let consensus = subset_consensus(4); + consensus + .message_bus() + .failures + .borrow_mut() + .insert(1, SendError::ReplicaNotConnected(1)); + let result = futures::executor::block_on(replicate_frozen_prepare( + &consensus, + prepare_message(1, 0, 42).into_frozen(), + )); + assert!(result.is_err(), "the unreachable backup stays visible"); + assert_eq!( + *consensus.message_bus().attempts.borrow(), + [1, 3], + "node 3 still gets its copy" + ); + } + /// `advance_commit_min` is strictly sequential, so draining op 7 with 6 never /// applied panics the shard pump. The commit gate must hold instead. #[test]
diff --git a/core/consensus/src/replica_set.rs b/core/consensus/src/replica_set.rs new file mode 100644 index 0000000..4b987a3 --- /dev/null +++ b/core/consensus/src/replica_set.rs
@@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::REPLICAS_MAX; +use std::{error::Error, fmt}; + +/// Cluster node ids of one consensus group, in member order. +/// +/// Consensus state and frame headers address replicas by member index; only +/// the message bus addresses nodes, so sends translate through this set. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ReplicaSet { + nodes: [u8; REPLICAS_MAX], + len: u8, +} + +impl ReplicaSet { + /// Nodes `0..count`, where member index and node id coincide. + /// + /// # Panics + /// If `count` is zero or above [`REPLICAS_MAX`]. + #[must_use] + pub fn all(count: u8) -> Self { + assert!( + (1..=REPLICAS_MAX).contains(&usize::from(count)), + "replica set size {count} outside 1..={REPLICAS_MAX}" + ); + let mut nodes = [0; REPLICAS_MAX]; + for (node, slot) in (0..count).zip(nodes.iter_mut()) { + *slot = node; + } + Self { nodes, len: count } + } + + /// # Errors + /// If `nodes` is empty, longer than [`REPLICAS_MAX`], or repeats a node. + pub fn new(nodes: &[u8]) -> Result<Self, ReplicaSetError> { + if nodes.is_empty() { + return Err(ReplicaSetError::Empty); + } + if nodes.len() > REPLICAS_MAX { + return Err(ReplicaSetError::TooMany { len: nodes.len() }); + } + for (index, node) in nodes.iter().enumerate() { + if nodes[..index].contains(node) { + return Err(ReplicaSetError::Duplicate { node: *node }); + } + } + let mut members = [0; REPLICAS_MAX]; + members[..nodes.len()].copy_from_slice(nodes); + Ok(Self { + nodes: members, + #[allow(clippy::cast_possible_truncation)] + len: nodes.len() as u8, + }) + } + + #[must_use] + #[expect( + clippy::len_without_is_empty, + reason = "construction refuses an empty set" + )] + pub const fn len(&self) -> u8 { + self.len + } + + #[must_use] + pub fn nodes(&self) -> &[u8] { + &self.nodes[..usize::from(self.len)] + } + + /// Node id of `member`, `None` when the index is outside the group. + #[must_use] + pub fn node(&self, member: u8) -> Option<u8> { + self.nodes().get(usize::from(member)).copied() + } + + /// Member index of `node`, `None` when the node is not in the group. + #[must_use] + pub fn member(&self, node: u8) -> Option<u8> { + self.nodes() + .iter() + .position(|candidate| *candidate == node) + .and_then(|index| u8::try_from(index).ok()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReplicaSetError { + Empty, + TooMany { len: usize }, + Duplicate { node: u8 }, +} + +impl fmt::Display for ReplicaSetError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Empty => formatter.write_str("replica set has no members"), + Self::TooMany { len } => write!( + formatter, + "replica set has {len} members, above the {REPLICAS_MAX} limit" + ), + Self::Duplicate { node } => { + write!(formatter, "replica set lists node {node} twice") + } + } + } +} + +impl Error for ReplicaSetError {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn given_all_nodes_when_translating_should_map_index_to_same_node() { + let set = ReplicaSet::all(3); + + assert_eq!(set.nodes(), &[0, 1, 2]); + assert_eq!(set.node(2), Some(2)); + assert_eq!(set.member(1), Some(1)); + assert_eq!(set.node(3), None); + } + + #[test] + fn given_subset_when_translating_should_map_both_ways() { + let set = ReplicaSet::new(&[4, 1, 3]).expect("valid set"); + + assert_eq!(set.len(), 3); + assert_eq!(set.node(0), Some(4)); + assert_eq!(set.node(2), Some(3)); + assert_eq!(set.member(4), Some(0)); + assert_eq!(set.member(3), Some(2)); + assert_eq!(set.member(0), None); + assert_eq!(set.node(3), None); + } + + #[test] + fn given_invalid_nodes_when_building_should_refuse() { + assert_eq!(ReplicaSet::new(&[]), Err(ReplicaSetError::Empty)); + assert_eq!( + ReplicaSet::new(&[1, 2, 1]), + Err(ReplicaSetError::Duplicate { node: 1 }) + ); + let too_many: Vec<u8> = (0..=u8::try_from(REPLICAS_MAX).expect("fits")).collect(); + assert_eq!( + ReplicaSet::new(&too_many), + Err(ReplicaSetError::TooMany { + len: REPLICAS_MAX + 1 + }) + ); + } +}
diff --git a/core/consensus/src/replication.rs b/core/consensus/src/replication.rs index c6576a6..7c22186 100644 --- a/core/consensus/src/replication.rs +++ b/core/consensus/src/replication.rs
@@ -341,7 +341,7 @@ if !self.admit_replication(replica, op, size, send) { return Ok(false); } - if let Err(error) = self.message_bus().send_to_replica(replica, prepare).await { + if let Err(error) = self.send_to_member(replica, prepare).await { self.record_replication_failure(replica, op); return Err(error); } @@ -441,8 +441,7 @@ }, ); if self - .message_bus() - .send_to_replica(replica, probe.into_generic().into_frozen()) + .send_to_member(replica, probe.into_generic().into_frozen()) .await .is_err() {
diff --git a/core/integration/tests/cli/topic/test_topic_get_command.rs b/core/integration/tests/cli/topic/test_topic_get_command.rs index 48a6a6d..35cda4c 100644 --- a/core/integration/tests/cli/topic/test_topic_get_command.rs +++ b/core/integration/tests/cli/topic/test_topic_get_command.rs
@@ -217,7 +217,7 @@ let mut assertion = assertion .stdout(contains("Partition id | Created")) .stdout(contains("Segments count | Current offset | Size")) - .stdout(contains("Messages count")) + .stdout(contains("Messages count | Members | In sync")) .stdout(contains("Option | Value")) .stdout(contains("Source")) .stdout(contains( @@ -262,7 +262,7 @@ if self.populate_partitions { assert!(row.contains(&format!("| 1 | {index} |")), "{row}"); assert!(row.contains(" B |") && !row.contains("| 0 B |"), "{row}"); - assert!(row.ends_with(&format!("| {} |", index + 1)), "{row}"); + assert!(row.contains(&format!(" B | {} |", index + 1)), "{row}"); } else { assert!(row.contains("| 1 | 0 | 0 B | 0 |"), "{row}"); }
diff --git a/core/integration/tests/cluster/mod.rs b/core/integration/tests/cluster/mod.rs index 3a0f34c..6bc85ee 100644 --- a/core/integration/tests/cluster/mod.rs +++ b/core/integration/tests/cluster/mod.rs
@@ -36,6 +36,7 @@ mod replica_links; mod replica_raw_request; mod replica_read_batching; +mod replication_factor; mod rolling_sigkill_repro; mod root_credentials; mod staggered_bootstrap;
diff --git a/core/integration/tests/cluster/partition_primary_routing.rs b/core/integration/tests/cluster/partition_primary_routing.rs index 367216c..7be6e4b 100644 --- a/core/integration/tests/cluster/partition_primary_routing.rs +++ b/core/integration/tests/cluster/partition_primary_routing.rs
@@ -749,9 +749,11 @@ .await .unwrap(); let stream = Identifier::named(STREAM_NAME).unwrap(); + // Every node stores every partition, so any node can take a primary. let options = TopicCreateOptions { partitions_count: Some(1), durability: Durability::Persisted, + replication_factor: Some(5), message_expiry: Some(IggyExpiry::NeverExpire), ..TopicCreateOptions::default() };
diff --git a/core/integration/tests/cluster/partition_state_transfer.rs b/core/integration/tests/cluster/partition_state_transfer.rs index 6eb5a7d..d542cd1 100644 --- a/core/integration/tests/cluster/partition_state_transfer.rs +++ b/core/integration/tests/cluster/partition_state_transfer.rs
@@ -71,8 +71,13 @@ /// time and the kill must land within the pull, so the ordinary 200ms /// cadence is too coarse here. const KILL_GATE_POLL: Duration = Duration::from_millis(10); +/// Cluster size and replication factor of the dead-transfer-peer spec. +const TRANSFER_PEER_DIES_NODES: u32 = 5; const CONVERSION_MARKER: &str = "partition repair floor unreachable; converting to state transfer"; +/// A wiped member whose storage began more than the storage-epoch skew after +/// the partition was created joins awaiting a transfer instead of repairing. +const LOST_STORAGE_MARKER: &str = "waiting for a member that kept its history"; const INSTALL_MARKER: &str = "partition state transfer installed"; const INSTALL_FAILED_MARKER: &str = "partition state transfer install failed"; /// The `OfferErasesHeldOps` reason, logged inside an [`INSTALL_FAILED_MARKER`] @@ -185,13 +190,21 @@ // this spec isolates the PARTITION plane. let _seed_client = client; - // Wipe node 2 and rejoin: no local history at all, so repair cannot - // connect any floor and the refusal converts to a transfer. + // Wipe node 2 and rejoin with no local history at all. Inside the + // storage-epoch skew of the create it joins fresh, repair cannot connect + // any floor and the refusal converts to a transfer. Past it, the node + // knows it lost the partition and awaits the transfer from the start. + // Which one runs depends on how long the seeding took. harness .restart_node_from_clean_slate(2) .expect("clean-slate restart of node 2"); - await_marker(harness, 2, CONVERSION_MARKER).await; + await_new_marker_any( + harness, + 2, + &[(CONVERSION_MARKER, 0), (LOST_STORAGE_MARKER, 0)], + ) + .await; await_marker(harness, 2, INSTALL_MARKER).await; // Disk proof on the rejoined node: transferred segment bytes and a @@ -295,12 +308,16 @@ } #[iggy_harness( - cluster_nodes = 3, + cluster_nodes = 5, server( sharding.cpu_allocation = "0..1", partition.evicted_ring_capacity = "64" ) )] +// Five members, so the three left once node 0 dies and node 2 is wiped still +// form a quorum that can elect past node 0. A wiped member never votes before +// its transfer: three members could not elect with one down and one wiped. +// // The topic's 8 MiB segment cap makes the 64 MiB bulky seed span several sealed // segments, so unlike the other specs (single-segment under the default 1 GiB // cap) this one installs from a MULTI-ARTIFACT manifest, one spill per artifact. @@ -315,7 +332,12 @@ .root_client_for_node(0) .await .expect("connect a root client to the node"); - seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await; + seed_topic( + &client, + Some(SEGMENT_SIZE_MULTI_ARTIFACT), + Some(TRANSFER_PEER_DIES_NODES), + ) + .await; // Bulky payloads so the pull spans many 256 KiB chunks: the kill below // must land while the transfer is provably in flight, and a small // partition finishes inside the marker-poll latency, leaving the @@ -324,26 +346,29 @@ sleep(Duration::from_secs(1)).await; let _seed_client = client; - // Wipe node 2, wait until its rejoin CONVERTED to a transfer, then kill - // the serving peer (the view-0 primary, node 0) mid-pull. Node 2 must not + // Wipe node 2, wait until its rejoin armed a transfer at the view-0 + // primary, node 0, then kill that peer mid-pull. Node 2 must not // retry into the corpse forever: it re-arms against the next peer -- via // the stall budget, or via the StartView it adopts once the survivors - // elect past node 0 -- and the transfer re-runs against the new primary - // (node 1). + // elect past node 0 -- and the transfer re-runs against the new primary. harness .restart_node_from_clean_slate(2) .expect("clean-slate restart of node 2"); - // Kill node 0 the moment node 2 CONVERTED: the transfer is then armed - // at node 0 but the 64 MiB pull cannot possibly finish inside the kill - // latency, so node 2 deterministically ends up parked against a dead - // peer, whether the descriptor made it out or not. (Gating on the serving - // marker instead raced the pull itself: a release-build pull finishes in - // a few hundred ms.) + // Kill node 0 the moment node 2 armed the transfer there: by converting + // its refused repair floor when it rejoined inside the storage-epoch skew + // of the create, or, past it, by requesting one on adopting node 0's view. + // The 64 MiB pull cannot possibly finish inside the kill latency, so node + // 2 deterministically ends up parked against a dead peer, whether the + // descriptor made it out or not. (Gating on the serving marker instead + // raced the pull itself: a release-build pull finishes in a few hundred + // ms.) let deadline = Instant::now() + TRANSFER_BUDGET; - while !harness.node(2).stdout_contains(CONVERSION_MARKER) { + while !(harness.node(2).stdout_contains(CONVERSION_MARKER) + || harness.node(2).stdout_contains(REARM_ON_VIEW_MARKER)) + { assert!( Instant::now() < deadline, - "node 2 never converted its refused repair floor to a transfer" + "node 2 never armed a transfer at node 0" ); sleep(KILL_GATE_POLL).await; } @@ -352,7 +377,13 @@ // have installed and node 1 can have fully served an earlier attempt while // node 0 was still up, and a `contains` would call those the recovery. let installs_before_kill = harness.node(2).stdout_occurrences(INSTALL_MARKER); - let served_before_kill = harness.node(1).stdout_occurrences(FULLY_SERVED_MARKER); + let served_by_survivors = |harness: &TestHarness| -> usize { + [1, 3, 4] + .into_iter() + .map(|node| harness.node(node).stdout_occurrences(FULLY_SERVED_MARKER)) + .sum() + }; + let served_before_kill = served_by_survivors(harness); let abandons_before_kill = harness.node(2).stdout_occurrences(ABANDON_MARKER); let rearms_before_kill = harness.node(2).stdout_occurrences(REARM_ON_VIEW_MARKER); harness @@ -372,14 +403,21 @@ .await; // Recovery: the scheduled re-arm targets the surviving primary. No - // follow-up commit is asserted -- the cluster is quorum-marginal with - // one node down, and an unanswered read mid-election is not a verdict. + // follow-up commit is asserted -- an unanswered read mid-election is not + // a verdict. // - // Node 1 is named explicitly, not "any survivor": with node 0 dead it is - // the only replica left that can serve, so a marker from it is proof the - // re-arm found a new peer rather than proof of the pre-kill attempt. + // Only the survivors are counted, not node 0: with node 0 dead, a marker + // from one of them is proof the re-arm found a new peer rather than proof + // of the pre-kill attempt. await_new_marker(harness, 2, INSTALL_MARKER, installs_before_kill).await; - await_new_marker(harness, 1, FULLY_SERVED_MARKER, served_before_kill).await; + let deadline = Instant::now() + TRANSFER_BUDGET; + while served_by_survivors(harness) <= served_before_kill { + assert!( + Instant::now() < deadline, + "no survivor fully served node 2 a transfer within {TRANSFER_BUDGET:?}" + ); + sleep(MARKER_POLL).await; + } // The manifest the install consumed carried one artifact per sealed // segment, and each was spilled and renamed separately: the seeded 64 MiB @@ -418,7 +456,7 @@ .root_client_for_node(0) .await .expect("connect a root client to the node"); - seed_topic(&client, None).await; + seed_topic(&client, None, None).await; harness .isolate_node(LAGGING_NODE) @@ -516,7 +554,7 @@ .root_client_for_node(0) .await .expect("connect a root client to the node"); - seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await; + seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT), None).await; produce_bulky(&client, 2 * BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; let mut expected: Vec<String> = (0..2 * BULKY_SENDS_PER_ROUND) .map(bulky_payload_prefix) @@ -600,7 +638,7 @@ .root_client_for_node(0) .await .expect("connect a root client to the node"); - seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await; + seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT), None).await; produce_bulky(&client, 2 * BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; let mut expected: Vec<String> = (0..2 * BULKY_SENDS_PER_ROUND) .map(bulky_payload_prefix) @@ -689,7 +727,7 @@ .root_client_for_node(0) .await .expect("connect a root client to the node"); - seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await; + seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT), None).await; produce_bulky(&client, 2 * BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; let mut expected: Vec<String> = (0..2 * BULKY_SENDS_PER_ROUND) .map(bulky_payload_prefix) @@ -754,7 +792,7 @@ .root_client_for_node(0) .await .expect("connect a root client to the node"); - seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT)).await; + seed_topic(&client, Some(SEGMENT_SIZE_MULTI_ARTIFACT), None).await; produce_bulky(&client, BULKY_SENDS_PER_ROUND, BULKY_PAYLOAD_LEN).await; let mut expected: Vec<String> = (0..BULKY_SENDS_PER_ROUND) .map(bulky_payload_prefix) @@ -794,7 +832,7 @@ .root_client_for_node(0) .await .expect("connect a root client to the node"); - seed_topic(&client, None).await; + seed_topic(&client, None, None).await; produce(&client, 40).await; sleep(Duration::from_secs(1)).await; harness.stop_node(2).expect("stop node 2"); @@ -1051,10 +1089,15 @@ /// `segment_size` is `None` for the specs that only need the default cap; the /// multi-artifact spec passes one small enough that its bulky seed spans -/// several sealed segments. `messages_required_to_save = 1` is the forcing +/// several sealed segments, and stores the partition on all five of its nodes. +/// `messages_required_to_save = 1` is the forcing /// function every spec here shares: each commit flushes and ring-evicts, which /// is what marches `repair_retained_from` forward. -async fn seed_topic(client: &IggyClient, segment_size: Option<u64>) { +async fn seed_topic( + client: &IggyClient, + segment_size: Option<u64>, + replication_factor: Option<u32>, +) { client .create_stream(STREAM_NAME, &StreamCreateOptions::default()) .await @@ -1070,6 +1113,7 @@ segment_size: segment_size.map(IggyByteSize::from), durability: Durability::Persisted, consumer_offset_durability: Durability::Persisted, + replication_factor, ..TopicCreateOptions::default() }, ) @@ -1124,7 +1168,7 @@ } async fn seed_partition(client: &IggyClient) { - seed_topic(client, None).await; + seed_topic(client, None, None).await; produce(client, MESSAGES_COUNT).await; assert_eq!( poll_count(client, MESSAGES_COUNT).await,
diff --git a/core/integration/tests/cluster/replication_factor.rs b/core/integration/tests/cluster/replication_factor.rs new file mode 100644 index 0000000..6cbe35d --- /dev/null +++ b/core/integration/tests/cluster/replication_factor.rs
@@ -0,0 +1,959 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A topic's replication factor bounds the nodes that store each partition, +//! while a client connected to any node still reaches every partition. + +use std::collections::{BTreeSet, HashMap}; +use std::str::FromStr; +use std::time::Duration; + +use super::partition_primary_routing::partition_route; +use super::register_forwarding::connect_without_login; +use crate::server::http_client::{HttpClient, node_url}; +use iggy::prelude::*; +use iggy_common::{HeaderKey, HeaderValue, topic_option_keys}; +use integration::harness::{TestHarness, TestServerConfig, resolve_config_paths}; +use integration::iggy_harness; +use serial_test::parallel; +use tokio::time::{sleep, timeout}; + +const STREAM_NAME: &str = "replication-factor-stream"; +const TOPIC_NAME: &str = "replication-factor-topic"; +const WITNESS_TOPIC_NAME: &str = "replication-factor-witness"; +const PARTITIONS: u32 = 3; +const POLL_BUDGET: Duration = Duration::from_secs(20); +const POLL_INTERVAL: Duration = Duration::from_millis(200); +const METRICS_BUDGET: Duration = Duration::from_secs(60); + +async fn root_client(harness: &TestHarness, node: usize) -> TcpClient { + let client = connect_without_login(harness.node(node).tcp_addr().unwrap()).await; + client + .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD) + .await + .unwrap(); + client +} + +#[iggy_harness(cluster_nodes = 3, server(sharding.cpu_allocation = "2"))] +async fn given_a_factor_of_one_when_writing_through_any_node_should_store_each_partition_once( + harness: &mut TestHarness, +) { + let writer = root_client(harness, 0).await; + writer + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); + let stream = Identifier::named(STREAM_NAME).unwrap(); + let topic = Identifier::named(TOPIC_NAME).unwrap(); + writer + .create_topic( + &stream, + TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS), + replication_factor: Some(1), + durability: Durability::Persisted, + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + + let stream_id = writer.get_stream(&stream).await.unwrap().unwrap().id; + let details = writer.get_topic(&stream, &topic).await.unwrap().unwrap(); + let factor = details + .options + .get(&HeaderKey::from_str(topic_option_keys::REPLICATION_FACTOR).unwrap()) + .expect("the topic reports its replication factor"); + assert_eq!(factor.value, HeaderValue::from(1u32)); + assert!(factor.explicit); + + let mut primaries = BTreeSet::new(); + for partition_id in 0..PARTITIONS { + let route = partition_route(&writer, &stream, &topic, partition_id).await; + primaries.insert(route.primary_replica); + } + assert_eq!( + primaries.len(), + harness.cluster_size(), + "one partition on each node" + ); + + let reader = root_client(harness, 2).await; + for partition_id in 0..PARTITIONS { + let payload = format!("stored-once-{partition_id}"); + let mut messages = vec![IggyMessage::from_str(&payload).unwrap()]; + writer + .send_messages( + &stream, + &topic, + &Partitioning::partition_id(partition_id), + &mut messages, + ) + .await + .unwrap(); + let polled = timeout(POLL_BUDGET, async { + loop { + let polled = reader + .poll_messages( + &stream, + &topic, + partition_id, + &Consumer::default(), + &PollingStrategy::offset(0), + PollOptions { + count: 1, + ..PollOptions::default() + }, + ) + .await + .unwrap(); + if !polled.messages.is_empty() { + break polled; + } + sleep(POLL_INTERVAL).await; + } + }) + .await + .unwrap_or_else(|_| panic!("partition {partition_id} never served its message")); + assert_eq!(polled.messages[0].payload.as_ref(), payload.as_bytes()); + + let route = partition_route(&reader, &stream, &topic, partition_id).await; + let directory = format!( + "streams/{stream_id}/topics/{}/partitions/{partition_id}", + details.id + ); + let holders: Vec<usize> = (0..harness.cluster_size()) + .filter(|node| harness.node(*node).data_path().join(&directory).exists()) + .collect(); + assert_eq!( + holders, + [usize::from(route.primary_replica)], + "partition {partition_id} is stored only on its primary" + ); + } +} + +#[iggy_harness(cluster_nodes = 3)] +async fn given_a_factor_the_cluster_cannot_hold_when_creating_should_refuse( + harness: &mut TestHarness, +) { + let client = root_client(harness, 0).await; + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); + let stream = Identifier::named(STREAM_NAME).unwrap(); + for factor in [0, 4] { + let error = client + .create_topic( + &stream, + TOPIC_NAME, + &TopicCreateOptions { + replication_factor: Some(factor), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap_err(); + assert_eq!( + error, + IggyError::InvalidReplicationFactor, + "factor {factor} on three nodes" + ); + } +} + +/// `metric`'s value summed over every shard of every listed node. +async fn summed_gauge(sessions: &[HttpClient], metric: &str) -> i64 { + let mut total = 0; + for session in sessions { + let text = session.get("/metrics").await.text().await.unwrap(); + total += text + .lines() + .filter(|line| line.starts_with(&format!("{metric}{{"))) + .filter_map(|line| line.rsplit(' ').next()?.parse::<i64>().ok()) + .sum::<i64>(); + } + total +} + +async fn await_replication_gauges(sessions: &[HttpClient], expected: (i64, i64)) { + timeout(METRICS_BUDGET, async { + loop { + let observed = ( + summed_gauge(sessions, "partitions_under_replicated").await, + summed_gauge(sessions, "partitions_below_write_quorum").await, + ); + if observed == expected { + break; + } + sleep(POLL_INTERVAL).await; + } + }) + .await + .unwrap_or_else(|_| panic!("replication gauges never reached {expected:?}")); +} + +#[iggy_harness(cluster_nodes = 3, server(sharding.cpu_allocation = "2"))] +async fn given_a_member_down_when_scraping_metrics_should_report_its_partitions_under_replicated( + harness: &mut TestHarness, +) { + const DOWN: usize = 2; + let client = root_client(harness, 0).await; + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); + let stream = Identifier::named(STREAM_NAME).unwrap(); + let topic = Identifier::named(TOPIC_NAME).unwrap(); + client + .create_topic( + &stream, + TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS), + replication_factor: Some(3), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + for partition_id in 0..PARTITIONS { + partition_route(&client, &stream, &topic, partition_id).await; + } + let survivors = [ + HttpClient::login_root_no_redirect(node_url(harness, 0)).await, + HttpClient::login_root_no_redirect(node_url(harness, 1)).await, + ]; + await_replication_gauges(&survivors, (0, 0)).await; + + harness.kill_node(DOWN).unwrap(); + // Every partition keeps two of its three copies: short of its factor, still + // a write quorum. + await_replication_gauges(&survivors, (i64::from(PARTITIONS), 0)).await; + + harness.restart_node(DOWN).unwrap(); + await_replication_gauges(&survivors, (0, 0)).await; +} + +/// Every login below must succeed within this; ended sessions free their +/// slots only as fast as finalization releases them. +const LOGIN_BUDGET: Duration = Duration::from_secs(30); + +async fn login_within_budget(harness: &TestHarness, node: usize) -> TcpClient { + timeout(LOGIN_BUDGET, async { + loop { + let client = connect_without_login(harness.node(node).tcp_addr().unwrap()).await; + if client + .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD) + .await + .is_ok() + { + break client; + } + sleep(POLL_INTERVAL).await; + } + }) + .await + .expect("a login must not wait on a partition that lacks its member quorum") +} + +async fn send_within_budget( + client: &TcpClient, + stream: &Identifier, + topic: &Identifier, + partition_id: u32, + payload: &str, +) { + send_within(client, stream, topic, partition_id, payload, POLL_BUDGET).await; +} + +async fn send_within( + client: &TcpClient, + stream: &Identifier, + topic: &Identifier, + partition_id: u32, + payload: &str, + budget: Duration, +) { + timeout(budget, async { + loop { + let mut messages = vec![IggyMessage::from_str(payload).unwrap()]; + if client + .send_messages( + stream, + topic, + &Partitioning::partition_id(partition_id), + &mut messages, + ) + .await + .is_ok() + { + break; + } + sleep(POLL_INTERVAL).await; + } + }) + .await + .unwrap_or_else(|_| panic!("partition {partition_id} never accepted {payload}")); +} + +#[iggy_harness(cluster_nodes = 3, server(sharding.cpu_allocation = "2"))] +async fn given_a_factor_of_one_when_one_reader_fetches_every_partition_should_return_each_message( + harness: &mut TestHarness, +) { + let writer = root_client(harness, 0).await; + writer + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); + let stream = Identifier::named(STREAM_NAME).unwrap(); + let topic = Identifier::named(TOPIC_NAME).unwrap(); + writer + .create_topic( + &stream, + TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS), + replication_factor: Some(1), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + let mut expected = BTreeSet::new(); + for partition_id in 0..PARTITIONS { + partition_route(&writer, &stream, &topic, partition_id).await; + let payload = format!("partition-{partition_id}"); + send_within_budget(&writer, &stream, &topic, partition_id, &payload).await; + expected.insert(payload); + } + + // One fetch names partitions that live on three different nodes. + let client = IggyClient::new(ClientWrapper::Tcp(root_client(harness, 0).await)); + let mut reader = client + .reader( + &stream, + &topic, + &Consumer::new(Identifier::named("every-partition").unwrap()), + ) + .partitions(0..PARTITIONS) + .start(PollingStrategy::first()) + .build() + .await + .unwrap(); + let mut read = BTreeSet::new(); + timeout(POLL_BUDGET, async { + while read.len() < expected.len() { + let batch = reader.next().await.unwrap(); + for message in batch.messages { + read.insert(String::from_utf8(message.payload.to_vec()).unwrap()); + } + } + }) + .await + .expect("the reader reaches every partition"); + assert_eq!(read, expected); +} + +#[iggy_harness( + cluster_nodes = 3, + server(metadata.clients_table_max = "8", sharding.cpu_allocation = "2") +)] +async fn given_an_rf1_node_down_when_sessions_churn_should_keep_admitting_logins( + harness: &mut TestHarness, +) { + const DOWN: usize = 2; + const ROUNDS: usize = 24; + let admin = root_client(harness, 0).await; + admin + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); + let stream = Identifier::named(STREAM_NAME).unwrap(); + let topic = Identifier::named(TOPIC_NAME).unwrap(); + admin + .create_topic( + &stream, + TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS), + replication_factor: Some(1), + durability: Durability::Persisted, + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + let mut stranded = None; + for partition_id in 0..PARTITIONS { + let route = partition_route(&admin, &stream, &topic, partition_id).await; + if usize::from(route.primary_replica) == DOWN { + stranded = Some(partition_id); + } + } + let stranded = stranded.expect("placement gives every node one partition"); + + // This session's receipt lives only on the node about to go down, so its + // retirement there can only be owed. + let writer = root_client(harness, 0).await; + send_within_budget(&writer, &stream, &topic, stranded, "before-outage").await; + harness.kill_node(DOWN).unwrap(); + writer.logout_user().await.unwrap(); + drop(writer); + + for _ in 0..ROUNDS { + let client = login_within_budget(harness, 0).await; + client.logout_user().await.unwrap(); + } + + harness.restart_node(DOWN).unwrap(); + let returned = [HttpClient::login_root_no_redirect(node_url(harness, DOWN)).await]; + timeout(METRICS_BUDGET, async { + while summed_gauge(&returned, "retirement_debts_paid_total").await == 0 { + sleep(POLL_INTERVAL).await; + } + }) + .await + .expect("the returning primary pays the retirement it owes"); + let returning = login_within_budget(harness, 0).await; + send_within_budget(&returning, &stream, &topic, stranded, "after-outage").await; + let polled = timeout(POLL_BUDGET, async { + loop { + if let Ok(polled) = returning + .poll_messages( + &stream, + &topic, + stranded, + &Consumer::default(), + &PollingStrategy::offset(0), + PollOptions { + count: 2, + ..PollOptions::default() + }, + ) + .await + && polled.messages.len() == 2 + { + break polled; + } + sleep(POLL_INTERVAL).await; + } + }) + .await + .expect("the returning partition serves both messages once its debt is paid"); + assert_eq!(polled.messages[1].payload.as_ref(), b"after-outage"); + timeout(METRICS_BUDGET, async { + while summed_gauge(&returned, "partitions_owing_retirement").await != 0 { + sleep(POLL_INTERVAL).await; + } + }) + .await + .expect("no partition keeps refusing sessions once its debt is paid"); +} + +/// The partitions of the topic, once every one of them lists all its members +/// in sync, read through `node`. +async fn await_in_sync( + client: &TcpClient, + stream: &Identifier, + topic: &Identifier, + expected: impl Fn(&Partition) -> Vec<u8>, +) -> Vec<Partition> { + timeout(POLL_BUDGET, async { + loop { + let partitions = client + .get_topic(stream, topic) + .await + .unwrap() + .unwrap() + .partitions; + if partitions + .iter() + .all(|partition| partition.in_sync_members == expected(partition)) + { + return partitions; + } + sleep(POLL_INTERVAL).await; + } + }) + .await + .unwrap_or_else(|_| panic!("in-sync members never reached the expected lists")) +} + +#[iggy_harness(cluster_nodes = 3, server(sharding.cpu_allocation = "2"))] +async fn given_no_factor_when_creating_on_the_default_setting_should_store_every_partition_on_every_node( + harness: &mut TestHarness, +) { + let client = root_client(harness, 0).await; + let specs = client.describe_options(OptionsScope::Topic).await.unwrap(); + let replication_factor = specs + .iter() + .find(|spec| spec.key == "replication_factor") + .unwrap(); + assert_eq!(replication_factor.default_value, 3_u32.to_le_bytes()); + + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); + let stream = Identifier::named(STREAM_NAME).unwrap(); + let topic = Identifier::named(TOPIC_NAME).unwrap(); + client + .create_topic( + &stream, + TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + let partitions = await_in_sync(&client, &stream, &topic, |partition| { + partition.members.clone() + }) + .await; + for partition in &partitions { + let mut members = partition.members.clone(); + members.sort_unstable(); + assert_eq!(members, [0, 1, 2], "partition {}", partition.id); + } +} + +#[iggy_harness( + cluster_nodes = 3, + server( + cluster.default_replication_factor = "2", + sharding.cpu_allocation = "2" + ) +)] +async fn given_a_configured_default_factor_when_a_member_goes_down_should_report_it_out_of_sync( + harness: &mut TestHarness, +) { + const DOWN: u8 = 2; + let client = root_client(harness, 0).await; + let specs = client.describe_options(OptionsScope::Topic).await.unwrap(); + let replication_factor = specs + .iter() + .find(|spec| spec.key == "replication_factor") + .unwrap(); + assert_eq!(replication_factor.default_value, 2_u32.to_le_bytes()); + + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); + let stream = Identifier::named(STREAM_NAME).unwrap(); + let topic = Identifier::named(TOPIC_NAME).unwrap(); + client + .create_topic( + &stream, + TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(PARTITIONS), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + let details = client.get_topic(&stream, &topic).await.unwrap().unwrap(); + let factor = details + .options + .get(&HeaderKey::from_str(topic_option_keys::REPLICATION_FACTOR).unwrap()) + .expect("the topic stores the factor it was given"); + assert_eq!(factor.value, HeaderValue::from(2_u32)); + + let partitions = await_in_sync(&client, &stream, &topic, |partition| { + partition.members.clone() + }) + .await; + let distinct: BTreeSet<u8> = partitions + .iter() + .flat_map(|partition| partition.members.iter().copied()) + .collect(); + assert!(distinct.len() > 1, "the members spread over the nodes"); + for partition in &partitions { + assert_eq!(partition.members.len(), 2, "partition {}", partition.id); + } + assert!( + partitions + .iter() + .any(|partition| partition.members.contains(&DOWN)), + "three partitions of two members each use every node of three" + ); + + harness.kill_node(usize::from(DOWN)).unwrap(); + await_in_sync(&client, &stream, &topic, |partition| { + partition + .members + .iter() + .copied() + .filter(|member| *member != DOWN) + .collect() + }) + .await; +} + +/// Past the clock skew a node allows between a partition's creation and the +/// start of its own storage, so a node that comes back empty knows it lost +/// the partition rather than never having had it. +const STORAGE_EPOCH_SKEW: Duration = Duration::from_secs(11); +/// Rejoining on empty storage takes a metadata catch-up, an election and a +/// partition state transfer before a write commits again. +const REJOIN_BUDGET: Duration = Duration::from_secs(120); +/// Consecutive polls, 5 seconds, a check must pass: past the 3 second probe +/// timeout, after which a new metadata primary narrows every in-sync list to +/// the nodes it hears. +const STEADY_POLLS: u32 = 25; + +/// Waits until partition 0 of `topic` lists in-sync members that pass `check` +/// on [`STEADY_POLLS`] polls in a row. +async fn await_steady_in_sync( + client: &TcpClient, + topic: &Identifier, + check: impl Fn(&[u8]) -> bool, +) { + let stream = Identifier::named(STREAM_NAME).unwrap(); + timeout(REJOIN_BUDGET, async { + let mut passed = 0; + while passed < STEADY_POLLS { + let in_sync = client + .get_topic(&stream, topic) + .await + .ok() + .flatten() + .map(|details| details.partitions[0].in_sync_members.clone()); + passed = if in_sync.is_some_and(|in_sync| check(&in_sync)) { + passed + 1 + } else { + 0 + }; + sleep(POLL_INTERVAL).await; + } + }) + .await + .unwrap_or_else(|_| panic!("the in-sync list of {topic} never settled")); +} + +async fn await_steady_replication_gauges(sessions: &[HttpClient], expected: (i64, i64)) { + timeout(REJOIN_BUDGET, async { + let mut passed = 0; + while passed < STEADY_POLLS { + let observed = ( + summed_gauge(sessions, "partitions_under_replicated").await, + summed_gauge(sessions, "partitions_below_write_quorum").await, + ); + passed = if observed == expected { passed + 1 } else { 0 }; + sleep(POLL_INTERVAL).await; + } + }) + .await + .unwrap_or_else(|_| panic!("replication gauges never settled at {expected:?}")); +} + +async fn create_factor_two_topic(client: &TcpClient, durability: Durability) { + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .unwrap(); + client + .create_topic( + &Identifier::named(STREAM_NAME).unwrap(), + TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(1), + replication_factor: Some(2), + durability, + consumer_offset_durability: durability, + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); +} + +/// Replacing one member's disk at a time with `--fresh` is the documented +/// procedure. On a factor-2 topic whose writes wait for the disk, the member +/// that kept its data must still elect and copy the history back. +#[iggy_harness(cluster_nodes = 3, server(sharding.cpu_allocation = "2"))] +async fn given_a_factor_of_two_when_its_primary_returns_on_empty_storage_should_serve_again( + harness: &mut TestHarness, +) { + let stream = Identifier::named(STREAM_NAME).unwrap(); + let topic = Identifier::named(TOPIC_NAME).unwrap(); + let admin = root_client(harness, 0).await; + create_factor_two_topic(&admin, Durability::Persisted).await; + let primary = usize::from( + partition_route(&admin, &stream, &topic, 0) + .await + .primary_replica, + ); + send_within_budget(&admin, &stream, &topic, 0, "before-wipe").await; + drop(admin); + sleep(STORAGE_EPOCH_SKEW).await; + + harness.restart_node_from_clean_slate(primary).unwrap(); + let survivor = (0..3).find(|node| *node != primary).unwrap(); + let client = login_within_budget(harness, survivor).await; + let polled = timeout(REJOIN_BUDGET, async { + let mut sent = false; + loop { + if !sent { + let mut messages = vec![IggyMessage::from_str("after-wipe").unwrap()]; + sent = client + .send_messages( + &stream, + &topic, + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .is_ok(); + } + if sent + && let Ok(polled) = client + .poll_messages( + &stream, + &topic, + 0, + &Consumer::default(), + &PollingStrategy::offset(0), + PollOptions { + count: 2, + ..PollOptions::default() + }, + ) + .await + && polled.messages.len() == 2 + { + break polled; + } + sleep(POLL_INTERVAL).await; + } + }) + .await + .expect("the member that kept its data elects and the partition serves again"); + let payloads: Vec<_> = polled + .messages + .iter() + .map(|message| message.payload.as_ref().to_vec()) + .collect(); + assert_eq!(payloads, [b"before-wipe".to_vec(), b"after-wipe".to_vec()]); +} + +/// Cut off long enough that a primary hearing no member steps down: past the +/// 5 second `heartbeat_timeout`. +const STEP_DOWN_ISOLATION: Duration = Duration::from_secs(8); +/// Views the pair goes through before the rebuilt member must have led one. +const STEP_DOWN_ROUNDS: usize = 6; + +/// A member rebuilt by a state transfer after it came back on empty storage +/// must elect like any other. Restarting the other member moves the pair to +/// new views until the rebuilt member leads one. It is then cut off from its +/// peers until it steps down, and the pair must elect again after the heal. +#[tokio::test(flavor = "multi_thread")] +#[parallel] +async fn given_a_member_rebuilt_by_state_transfer_when_each_leader_steps_down_should_elect_again() { + let mut harness = relayed_cluster(); + harness.start().await.expect("start the cluster"); + let stream = Identifier::named(STREAM_NAME).unwrap(); + let topic = Identifier::named(TOPIC_NAME).unwrap(); + let admin = root_client(&harness, 0).await; + create_factor_two_topic(&admin, Durability::Persisted).await; + let route = partition_route(&admin, &stream, &topic, 0).await; + let rebuilt = route.primary_replica; + let members = admin + .get_topic(&stream, &topic) + .await + .unwrap() + .unwrap() + .partitions[0] + .members + .clone(); + let outsider = (0..3) + .find(|node| !members.contains(node)) + .expect("a factor of two leaves one of three nodes out"); + send_within_budget(&admin, &stream, &topic, 0, "before-wipe").await; + drop(admin); + sleep(STORAGE_EPOCH_SKEW).await; + + harness + .restart_node_from_clean_slate(usize::from(rebuilt)) + .unwrap(); + let client = login_within_budget(&harness, usize::from(outsider)).await; + // Both members ack every write, so this commits only once the transfer + // gave the rebuilt member the history. + send_within(&client, &stream, &topic, 0, "after-wipe", REJOIN_BUDGET).await; + + let mut view = route.view; + let mut rebuilt_led = false; + for round in 0..STEP_DOWN_ROUNDS { + let route = timeout(REJOIN_BUDGET, async { + loop { + let route = partition_route(&client, &stream, &topic, 0).await; + if route.view > view { + break route; + } + sleep(POLL_INTERVAL).await; + } + }) + .await + .expect("the pair's new view reaches the outsider"); + view = route.view; + let primary = usize::from(route.primary_replica); + if route.primary_replica == rebuilt { + harness.isolate_node(primary).unwrap(); + sleep(STEP_DOWN_ISOLATION).await; + harness.heal_node(primary).unwrap(); + send_within(&client, &stream, &topic, 0, "stepped-down", REJOIN_BUDGET).await; + rebuilt_led = true; + break; + } + harness.restart_node(primary).unwrap(); + send_within( + &client, + &stream, + &topic, + 0, + &format!("round-{round}"), + REJOIN_BUDGET, + ) + .await; + } + assert!(rebuilt_led, "node {rebuilt} never led the pair"); +} + +fn relayed_cluster() -> TestHarness { + let overrides = HashMap::from([("sharding.cpu_allocation".to_owned(), "2".to_owned())]); + TestHarness::builder() + .server( + TestServerConfig::builder() + .extra_envs(resolve_config_paths(&overrides).expect("valid server config paths")) + .build(), + ) + .cluster_nodes(3) + .peer_relays() + .build() + .expect("build the harness") +} + +/// Both members of a partition come back on empty storage. Neither serves +/// it, since that would restart it at offset 0, and deleting the topic, the +/// way to accept the loss, must complete and free its name. +#[iggy_harness(cluster_nodes = 5, server(sharding.cpu_allocation = "2"))] +async fn given_a_factor_of_two_when_both_members_return_on_empty_storage_should_let_its_topic_be_deleted_and_recreated( + harness: &mut TestHarness, +) { + const NODES: usize = 5; + let stream = Identifier::named(STREAM_NAME).unwrap(); + let topic = Identifier::named(TOPIC_NAME).unwrap(); + let creator = root_client(harness, 0).await; + create_factor_two_topic(&creator, Durability::Replicated).await; + let members = creator + .get_topic(&stream, &topic) + .await + .unwrap() + .unwrap() + .partitions[0] + .members + .clone(); + assert_eq!(members.len(), 2); + send_within_budget(&creator, &stream, &topic, 0, "lost").await; + let witness = Identifier::named(WITNESS_TOPIC_NAME).unwrap(); + creator + .create_topic( + &stream, + WITNESS_TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(1), + replication_factor: Some(u32::try_from(NODES).unwrap()), + ..TopicCreateOptions::default() + }, + ) + .await + .unwrap(); + drop(creator); + let outsider = (0..NODES) + .find(|node| !members.contains(&u8::try_from(*node).unwrap())) + .unwrap(); + let admin = root_client(harness, outsider).await; + sleep(STORAGE_EPOCH_SKEW).await; + + for &member in &members { + harness.stop_node(usize::from(member)).unwrap(); + } + for &member in &members { + harness + .restart_node_from_clean_slate(usize::from(member)) + .unwrap(); + } + for &member in &members { + drop(login_within_budget(harness, usize::from(member)).await); + } + // The metadata primary lists a node in sync only while it hears the node, + // so the partition every node stores shows when it hears every node again. + await_steady_in_sync(&admin, &witness, |in_sync| in_sync.len() == NODES).await; + await_steady_in_sync(&admin, &topic, <[u8]>::is_empty).await; + let mut sessions = Vec::with_capacity(NODES); + for node in 0..NODES { + sessions.push(HttpClient::login_root_no_redirect(node_url(harness, node)).await); + } + // Neither member leads the lost pair, whether or not the metadata primary + // still holds the last report from before the wipe. + await_steady_replication_gauges(&sessions, (1, 1)).await; + + timeout(REJOIN_BUDGET, async { + loop { + let _ = admin.delete_topic(&stream, &topic).await; + if matches!(admin.get_topic(&stream, &topic).await, Ok(None)) { + break; + } + sleep(POLL_INTERVAL).await; + } + }) + .await + .expect("the delete completes once both members retired their lost copies"); + create_factor_two_topic_in_stream(&admin).await; + send_within_budget(&admin, &stream, &topic, 0, "recreated").await; +} + +async fn create_factor_two_topic_in_stream(client: &TcpClient) { + timeout(POLL_BUDGET, async { + loop { + let created = client + .create_topic( + &Identifier::named(STREAM_NAME).unwrap(), + TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(1), + replication_factor: Some(2), + ..TopicCreateOptions::default() + }, + ) + .await; + if created.is_ok() { + break; + } + sleep(POLL_INTERVAL).await; + } + }) + .await + .expect("the freed name takes a new topic"); +}
diff --git a/core/integration/tests/data_integrity/storage_compat.rs b/core/integration/tests/data_integrity/storage_compat.rs index ff53a4f..d1390ac 100644 --- a/core/integration/tests/data_integrity/storage_compat.rs +++ b/core/integration/tests/data_integrity/storage_compat.rs
@@ -1540,6 +1540,7 @@ preallocate_segments: seed .preallocate_segments .and(recovered.preallocate_segments), + replication_factor: seed.replication_factor.and(recovered.replication_factor), raw: BTreeMap::new(), }; assert_eq!(
diff --git a/core/integration/tests/sdk/options.rs b/core/integration/tests/sdk/options.rs index 84fa26b..f4d3e43 100644 --- a/core/integration/tests/sdk/options.rs +++ b/core/integration/tests/sdk/options.rs
@@ -104,6 +104,13 @@ assert_eq!(flags("max_topic_size"), (true, true, false, false)); assert_eq!(flags("compression_algorithm"), (false, false, false, false)); assert_eq!(flags("segment_size"), (false, false, false, false)); + // The default replication factor is the server's `"all"`: every node. + let nodes = u32::try_from(harness.cluster_size()).unwrap(); + let replication_factor = specs + .iter() + .find(|spec| spec.key == "replication_factor") + .unwrap(); + assert_eq!(replication_factor.default_value, nodes.to_le_bytes()); } #[iggy_harness]
diff --git a/core/message_bus/src/replica/handshake.rs b/core/message_bus/src/replica/handshake.rs index 90cddb7..a436e81 100644 --- a/core/message_bus/src/replica/handshake.rs +++ b/core/message_bus/src/replica/handshake.rs
@@ -578,11 +578,25 @@ }) } -/// Protocol, release and storage identity, independent of executable packaging. +/// Protocol, replica wire, release and storage identity, independent of +/// executable packaging. #[must_use] pub fn binary_identity(release: &str, storage_format: &[u8]) -> [u8; auth::IDENTITY_LEN] { + identity_at( + iggy_binary_protocol::REPLICA_WIRE_REVISION, + release, + storage_format, + ) +} + +fn identity_at( + wire_revision: u32, + release: &str, + storage_format: &[u8], +) -> [u8; auth::IDENTITY_LEN] { let mut hasher = blake3::Hasher::new_derive_key("apache.iggy replica identity v2"); hasher.update(&iggy_binary_protocol::IGGY_PROTOCOL_VERSION.to_le_bytes()); + hasher.update(&wire_revision.to_le_bytes()); hasher.update(&(release.len() as u64).to_le_bytes()); hasher.update(release.as_bytes()); hasher.update(&(storage_format.len() as u64).to_le_bytes()); @@ -643,6 +657,15 @@ assert_ne!(identity, binary_identity("release-2", b"storage-1")); assert_ne!(identity, binary_identity("release-1", b"storage-2")); assert_ne!( + identity, + super::identity_at( + iggy_binary_protocol::REPLICA_WIRE_REVISION + 1, + "release-1", + b"storage-1" + ), + "builds that send replicas different encodings refuse each other" + ); + assert_ne!( binary_identity("release", b"-storage"), binary_identity("release-", b"storage") );
diff --git a/core/metadata/benches/change_log.rs b/core/metadata/benches/change_log.rs index 9a65834..3242a54 100644 --- a/core/metadata/benches/change_log.rs +++ b/core/metadata/benches/change_log.rs
@@ -172,6 +172,7 @@ partition_id, consensus_group_id: 1 + u64::from(partition_id), initial_view: 0, + members: Vec::new(), }) .collect(), },
diff --git a/core/metadata/src/change_log/current.rs b/core/metadata/src/change_log/current.rs index 7942ae0..c987cc6 100644 --- a/core/metadata/src/change_log/current.rs +++ b/core/metadata/src/change_log/current.rs
@@ -506,6 +506,7 @@ partition_id: 0, consensus_group_id: partition_group, initial_view: 0, + members: Vec::new(), }], }; assert_eq!(StateHandler::apply(&request, &mut streams, now).code, 0);
diff --git a/core/metadata/src/change_log/tests.rs b/core/metadata/src/change_log/tests.rs index 7c935b0..c6481a1 100644 --- a/core/metadata/src/change_log/tests.rs +++ b/core/metadata/src/change_log/tests.rs
@@ -2127,6 +2127,7 @@ partition_id, consensus_group_id: first_group + u64::from(partition_id), initial_view: 0, + members: Vec::new(), }) .collect() }
diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index f6b7931..20288cd 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs
@@ -21,7 +21,7 @@ AppliedChange, ChangeLogCapacities, ChangeLogSnapshot, CurrentState, MetadataChangeLog, change_actor, change_page, current_page, session_closed, session_registered, session_revoked, }; -use crate::impls::placement::PlacementState; +use crate::impls::placement::{PlacementCounts, PlacementState}; use crate::stm::authz::{ PartitionsCreate, admits_partitions_create, gated_apply, reject_stale_session, }; @@ -60,7 +60,7 @@ UpdateStreamResolvedRequest, }; use iggy_binary_protocol::requests::system::{ - FinalizeSessionRequest, ResourceTarget, SessionIdentity, UpdateResourceRequest, + FinalizeSessionRequest, ResourceTarget, RetirementDebt, SessionIdentity, UpdateResourceRequest, }; use iggy_binary_protocol::requests::topics::CreateTopicRequest as WireCreateTopicRequest; use iggy_binary_protocol::requests::topics::CreateTopicWithAssignmentsRequest as PersistedCreateTopicRequest; @@ -77,11 +77,11 @@ use iggy_common::variadic; use iggy_common::wire_conversions::{resource_options_from_wire, resource_options_to_wire}; use iggy_common::{ - HeaderKey, IggyError, MetadataChangeStart, MetadataChangesQuery, OptionAdmissionContext, - OptionKey, OptionOperation, OptionValue, OptionVisibility, OptionsProvenance, OptionsScope, - ResourceOptions, ServerKeyContext, TopicCreateOptions, TopicResolvedOptions, UserId, admit, - critical_keys_marker, derived_defaults, stream_wrap_set, topic_option_keys, - validate_topic_segment_size, + HeaderKey, HeaderValue, IggyError, MetadataChangeStart, MetadataChangesQuery, + OptionAdmissionContext, OptionKey, OptionOperation, OptionValue, OptionVisibility, + OptionsProvenance, OptionsScope, ResourceOptions, ServerKeyContext, TopicCreateOptions, + TopicResolvedOptions, UserId, admit, critical_keys_marker, derived_defaults, stream_wrap_set, + topic_option_keys, validate_topic_segment_size, }; use journal::local_gate::LocalGate; use journal::superblock::{ @@ -902,8 +902,9 @@ /// The backup commit walk's per-op logic, shared so the simulator's WAL /// reconstruction reaches identical state from the same log through one apply path. /// Register creates an identity or resolves its matching retry. Logout ends the -/// exact session and rebalances its memberships; `FinalizeSession` removes it after -/// partition retirement. Every other op applies to the state +/// exact session and rebalances its memberships; `FinalizeSession` removes it once +/// every partition group retired it or records that group's retirement debt. Every +/// other op applies to the state /// machine and caches the reply for at-most-once dedup. `fire_notifier` runs the /// post-commit hook (a no-op during reconstruction, before it is wired). Returns /// the committed reply without advancing `commit_min`; the caller owns that counter. @@ -1058,11 +1059,13 @@ "invalid session finalization", ) }); - if table_mutations_allowed - && mux_stm.streams().read(|inner| inner.namespace_revision) - == request.namespace_revision - { - client_table.borrow_mut().finalize_session(request.identity); + if mux_stm.streams().read(|inner| inner.namespace_revision) == request.namespace_revision { + if table_mutations_allowed { + client_table.borrow_mut().finalize_session(request.identity); + } + mux_stm + .streams() + .record_retirement_debt(&request.debt, header.op); } return CommittedApply { reply: build_reply_message(&header, &bytes::Bytes::new()), @@ -1286,6 +1289,9 @@ /// `[personal_access_token] max_tokens_per_user`, zero for no cap. See /// [`Self::admit_personal_access_token`]. max_tokens_per_user: Cell<u32>, + /// `[cluster] default_replication_factor`, `None` for every node. See + /// [`Self::default_replication_factor`]. + default_replication_factor: Cell<Option<u8>>, /// Highest metadata op whose apply has been PUBLISHED on this node, plus /// the reads parked on it. Shared by every shard; see /// [`AppliedFrontier`] for the ordering and the wake contract. @@ -1362,6 +1368,7 @@ prepare_gap_drops: Cell::new(0), partitions_max: Cell::new(0), max_tokens_per_user: Cell::new(0), + default_replication_factor: Cell::new(None), applied_frontier: Arc::default(), stream_key_context: Arc::default(), change_log: Rc::new(MetadataChangeLog::new(ChangeLogCapacities::default())), @@ -1653,6 +1660,48 @@ self.partitions_max.set(partitions_max); } + /// Give a topic created without a replication factor `factor` nodes, or + /// every node for `None`, from `[cluster] default_replication_factor`. Only + /// the primary reads it, when it admits a `CreateTopic`, and the topic + /// stores the factor it resolved, so every node needs the same value. + pub fn set_default_replication_factor(&self, factor: Option<u8>) { + self.default_replication_factor.set(factor); + } + + /// The failure domain of each node, indexed by node id, from `[cluster] + /// nodes`. Placement spreads a partition's members over as many as it can. + /// Only the primary reads it, when it admits a create, so every node needs + /// the same list. + pub fn set_node_racks(&self, racks: &[Option<String>]) { + let mut names: Vec<&str> = Vec::new(); + let domains = racks + .iter() + .map(|rack| { + rack.as_deref().map(|rack| { + let index = names + .iter() + .position(|known| *known == rack) + .unwrap_or_else(|| { + names.push(rack); + names.len() - 1 + }); + u32::try_from(index).unwrap_or(u32::MAX) + }) + }) + .collect(); + self.placement.borrow_mut().set_racks(domains); + } + + /// The replication factor a topic created without one gets in a cluster + /// of `nodes` nodes: the configured number lowered to `nodes`, or every + /// node. + #[must_use] + pub fn default_replication_factor(&self, nodes: u8) -> u8 { + self.default_replication_factor + .get() + .map_or(nodes, |factor| factor.min(nodes)) + } + /// Cap the personal access tokens of each user at `[personal_access_token] /// max_tokens_per_user`, zero for no cap. Only the primary checks it, with /// the value of its own node, when it admits a `CreatePersonalAccessToken`, @@ -3238,8 +3287,9 @@ .map_err(|Canceled| MetadataSubmitError::Canceled) } - /// Finalize only after the caller proves a quorum has retired this exact - /// ended session in every currently allocated partition group. + /// Finalize once every currently allocated partition group either proved + /// this exact ended session retired on a quorum of its members or is + /// listed in `debt`, which its next primary pays before serving sessions. /// /// # Errors /// Returns an admission or consensus error before finalization commits. @@ -3248,6 +3298,7 @@ &self, identity: SessionIdentity, namespace_revision: u64, + debt: Vec<RetirementDebt>, ) -> Result<Option<u64>, MetadataSubmitError> { let consensus = self .consensus @@ -3291,6 +3342,7 @@ &FinalizeSessionRequest { identity, namespace_revision, + debt, } .to_bytes(), ), @@ -4800,7 +4852,7 @@ Ok(()) } - fn placement_counts(&self) -> Result<Vec<u64>, IggyError> { + fn placement_counts(&self) -> Result<PlacementCounts, IggyError> { let headers = self.pending_placement_headers()?; let consensus = self.consensus.as_ref().unwrap(); let mut placement = self.placement.borrow_mut(); @@ -4812,6 +4864,38 @@ Ok(counts) } + /// The committed topic's replication factor. `None` for a topic created + /// before replication factors, or one that does not exist. A topic not + /// committed yet refuses transiently while a create waits in the pipeline: + /// that create may be the one named, and its factor is unknown until it + /// commits. + fn committed_replication_factor( + &self, + stream_id: &WireIdentifier, + topic_id: &WireIdentifier, + ) -> Result<Option<u8>, IggyError> { + let committed = self.mux_stm.streams().read(|inner| { + let stream = inner.resolve_stream_id(stream_id)?; + let topic = inner.resolve_topic_id(stream, topic_id)?; + let topic = inner.items.get(stream)?.topics.get(topic)?; + Some(TopicCreateOptions::from_resource_options(&topic.options).replication_factor) + }); + let Some(factor) = committed else { + let creating_topic = self + .pending_placement_headers()? + .iter() + .any(|header| header.operation == Operation::CreateTopicWithAssignments); + return if creating_topic { + Err(IggyError::TransientNotAccepted) + } else { + Ok(None) + }; + }; + factor + .map(|factor| u8::try_from(factor).map_err(|_| IggyError::InvalidReplicationFactor)) + .transpose() + } + /// Partitions the creates in the prepare pipeline add when they commit. fn pending_partitions(&self) -> Result<u64, IggyError> { let headers = self.pending_placement_headers()?; @@ -4937,7 +5021,14 @@ // canonicalizes kinds (a `"128MiB"` string becomes `Uint64`). request.options = resource_options_to_wire(&admitted.explicit, OptionsProvenance::All)?; - let derived = derived_defaults(OptionsScope::Topic, &admitted.explicit)?; + let mut derived = derived_defaults(OptionsScope::Topic, &admitted.explicit)?; + let replication_factor = resolve_replication_factor( + TopicCreateOptions::from_resource_options(&admitted.explicit) + .replication_factor, + self.default_replication_factor(consensus.replica_count()), + consensus.replica_count(), + &mut derived, + )?; let mut stored = derived.clone(); stored.extend(admitted.explicit.clone()); let resolved = TopicResolvedOptions::from(&stored); @@ -4966,14 +5057,17 @@ .into_iter() .enumerate() .map(|(partition_id, consensus_group_id)| { + let (members, initial_view) = self.placement.borrow().select( + &mut counts, + consensus.view(), + Some(replication_factor), + )?; Ok(CreatedPartitionAssignment { partition_id: u32::try_from(partition_id) .map_err(|_| IggyError::InvalidCommand)?, consensus_group_id, - initial_view: self - .placement - .borrow() - .select(&mut counts, consensus.view())?, + initial_view, + members, }) }) .collect::<Result<Vec<_>, _>>()?; @@ -4997,7 +5091,11 @@ // `CreatePartitionsResult::{Stream,Topic}NotFound`. A preflight // read here would decide against possibly-uncommitted state and // drop without a reply (TOCTOU + wedge). See the metadata - // validation design doc. + // validation design doc. The replication factor comes from the + // committed topic: a missing topic gets no member list, and + // apply answers it with `TopicNotFound`. + let replication_factor = + self.committed_replication_factor(&request.stream_id, &request.topic_id)?; let mut counts = self.placement_counts()?; let partitions = self .allocator @@ -5005,14 +5103,17 @@ .into_iter() .enumerate() .map(|(offset, consensus_group_id)| { + let (members, initial_view) = self.placement.borrow().select( + &mut counts, + consensus.view(), + replication_factor, + )?; Ok(CreatedPartitionAssignment { partition_id: u32::try_from(offset) .map_err(|_| IggyError::InvalidCommand)?, consensus_group_id, - initial_view: self - .placement - .borrow() - .select(&mut counts, consensus.view())?, + initial_view, + members, }) }) .collect::<Result<Vec<_>, _>>()?; @@ -5463,6 +5564,35 @@ ) } +/// A new topic's replication factor: the explicit value, which must fit the +/// cluster's `nodes`, or this node's `default`. The default is written back +/// into `derived`, so the topic stores what placement used. +fn resolve_replication_factor( + explicit: Option<u32>, + default: u8, + nodes: u8, + derived: &mut ResourceOptions, +) -> Result<u8, IggyError> { + let factor = explicit.unwrap_or_else(|| { + let factor = u32::from(default); + derived.insert( + HeaderKey::from(OptionKey::ReplicationFactor), + OptionValue::derived(HeaderValue::from(factor)), + ); + factor + }); + match u8::try_from(factor) { + Ok(factor) if factor >= 1 && factor <= nodes => Ok(factor), + _ => { + debug!( + replication_factor = factor, + nodes, "refusing a topic whose replication factor the cluster cannot hold" + ); + Err(IggyError::InvalidReplicationFactor) + } + } +} + /// Rewrite a generic patch into the resolved operation of its scope: the /// column values extracted, the set keys canonical and explicit, each reset key /// resolved to its catalog default in the derived block. @@ -7064,6 +7194,27 @@ } #[test] + fn given_a_configured_default_factor_when_admitting_a_create_without_one_should_place_that_many_members() + { + for (configured, expected) in [(None, 3), (Some(2), 2), (Some(5), 3)] { + let metadata = placement_metadata(); + metadata.set_default_replication_factor(configured); + assert_eq!(metadata.default_replication_factor(3), expected); + let prepare = metadata + .prepare_request(placement_request(1, "topic", 2)) + .unwrap(); + let request = PersistedCreateTopicRequest::decode_from(prepare.body()).unwrap(); + assert!( + request + .partitions + .iter() + .all(|partition| partition.members.len() == usize::from(expected)), + "configured {configured:?}" + ); + } + } + + #[test] fn given_an_offline_replica_when_admitting_creates_should_spread_over_responsive_replicas() { let metadata = placement_metadata(); metadata.observe_replica_readiness(&[false, true, true]); @@ -7108,6 +7259,38 @@ } #[test] + fn given_a_pending_create_topic_when_admitting_create_partitions_should_refuse_transiently() { + let add_partitions = || { + let body = WireCreatePartitionsRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::named("pending").unwrap(), + partitions_count: 1, + } + .to_bytes(); + pat_request(2, 1, Operation::CreatePartitions, &body) + }; + let metadata = placement_metadata(); + assert!( + metadata.prepare_request(add_partitions()).is_ok(), + "a topic nobody is creating is apply's TopicNotFound" + ); + + let metadata = placement_metadata(); + let create = metadata + .prepare_request(placement_request(1, "pending", 2)) + .unwrap(); + metadata + .consensus + .as_ref() + .unwrap() + .pipeline_message(PlaneKind::Metadata, &create); + assert!(matches!( + metadata.prepare_request(add_partitions()), + Err(IggyError::TransientNotAccepted) + )); + } + + #[test] fn given_rolled_back_placement_when_admitting_another_batch_should_release_reservations() { let metadata = placement_metadata(); let consensus = metadata.consensus.as_ref().unwrap(); @@ -7654,7 +7837,19 @@ } } assert!(!expected.is_empty(), "the topic catalog has defaulted rows"); + // The factor default is lowered to this single-node cluster. + expected.insert( + HeaderKey::from(OptionKey::ReplicationFactor), + OptionValue::derived(HeaderValue::from(1u32)), + ); assert_eq!(derived, expected); + assert!( + persisted + .partitions + .iter() + .all(|partition| partition.members == [0]), + "every partition lists the one node" + ); } #[test] @@ -8822,6 +9017,7 @@ partition_id: 0, consensus_group_id: 1, initial_view: view, + members: Vec::new(), }], }, inner, @@ -10433,7 +10629,7 @@ assert!( metadata - .submit_session_finalization(identity, namespace_revision) + .submit_session_finalization(identity, namespace_revision, Vec::new()) .await .unwrap() .is_some(), @@ -10478,6 +10674,7 @@ partition_id: 1, consensus_group_id: 2, initial_view: 0, + members: Vec::new(), }], } .to_bytes(), @@ -10490,7 +10687,7 @@ assert_ne!(current_revision, revision); assert_eq!( metadata - .submit_session_finalization(identity, revision) + .submit_session_finalization(identity, revision, Vec::new()) .await .unwrap(), None @@ -10507,6 +10704,7 @@ &FinalizeSessionRequest { identity, namespace_revision: reported_revision, + debt: Vec::new(), } .to_bytes(), ), @@ -10528,6 +10726,107 @@ } #[compio::test] + async fn given_finalization_with_debt_when_applied_should_record_it_for_the_named_incarnation_only() + { + const CLIENT: u128 = 1; + let (_dir, metadata) = metadata_with_group_member(CLIENT).await; + let session = metadata.client_table.borrow().get_epoch(CLIENT).unwrap(); + metadata + .submit_logout_in_process(CLIENT, session, 1) + .await + .unwrap(); + let identity = metadata + .client_table + .borrow() + .ended_sessions() + .next() + .unwrap(); + let namespace = IggyNamespace::new(0, 0, 0); + let created_revision = metadata + .mux_stm + .streams() + .created_revision_for_namespace(namespace) + .unwrap(); + let consensus = metadata.consensus.as_ref().unwrap(); + let prepare = consensus::seal_prepare_capacity( + build_prepare_message( + consensus, + &RoutedRequestHeader::default(), + Operation::FinalizeSession, + &FinalizeSessionRequest { + identity, + namespace_revision: metadata.mux_stm.namespace_revision(), + debt: vec![ + RetirementDebt { + namespace: namespace.inner(), + created_revision, + }, + RetirementDebt { + namespace: namespace.inner(), + created_revision: created_revision + 1, + }, + ], + } + .to_bytes(), + ), + metadata.client_table_capacity(), + ); + let finalize_op = prepare.header().op; + apply_committed_prepare( + metadata.mux_stm.as_ref(), + &metadata.client_table, + true, + |_| {}, + prepare, + ); + assert_eq!(metadata.client_table.borrow().ended_sessions().next(), None); + let streams = metadata.mux_stm.streams(); + assert_eq!( + streams.retirement_debt_for(namespace, created_revision), + Some(finalize_op) + ); + assert_eq!( + streams.retirement_debt_for(namespace, created_revision + 1), + None + ); + assert_eq!( + streams.retirement_debts_stored_by(0, 1), + ( + vec![(namespace, created_revision, finalize_op)], + finalize_op + ) + ); + assert_eq!( + streams.retirement_debts_since(0, 0, 1), + Some(( + vec![(namespace, created_revision, finalize_op)], + finalize_op + )) + ); + assert_eq!( + streams.retirement_debts_since(finalize_op, 0, 1), + Some((Vec::new(), finalize_op)) + ); + let restored = <Streams as crate::stm::snapshot::Snapshotable>::from_snapshot( + crate::stm::snapshot::Snapshotable::to_snapshot(streams), + ) + .expect("snapshot restores"); + assert_eq!( + restored.retirement_debt_for(namespace, created_revision), + Some(finalize_op) + ); + assert_eq!( + restored.read(|inner| inner.retirement_debt_revision), + finalize_op + ); + assert_eq!( + restored.retirement_debts_since(0, 0, 1), + None, + "a restored reader must rescan" + ); + } + + #[compio::test] async fn stale_logout_preserves_a_newer_session_and_its_membership() { const CLIENT: u128 = 1; const OLD_SESSION: u64 = 0; @@ -10744,6 +11043,7 @@ &FinalizeSessionRequest { identity: ended[0], namespace_revision: metadata.mux_stm.namespace_revision(), + debt: Vec::new(), } .to_bytes(), ), @@ -11206,6 +11506,7 @@ partition_id: 0, consensus_group_id: 1, initial_view: 0, + members: Vec::new(), }], }, &mut inner,
diff --git a/core/metadata/src/impls/placement.rs b/core/metadata/src/impls/placement.rs index 42b1d31..4ae3d18 100644 --- a/core/metadata/src/impls/placement.rs +++ b/core/metadata/src/impls/placement.rs
@@ -16,8 +16,9 @@ // under the License. use crate::impls::metadata::{PartitionPlacementView, PartitionPrimaryTransfer}; -use crate::stm::stream::{Partition, Streams, StreamsInner, find_partition}; +use crate::stm::stream::{Partition, Streams, StreamsInner, find_partition, primary_of}; use ahash::AHashMap; +use consensus::{ReplicaSet, quorum_view_change}; use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment; use iggy_binary_protocol::requests::partitions::CreatePartitionsWithAssignmentsRequest; use iggy_binary_protocol::requests::topics::CreateTopicWithAssignmentsRequest; @@ -25,6 +26,7 @@ use iggy_common::IggyError; use server_common::Message; use server_common::sharding::IggyNamespace; +use std::cmp::Reverse; /// Summaries are only a cache. The live canonical pipeline decides which /// reservations count, so commit, rejection and log replacement need no undo. @@ -38,14 +40,79 @@ /// Changes with every observation, so a scan can tell its input moved. observed_generation: u64, last_scan: Option<(ScanInputs, Option<PartitionPrimaryTransfer>)>, - reservations: AHashMap<(u64, u128), Vec<u64>>, + reservations: AHashMap<(u64, u128), PlacementCounts>, eligible: Option<(u32, Vec<bool>)>, + /// A readiness coordinator has reported. A metadata plane without one has + /// no evidence to wait for and counts every node as ready. + readiness_reported: bool, + /// Per node, the failure domain it sits in, `None` for a node that named + /// none and so shares a domain with no other. + racks: Vec<Option<u32>>, } +/// Per node: partitions it is primary of, and partitions it stores. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct PlacementCounts { + pub(super) primaries: Vec<u64>, + pub(super) replicas: Vec<u64>, +} + +impl PlacementCounts { + fn new(nodes: u8) -> Self { + Self { + primaries: vec![0; usize::from(nodes)], + replicas: vec![0; usize::from(nodes)], + } + } + + fn nodes(&self) -> u8 { + u8::try_from(self.primaries.len()).unwrap_or(u8::MAX) + } + + /// Count one partition whose `created_view` names its primary among + /// `members`, or among every node when it has no member list. + fn record(&mut self, members: Option<ReplicaSet>, created_view: u32) { + let nodes = self.nodes(); + if let Some(primary) = primary_of(members, created_view, nodes) + && let Some(count) = self.primaries.get_mut(usize::from(primary)) + { + *count += 1; + } + match members { + Some(members) => { + for node in members.nodes() { + if let Some(count) = self.replicas.get_mut(usize::from(*node)) { + *count += 1; + } + } + } + None => { + for count in &mut self.replicas { + *count += 1; + } + } + } + } + + fn add(&mut self, other: &Self) { + for (count, added) in self.primaries.iter_mut().zip(&other.primaries) { + *count += added; + } + for (count, added) in self.replicas.iter_mut().zip(&other.replicas) { + *count += added; + } + } +} + +/// A candidate move's order: widest gap, lowest source node, after the cursor, +/// lowest namespace. +type MoveRank = (u64, Reverse<u8>, bool, Reverse<IggyNamespace>); + #[derive(Clone, Copy, PartialEq, Eq)] struct ObservedView { report: PartitionPlacementView, created_view: u32, + members: Option<ReplicaSet>, } /// Everything a rebalance scan reads. `revision` moves with the partition set @@ -60,7 +127,12 @@ } impl PlacementState { + pub(super) fn set_racks(&mut self, racks: Vec<Option<u32>>) { + self.racks = racks; + } + pub(super) fn observe_replicas(&mut self, view: u32, eligible: &[bool]) { + self.readiness_reported = true; self.eligible = (!eligible.is_empty()).then(|| (view, eligible.to_vec())); } @@ -80,6 +152,7 @@ let mut observed = ObservedView { report: *report, created_view: partition.created_view, + members: partition.members, }; if let Some(previous) = self.observations.get(&namespace).copied() { if previous.report.created_revision == report.created_revision { @@ -97,14 +170,24 @@ }); } - pub(super) fn committed_counts(&mut self, streams: &Streams, replicas: u8) -> Vec<u64> { + pub(super) fn committed_counts(&mut self, streams: &Streams, nodes: u8) -> PlacementCounts { streams.read(|inner| { self.retire_removed(inner); - let mut counts = vec![0_u64; usize::from(replicas)]; - for (&view, &partitions) in &inner.created_views { - counts[(view % u32::from(replicas)) as usize] += partitions; + let index = &inner.placement_index; + let mut counts = PlacementCounts::new(nodes); + let without_members: u64 = index.created_views.values().sum(); + for (&view, &partitions) in &index.created_views { + counts.primaries[(view % u32::from(nodes)) as usize] += partitions; } - for (count, &moved) in counts.iter_mut().zip(self.moved(replicas)) { + for (&node, &partitions) in &index.created_primaries { + if let Some(count) = counts.primaries.get_mut(usize::from(node)) { + *count += partitions; + } + } + for (node, count) in (0..nodes).zip(counts.replicas.iter_mut()) { + *count = without_members + index.stored_replicas.get(&node).copied().unwrap_or(0); + } + for (count, &moved) in counts.primaries.iter_mut().zip(self.moved(nodes)) { *count = count.saturating_add_signed(moved); } counts @@ -137,7 +220,7 @@ { return *plan; } - let plan = self.scan(inner, replica_count, eligible, after); + let plan = self.scan(inner, replicas, eligible, after); self.last_scan = Some((inputs, plan)); plan }) @@ -166,18 +249,81 @@ }) } - pub(super) fn select(&self, counts: &mut [u64], view: u32) -> Result<u32, IggyError> { + /// Members and created view for one new partition, counted into `counts`. + /// + /// The primary is the ready node leading the fewest partitions. The other + /// members are the nodes storing the fewest, ready nodes first, so a node + /// that is down is chosen only when too few are up. `None` keeps the + /// layout from before replication factors: no member list, every node + /// stores it. + /// + /// # Errors + /// `InvalidReplicationFactor` for a factor of zero or above the node count, + /// `TransientNotAccepted` when no node is ready to lead or, once a + /// coordinator reports, while it has no readiness for this view. + pub(super) fn select( + &self, + counts: &mut PlacementCounts, + view: u32, + replication_factor: Option<u8>, + ) -> Result<(Vec<u8>, u32), IggyError> { + let nodes = counts.nodes(); + if replication_factor.is_some_and(|factor| factor == 0 || factor > nodes) { + return Err(IggyError::InvalidReplicationFactor); + } let eligible = self.eligible.as_ref().filter(|(observed_view, eligible)| { - *observed_view == view && eligible.len() == counts.len() + *observed_view == view && eligible.len() == usize::from(nodes) }); - let (replica, count) = counts - .iter_mut() - .enumerate() - .filter(|(replica, _)| eligible.is_none_or(|(_, eligible)| eligible[*replica])) - .min_by_key(|(replica, count)| (**count, *replica)) + // The assignment never moves, so guessing would leave a failed node + // holding the partition until it returns. + if self.readiness_reported && eligible.is_none() { + return Err(IggyError::TransientNotAccepted); + } + let ready = |node: u8| eligible.is_none_or(|(_, eligible)| eligible[usize::from(node)]); + let load = |node: u8| { + ( + counts.primaries[usize::from(node)], + counts.replicas[usize::from(node)], + ) + }; + let primary = (0..nodes) + .filter(|node| ready(*node)) + .min_by_key(|node| (load(*node), *node)) .ok_or(IggyError::TransientNotAccepted)?; - *count += 1; - u32::try_from(replica).map_err(|_| IggyError::TransientNotAccepted) + let Some(factor) = replication_factor else { + counts.record(None, u32::from(primary)); + return Ok((Vec::new(), u32::from(primary))); + }; + // A copy in a rack the partition already uses does not survive that + // rack's loss, so each follower goes to an unused rack while one holds + // a ready node. + let rack = |node: u8| self.racks.get(usize::from(node)).copied().flatten(); + let mut members = vec![primary]; + let mut used_racks: Vec<u32> = rack(primary).into_iter().collect(); + let mut candidates: Vec<u8> = (0..nodes).filter(|node| *node != primary).collect(); + for _ in 1..factor { + let (index, &follower) = candidates + .iter() + .enumerate() + .min_by_key(|&(_, &node)| { + let (primaries, replicas) = load(node); + let shares_rack = rack(node).is_some_and(|rack| used_racks.contains(&rack)); + (!ready(node), shares_rack, replicas, primaries, node) + }) + .ok_or(IggyError::InvalidReplicationFactor)?; + candidates.swap_remove(index); + used_racks.extend(rack(follower)); + members.push(follower); + } + members.sort_unstable(); + let created_view = members + .iter() + .position(|node| *node == primary) + .and_then(|index| u32::try_from(index).ok()) + .ok_or(IggyError::InvalidReplicationFactor)?; + let set = ReplicaSet::new(&members).map_err(|_| IggyError::InvalidReplicationFactor)?; + counts.record(Some(set), created_view); + Ok((members, created_view)) } pub(super) fn prune(&mut self, retained: impl Fn(u64, u128) -> bool) { @@ -194,22 +340,20 @@ pub(super) fn reserved_partitions(&self, header: &PrepareHeader) -> Result<u64, IggyError> { self.reservations .get(&(header.op, header.checksum)) - .map(|reserved| reserved.iter().sum()) + .map(|reserved| reserved.primaries.iter().sum()) .ok_or(IggyError::TransientNotAccepted) } pub(super) fn add_reservation( &self, header: &PrepareHeader, - counts: &mut [u64], + counts: &mut PlacementCounts, ) -> Result<(), IggyError> { let reserved = self .reservations .get(&(header.op, header.checksum)) .ok_or(IggyError::TransientNotAccepted)?; - for (count, reserved) in counts.iter_mut().zip(reserved) { - *count += reserved; - } + counts.add(reserved); Ok(()) } @@ -231,15 +375,20 @@ } _ => return Ok(None), }; - let mut counts = vec![0; usize::from(replicas)]; + let mut counts = PlacementCounts::new(replicas); let mut highest_group_id = 0; for CreatedPartitionAssignment { consensus_group_id, initial_view, + members, .. } in partitions { - counts[(initial_view % u32::from(replicas)) as usize] += 1; + let members = (!members.is_empty()) + .then(|| ReplicaSet::new(&members)) + .transpose() + .map_err(|_| IggyError::InvalidCommand)?; + counts.record(members, initial_view); highest_group_id = highest_group_id.max(consensus_group_id); } self.reservations @@ -247,16 +396,84 @@ Ok(Some(highest_group_id)) } + /// One primary move to a ready member of the same partition leading at + /// least two fewer partitions, the widest such gap first. A partition + /// qualifies only while enough of its members are ready to elect. Equal + /// gaps go to the lowest source node, then rotate with `after`. fn scan( &self, inner: &StreamsInner, - replicas: usize, + nodes: u8, eligible: &[bool], after: Option<IggyNamespace>, ) -> Option<PartitionPrimaryTransfer> { - let mut counts = vec![0_u64; replicas]; - let mut first: Vec<Option<PartitionPlacementView>> = vec![None; replicas]; - let mut next: Vec<Option<PartitionPlacementView>> = vec![None; replicas]; + let ready = |node: u8| eligible.get(usize::from(node)).copied().unwrap_or(false); + let mut counts = vec![0_u64; usize::from(nodes)]; + self.for_each_partition(inner, |_, partition, report| { + let view = report.map_or(partition.created_view, |report| report.view); + if let Some(count) = partition + .primary_node(view, nodes) + .and_then(|primary| counts.get_mut(usize::from(primary))) + { + *count += 1; + } + }); + let count = |node: u8| counts.get(usize::from(node)).copied().unwrap_or(u64::MAX); + let mut best: Option<(MoveRank, PartitionPrimaryTransfer)> = None; + self.for_each_partition(inner, |namespace, partition, report| { + let Some(report) = report else { + return; + }; + let Some(source) = partition + .primary_node(report.view, nodes) + .filter(|source| ready(*source)) + else { + return; + }; + let members = partition.replica_set(nodes); + let ready_members = members.nodes().iter().filter(|node| ready(**node)).count(); + if ready_members < quorum_view_change(members.len()) { + return; + } + let Some(target) = members + .nodes() + .iter() + .copied() + .filter(|node| *node != source && ready(*node)) + .min_by_key(|node| (count(*node), *node)) + else { + return; + }; + let gap = count(source).saturating_sub(count(target)); + if gap <= 1 { + return; + } + let key = ( + gap, + Reverse(source), + after.is_none_or(|cursor| namespace > cursor), + Reverse(namespace), + ); + if best.as_ref().is_none_or(|(best_key, _)| key > *best_key) { + best = Some(( + key, + PartitionPrimaryTransfer { + namespace, + created_revision: report.created_revision, + view: report.view, + target, + }, + )); + } + }); + best.map(|(_, plan)| plan) + } + + fn for_each_partition( + &self, + inner: &StreamsInner, + mut visit: impl FnMut(IggyNamespace, &Partition, Option<PartitionPlacementView>), + ) { for (stream_id, stream) in &inner.items { for (topic_id, topic) in &stream.topics { if inner.lifecycle_blocks(stream_id, Some(topic_id)) { @@ -264,49 +481,14 @@ } for partition in &topic.partitions { let namespace = IggyNamespace::new(stream_id, topic_id, partition.id); - let report = self.matching_view(namespace, partition); - let primary = report.map_or(partition.created_view, |report| report.view) - as usize - % replicas; - counts[primary] += 1; - let Some(report) = report.filter(|_| eligible[primary]) else { - continue; - }; - if first[primary].is_none_or(|first| namespace < first.namespace) { - first[primary] = Some(report); - } - if after.is_none_or(|cursor| namespace > cursor) - && next[primary].is_none_or(|next| namespace < next.namespace) - { - next[primary] = Some(report); - } + visit( + namespace, + partition, + self.matching_view(namespace, partition), + ); } } } - let mut available = eligible - .iter() - .enumerate() - .filter_map(|(replica, &eligible)| eligible.then_some(replica)); - let mut source = available.next()?; - let mut target = source; - for replica in available { - if counts[replica] > counts[source] { - source = replica; - } - if counts[replica] < counts[target] { - target = replica; - } - } - if counts[source].saturating_sub(counts[target]) <= 1 { - return None; - } - let report = next[source].or(first[source])?; - Some(PartitionPrimaryTransfer { - namespace: report.namespace, - created_revision: report.created_revision, - view: report.view, - target: u8::try_from(target).ok()?, - }) } fn matching_view( @@ -382,11 +564,18 @@ impl ObservedView { /// Counts this partition at its observed primary instead of its created /// one, or back for a negative `by`. - fn shift(self, replicas: u8, moved: &mut [i64], by: i64) { - let replicas = u32::from(replicas); + fn shift(self, nodes: u8, moved: &mut [i64], by: i64) { let observed = self.report.view.max(self.created_view); - moved[(observed % replicas) as usize] += by; - moved[(self.created_view % replicas) as usize] -= by; + if let Some(count) = primary_of(self.members, observed, nodes) + .and_then(|node| moved.get_mut(usize::from(node))) + { + *count += by; + } + if let Some(count) = primary_of(self.members, self.created_view, nodes) + .and_then(|node| moved.get_mut(usize::from(node))) + { + *count -= by; + } } } @@ -624,7 +813,10 @@ }], ); } - assert_eq!(placement.committed_counts(&streams, 5), [0, 40, 20, 20, 20]); + assert_eq!( + placement.committed_counts(&streams, 5).primaries, + [0, 40, 20, 20, 20] + ); for (eligible, expected_counts, expected_moves) in [ ([false, true, true, true, true], [0, 25, 25, 25, 25], 15), ([true; 5], [20; 5], 20), @@ -651,10 +843,15 @@ moves += 1; } assert_eq!(moves, expected_moves); - assert_eq!(placement.committed_counts(&streams, 5), expected_counts); + assert_eq!( + placement.committed_counts(&streams, 5).primaries, + expected_counts + ); } assert_eq!( - PlacementState::default().committed_counts(&streams, 5), + PlacementState::default() + .committed_counts(&streams, 5) + .primaries, [20; 5], "transfers must not rewrite committed initial views" ); @@ -874,19 +1071,227 @@ assert_eq!(transfer.target, 2); } + /// Nodes numbered rack by rack would put the first partitions' copies in + /// one rack each, so losing a rack would lose them all. #[test] - fn given_readiness_from_another_view_when_placing_should_ignore_stale_evidence() { + fn given_nodes_in_racks_when_placing_should_spread_each_partition_over_racks() { + const NODES: u8 = 9; + const PARTITIONS: usize = 30; + let rack_of = |node: u8| u32::from(node / 3); let mut placement = PlacementState::default(); + placement.set_racks((0..NODES).map(|node| Some(rack_of(node))).collect()); + let mut counts = PlacementCounts::new(NODES); + for _ in 0..PARTITIONS { + let (members, _) = placement.select(&mut counts, 0, Some(3)).unwrap(); + let mut racks: Vec<u32> = members.iter().map(|&node| rack_of(node)).collect(); + racks.sort_unstable(); + racks.dedup(); + assert_eq!(racks.len(), 3, "members {members:?} share a rack"); + } + assert!( + counts + .replicas + .iter() + .all(|copies| copies.abs_diff(10) <= 1), + "copies stay within one of the mean: {:?}", + counts.replicas + ); + } + + #[test] + fn given_replication_factor_when_placing_many_partitions_should_spread_primaries_and_copies() { + const NODES: u8 = 5; + let placement = PlacementState::default(); + let mut counts = PlacementCounts::new(NODES); + for _ in 0..10 { + let (members, created_view) = placement.select(&mut counts, 0, Some(3)).unwrap(); + assert_eq!(members.len(), 3); + assert!(members.is_sorted()); + assert!(usize::try_from(created_view).unwrap() < members.len()); + } + assert_eq!(counts.primaries, [2; 5]); + assert_eq!(counts.replicas.iter().sum::<u64>(), 30); + assert!( + counts.replicas.iter().all(|copies| copies.abs_diff(6) <= 1), + "copies stay within one of the mean: {:?}", + counts.replicas + ); + } + + #[test] + fn given_factor_outside_the_cluster_when_placing_should_refuse() { + let placement = PlacementState::default(); + for factor in [0, 4] { + assert_eq!( + placement.select(&mut PlacementCounts::new(3), 0, Some(factor)), + Err(IggyError::InvalidReplicationFactor) + ); + } + } + + #[test] + fn given_a_down_node_when_placing_should_use_it_only_when_too_few_are_up() { + let mut placement = PlacementState::default(); + placement.observe_replicas(0, &[true, false, true]); + for _ in 0..4 { + let mut counts = PlacementCounts::new(3); + assert_eq!( + placement.select(&mut counts, 0, Some(2)).unwrap().0, + [0, 2], + "two nodes are up" + ); + let (members, created_view) = placement.select(&mut counts, 0, Some(3)).unwrap(); + assert_eq!(members, [0, 1, 2]); + assert_ne!( + members[usize::try_from(created_view).unwrap()], + 1, + "a down node never leads" + ); + } + } + + #[test] + fn given_imbalance_outside_a_group_when_rebalancing_should_move_only_to_a_member() { + const NODES: u8 = 5; + let streams = Streams::default(); + let mut placement = PlacementState::default(); + for topic in 0..4 { + let namespace = IggyNamespace::new(0, topic, 0); + streams.seed_namespace_on( + namespace, + u64::try_from(topic).unwrap() + 1, + 0, + iggy_common::Durability::Persisted, + &[0, 1, 2], + ); + placement.observe( + &streams, + &[PartitionPlacementView { + namespace, + created_revision: streams.created_revision_for_namespace(namespace).unwrap(), + view: 0, + }], + ); + } + assert_eq!( + placement.committed_counts(&streams, NODES), + PlacementCounts { + primaries: vec![4, 0, 0, 0, 0], + replicas: vec![4, 4, 4, 0, 0], + } + ); + + let transfer = placement + .primary_rebalance_candidate(&streams, NODES, &[true; 5], None) + .unwrap(); + assert_eq!( + transfer.target, 1, + "nodes 3 and 4 lead fewer but store nothing" + ); + + let mut group_down = [true; 5]; + group_down[1] = false; + group_down[2] = false; + assert_eq!( + placement.primary_rebalance_candidate(&streams, NODES, &group_down, None), + None, + "one of three members cannot elect" + ); + } + + #[test] + fn given_the_busiest_node_has_no_lighter_member_when_rebalancing_should_move_another_group() { + const NODES: u8 = 5; + let streams = Streams::default(); + let mut placement = PlacementState::default(); + let layout: [(&[u8], u32); 8] = [ + (&[0, 1, 2], 0), + (&[0, 1, 2], 0), + (&[0, 1, 2], 0), + (&[1, 3, 4], 0), + (&[1, 3, 4], 0), + (&[1, 3, 4], 0), + (&[0, 1, 2], 2), + (&[0, 1, 2], 2), + ]; + for (topic, (members, view)) in layout.into_iter().enumerate() { + let namespace = IggyNamespace::new(0, topic, 0); + streams.seed_namespace_on( + namespace, + u64::try_from(topic).unwrap() + 1, + view, + iggy_common::Durability::Persisted, + members, + ); + placement.observe( + &streams, + &[PartitionPlacementView { + namespace, + created_revision: streams.created_revision_for_namespace(namespace).unwrap(), + view, + }], + ); + } + assert_eq!( + placement.committed_counts(&streams, NODES).primaries, + [3, 3, 2, 0, 0] + ); + + let transfer = placement + .primary_rebalance_candidate(&streams, NODES, &[true; 5], None) + .expect("node 1 can hand a partition to node 3"); + assert_eq!(transfer.target, 3); + assert_eq!( + streams + .members_for_namespace(transfer.namespace) + .flatten() + .unwrap() + .nodes(), + &[1, 3, 4] + ); + } + + #[test] + fn given_readiness_absent_or_from_another_view_when_placing_should_refuse_until_this_view_reports() + { + let counts = || PlacementCounts { + primaries: vec![0, 5, 5], + replicas: vec![0; 3], + }; + let mut placement = PlacementState::default(); + assert_eq!( + placement.select(&mut counts(), 1, Some(2)), + Ok((vec![0, 1], 0)), + "without a coordinator every node counts as ready" + ); placement.observe_replicas(1, &[false, true, true]); - assert_eq!(placement.select(&mut [0, 5, 5], 1), Ok(1)); - assert_eq!(placement.select(&mut [0, 5, 5], 2), Ok(0)); + assert_eq!( + placement.select(&mut counts(), 1, None), + Ok((Vec::new(), 1)) + ); + for factor in [None, Some(1), Some(2)] { + assert_eq!( + placement.select(&mut counts(), 2, factor), + Err(IggyError::TransientNotAccepted), + "evidence from an older view must not place factor {factor:?}" + ); + } placement.observe_replicas(2, &[false; 3]); assert_eq!( - placement.select(&mut [0, 5, 5], 2), + placement.select(&mut counts(), 2, None), Err(IggyError::TransientNotAccepted) ); placement.observe_replicas(2, &[]); - assert_eq!(placement.select(&mut [0, 5, 5], 2), Ok(0)); + assert_eq!( + placement.select(&mut counts(), 2, Some(1)), + Err(IggyError::TransientNotAccepted), + "cleared evidence must not presume every node ready" + ); + placement.observe_replicas(2, &[true, false, true]); + assert_eq!( + placement.select(&mut counts(), 2, Some(2)), + Ok((vec![0, 2], 0)) + ); } fn observe_views( @@ -923,9 +1328,13 @@ /// Every committed partition at its observed view, or its created view /// before an observation of its incarnation. - fn walked_counts(placement: &PlacementState, streams: &Streams, replicas: u8) -> Vec<u64> { + fn walked_counts( + placement: &PlacementState, + streams: &Streams, + replicas: u8, + ) -> PlacementCounts { streams.read(|inner| { - let mut counts = vec![0; usize::from(replicas)]; + let mut counts = PlacementCounts::new(replicas); for (stream_id, stream) in &inner.items { for (topic_id, topic) in &stream.topics { for partition in &topic.partitions { @@ -939,7 +1348,11 @@ .map_or(partition.created_view, |observed| { observed.report.view.max(partition.created_view) }); - counts[(view % u32::from(replicas)) as usize] += 1; + counts.primaries + [usize::from(partition.primary_node(view, replicas).unwrap())] += 1; + for node in partition.replica_set(replicas).nodes() { + counts.replicas[usize::from(*node)] += 1; + } } } }
diff --git a/core/metadata/src/impls/recovery.rs b/core/metadata/src/impls/recovery.rs index 63b4312..4340b62 100644 --- a/core/metadata/src/impls/recovery.rs +++ b/core/metadata/src/impls/recovery.rs
@@ -773,6 +773,7 @@ })?; if mux_stm.namespace_revision() == request.namespace_revision { client_table.finalize_session(request.identity); + mux_stm.record_retirement_debt(&request.debt, header.op); } change_log.append(header, Vec::new())?; last_applied_op = Some(header.op); @@ -2008,6 +2009,7 @@ let body = FinalizeSessionRequest { identity, namespace_revision: reported_revision, + debt: Vec::new(), } .to_bytes(); let mut prepare = make_prepare(SNAPSHOT_OP + 1, body.len()); @@ -2711,6 +2713,7 @@ partition_id, consensus_group_id: u64::from(partition_id) + 1, initial_view: 0, + members: Vec::new(), }) .collect(), }
diff --git a/core/metadata/src/stm/authz.rs b/core/metadata/src/stm/authz.rs index ac41cc3..3557b26 100644 --- a/core/metadata/src/stm/authz.rs +++ b/core/metadata/src/stm/authz.rs
@@ -51,6 +51,7 @@ CreatePartitionsWithAssignmentsRequest, DeletePartitionsRequest, }; use iggy_binary_protocol::requests::streams::{DeleteStreamRequest, UpdateStreamResolvedRequest}; +use iggy_binary_protocol::requests::system::RetirementDebt; use iggy_binary_protocol::requests::topics::{ CreateTopicWithAssignmentsRequest, DeleteTopicRequest, UpdateTopicResolvedRequest, }; @@ -148,6 +149,9 @@ /// Partition-set revision used to validate retirement coverage during WAL replay. fn namespace_revision(&self) -> u64; + /// Records the retirement debt a replayed `FinalizeSession` carried. + fn record_retirement_debt(&self, debts: &[RetirementDebt], metadata_op: u64); + fn is_active_account(&self, user_id: u32, created_at: u64) -> bool; /// Rebinds the consumer group memberships of the client a replayed @@ -218,6 +222,10 @@ fn namespace_revision(&self) -> u64 { self.streams().read(|inner| inner.namespace_revision) } + + fn record_retirement_debt(&self, debts: &[RetirementDebt], metadata_op: u64) { + self.streams().record_retirement_debt(debts, metadata_op); + } } /// The root user is always the first user created (slab id 0) and holds every @@ -631,6 +639,8 @@ fn namespace_revision(&self) -> u64 { 0 } + + fn record_retirement_debt(&self, _debts: &[RetirementDebt], _metadata_op: u64) {} } #[cfg(test)] @@ -736,6 +746,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], } .to_bytes()
diff --git a/core/metadata/src/stm/consumer_group.rs b/core/metadata/src/stm/consumer_group.rs index 8a7b349..23cad4d 100644 --- a/core/metadata/src/stm/consumer_group.rs +++ b/core/metadata/src/stm/consumer_group.rs
@@ -2194,6 +2194,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], }; let _ = StateHandler::apply(&create_topic, &mut inner, IggyTimestamp::now()); @@ -2267,6 +2268,7 @@ initial_view: 0, partition_id, consensus_group_id: u64::from(partition_id) + 2, + members: Vec::new(), }) .collect(), },
diff --git a/core/metadata/src/stm/lifecycle.rs b/core/metadata/src/stm/lifecycle.rs index f1bfef5..5d8ca8a 100644 --- a/core/metadata/src/stm/lifecycle.rs +++ b/core/metadata/src/stm/lifecycle.rs
@@ -64,6 +64,10 @@ pub partition_id: u32, pub fence: LifecycleFence, pub partition_op: Option<u64>, + /// Bit per node that retired its lost copy of a deleted partition. See + /// [`retired_member_op`]. + #[serde(default)] + pub retired_members: u32, } /// Retained until every affected partition durably installs its exact fence. @@ -118,6 +122,27 @@ /// incarnation offline completes it instead. pub const RETIRED_PARTITION_OP: u64 = u64::MAX; +/// The `partition_op` with which `node` reports retiring its lost copy of a +/// partition that a delete removes. +/// +/// It only records the member. Once too few members remain to commit, a member +/// completes the target with [`RETIRED_PARTITION_OP`], since then no log can +/// install its fence either. +#[must_use] +pub const fn retired_member_op(node: u8) -> u64 { + RETIRED_PARTITION_OP - 1 - node as u64 +} + +/// The node a [`retired_member_op`] names. +#[must_use] +pub const fn retiring_member(partition_op: u64) -> Option<u8> { + match (RETIRED_PARTITION_OP - 1).checked_sub(partition_op) { + #[allow(clippy::cast_possible_truncation)] + Some(node) if node < u32::BITS as u64 => Some(node as u8), + _ => None, + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct LifecycleCompletion { pub topic_id: u32, @@ -266,6 +291,9 @@ return state.lifecycle_blocks_topic_creation(stream_id); } if !topic_scoped { + if operation == Operation::DeleteStream { + return state.lifecycle_blocks_delete(stream_id, None); + } return state.lifecycle_blocks(stream_id, None); } let Ok((topic, topic_bytes)) = WireIdentifier::decode(&body[consumed..]) else { @@ -291,6 +319,9 @@ .and_then(|topic| topic.resolve_group_id(&group)); return state.lifecycle_blocks_consumer(stream_id, topic_id, group_id); } + if operation == Operation::DeleteTopic { + return state.lifecycle_blocks_delete(stream_id, Some(topic_id)); + } state.lifecycle_blocks(stream_id, Some(topic_id)) }) } @@ -303,6 +334,32 @@ .is_some() } + /// A topic or stream delete removes the topic's consumer groups too, so a + /// pending group deletion does not hold it back: the delete takes it over. + /// A group deletion whose partition can never elect a primary would + /// otherwise keep the topic from ever being deleted. + pub(crate) fn lifecycle_blocks_delete( + &self, + stream_id: usize, + topic_id: Option<usize>, + ) -> bool { + self.lifecycle_blockers(stream_id, topic_id) + .any(|intent| !matches!(intent.action, LifecycleAction::DeleteConsumerGroup { .. })) + } + + fn take_over_group_deletions(&mut self, stream_id: usize, topic_id: Option<usize>) { + let taken: Vec<u64> = self + .lifecycle_blockers(stream_id, topic_id) + .filter(|intent| matches!(intent.action, LifecycleAction::DeleteConsumerGroup { .. })) + .map(|intent| intent.context.metadata_op) + .collect(); + for metadata_op in taken { + if let Some(intent) = self.lifecycle_intents.remove(&metadata_op) { + self.lifecycle_scopes.remove(&intent.scope()); + } + } + } + /// A topic-scoped intent leaves the rest of its stream writable, so only a /// stream-wide intent blocks a new topic. Admission and apply share this rule. pub(crate) fn lifecycle_blocks_topic_creation(&self, stream_id: usize) -> bool { @@ -366,7 +423,16 @@ if self.finalizing_lifecycle.is_some() { return None; } - if self.lifecycle_blocks(stream_id, topic_id) { + let deletes = matches!( + action, + LifecycleAction::DeleteStream | LifecycleAction::DeleteTopic + ); + let busy = if deletes { + self.lifecycle_blocks_delete(stream_id, topic_id) + } else { + self.lifecycle_blocks(stream_id, topic_id) + }; + if busy { return Some(ApplyReply::err(IggyError::LifecycleBusy.as_code())); } let stream = self.items.get(stream_id)?; @@ -421,15 +487,19 @@ partition_id, fence, partition_op: None, + retired_members: 0, }); } } + if !partitions.is_empty() && self.apply_context.metadata_op == 0 { + return Some(ApplyReply::err(IggyError::InvalidCommand.as_code())); + } + if deletes { + self.take_over_group_deletions(stream_id, topic_id); + } if partitions.is_empty() { return None; } - if self.apply_context.metadata_op == 0 { - return Some(ApplyReply::err(IggyError::InvalidCommand.as_code())); - } let mut intent = LifecycleIntent { context: self.apply_context, stream_id: wire_stream_id, @@ -460,6 +530,7 @@ // A batch applies whole or not at all, so nothing is recorded until // every completion passes. let mut recorded = Vec::with_capacity(self.completions.len()); + let mut retirements = Vec::new(); for completion in &self.completions { let Some(&index) = intent .partition_index @@ -467,14 +538,28 @@ else { return ApplyReply::err(IggyError::InvalidCommand.as_code()); }; - match intent - .partitions - .get(index) - .map(|partition| partition.partition_op) - { - Some(None) => recorded.push((index, completion.partition_op)), - Some(Some(op)) if op == completion.partition_op => {} - _ => return ApplyReply::err(IggyError::InvalidCommand.as_code()), + let Some(partition) = intent.partitions.get(index) else { + return ApplyReply::err(IggyError::InvalidCommand.as_code()); + }; + if let Some(node) = retiring_member(completion.partition_op) { + // Only a delete may lose a copy for good. + if !matches!(partition.fence, LifecycleFence::History(_)) { + return ApplyReply::err(IggyError::InvalidCommand.as_code()); + } + if partition.partition_op.is_none() { + retirements.push((index, node)); + } + continue; + } + match partition.partition_op { + None => recorded.push((index, completion.partition_op)), + Some(op) if op == completion.partition_op => {} + Some(_) => return ApplyReply::err(IggyError::InvalidCommand.as_code()), + } + } + for &(index, node) in &retirements { + if let Some(partition) = intent.partitions.get_mut(index) { + partition.retired_members |= 1 << node; } } for &(index, partition_op) in &recorded { @@ -1486,14 +1571,15 @@ } assert!(!streams.request_lifecycle_blocked(Operation::SendMessages, &[])); assert!( - streams.request_lifecycle_blocked( + !streams.request_lifecycle_blocked( Operation::DeleteTopic, &DeleteTopicRequest { stream_id: WireIdentifier::numeric(0), topic_id: WireIdentifier::numeric(0), } .to_bytes() - ) + ), + "a topic delete takes the group deletion over" ); complete(&streams, 0, 12); assert!(streams.read(|inner| inner.lifecycle_blocks_consumer(0, 0, Some(0)))); @@ -1565,6 +1651,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], }; for stream_wide in [false, true] { @@ -1680,6 +1767,155 @@ assert_eq!(catalog(&restored), (REMOVAL_OP, 2, vec![1])); } + /// A group deletion can wait forever on a partition that cannot elect, so + /// it must not keep its topic from being deleted: the topic delete takes + /// it over and removes the group with the topic. + #[test] + fn given_a_pending_group_deletion_when_deleting_its_topic_should_take_it_over() { + const TOPIC_DELETE_OP: u64 = INTENT_OP + 10; + let streams = pending_owned_group_deletion(INTENT_OP - 1); + let delete = DeleteTopicRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + }; + assert!(!streams.request_lifecycle_blocked(Operation::DeleteTopic, &delete.to_bytes())); + let create_partitions = + iggy_binary_protocol::requests::partitions::CreatePartitionsRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + partitions_count: 1, + }; + assert!( + streams.request_lifecycle_blocked( + Operation::CreatePartitions, + &create_partitions.to_bytes() + ), + "only a delete of the scope takes a group deletion over" + ); + + let refused = apply( + &streams, + Operation::DeleteTopic, + &delete, + ApplyContext { + metadata_op: 0, + ..context() + }, + ); + assert_eq!(refused.code, IggyError::InvalidCommand.as_code()); + assert!( + streams.lifecycle_pending(INTENT_OP), + "a refused delete leaves the group deletion in place" + ); + + let reply = apply( + &streams, + Operation::DeleteTopic, + &delete, + ApplyContext { + metadata_op: TOPIC_DELETE_OP, + ..context() + }, + ); + assert_eq!(reply.code, 0); + assert!(!streams.lifecycle_pending(INTENT_OP)); + assert!(streams.lifecycle_pending(TOPIC_DELETE_OP)); + + for (partition_id, metadata_op) in [(0, TOPIC_DELETE_OP + 1), (1, TOPIC_DELETE_OP + 2)] { + let reply = apply( + &streams, + Operation::CompleteLifecycle, + &CompleteLifecycleRequest::new( + TOPIC_DELETE_OP, + 0, + vec![completion(partition_id, 3)], + ) + .unwrap(), + ApplyContext { + metadata_op, + ..ApplyContext::default() + }, + ); + assert_eq!(reply.code, 0); + } + assert!(!streams.lifecycle_pending(TOPIC_DELETE_OP)); + assert!(streams.read(|inner| inner.items[0].topics.is_empty())); + } + + #[test] + fn given_a_member_retirement_when_completing_should_record_it_and_leave_the_target_pending() { + assert_eq!(retiring_member(retired_member_op(5)), Some(5)); + assert_eq!(retiring_member(RETIRED_PARTITION_OP), None); + assert_eq!(retiring_member(12), None); + let streams = streams(); + let reply = apply( + &streams, + Operation::DeleteTopic, + &DeleteTopicRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + }, + context(), + ); + assert_eq!(reply.code, 0); + let retire = |streams: &Streams, metadata_op| { + complete_batch( + streams, + vec![completion(0, retired_member_op(2))], + metadata_op, + ) + .code + }; + assert_eq!(retire(&streams, INTENT_OP + 1), 0); + assert_eq!( + retire(&streams, INTENT_OP + 2), + 0, + "a repeated report changes nothing" + ); + let retired = + |streams: &Streams| streams.pending_lifecycles()[0].partitions[0].retired_members; + assert_eq!(retired(&streams), 1 << 2); + assert!( + streams.lifecycle_pending(INTENT_OP), + "a retirement completes no target" + ); + + let mut snapshot = crate::stm::snapshot::MetadataSnapshot::new(INTENT_OP + 2); + snapshot.streams = Some(streams.to_snapshot()); + let decoded = + crate::stm::snapshot::MetadataSnapshot::decode(&snapshot.encode().unwrap()).unwrap(); + let restored = Streams::from_snapshot(decoded.streams.unwrap()).unwrap(); + assert_eq!(retired(&restored), 1 << 2); + + for (partition_id, partition_op, metadata_op) in [ + (0, RETIRED_PARTITION_OP, INTENT_OP + 3), + (1, 7, INTENT_OP + 4), + ] { + let reply = complete_batch( + &streams, + vec![completion(partition_id, partition_op)], + metadata_op, + ); + assert_eq!(reply.code, 0); + } + assert!(!streams.lifecycle_pending(INTENT_OP)); + } + + #[test] + fn given_a_group_deletion_when_a_member_reports_retiring_its_copy_should_refuse() { + let streams = pending_owned_group_deletion(INTENT_OP - 1); + let reply = complete_batch( + &streams, + vec![completion(0, retired_member_op(1))], + INTENT_OP + 1, + ); + assert_eq!(reply.code, IggyError::InvalidCommand.as_code()); + assert_eq!( + streams.pending_lifecycles()[0].partitions[0].retired_members, + 0 + ); + } + #[test] fn given_completion_batch_when_decoding_should_accept_only_canonical_batches() { let batch =
diff --git a/core/metadata/src/stm/mod.rs b/core/metadata/src/stm/mod.rs index 7b88ea1..8216cf3 100644 --- a/core/metadata/src/stm/mod.rs +++ b/core/metadata/src/stm/mod.rs
@@ -20,6 +20,7 @@ pub mod id_slab; pub mod lifecycle; pub mod mux; +pub mod partition_directory; pub mod result; pub mod snapshot; pub mod stream;
diff --git a/core/metadata/src/stm/partition_directory.rs b/core/metadata/src/stm/partition_directory.rs new file mode 100644 index 0000000..65383da --- /dev/null +++ b/core/metadata/src/stm/partition_directory.rs
@@ -0,0 +1,519 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy_binary_protocol::primitives::partition_status::PartitionStatusEntry; +use server_common::sharding::IggyNamespace; +use std::collections::{BTreeMap, HashMap}; +use std::ops::Bound; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::Instant; + +/// What each partition's primary last reported, as this node heard it. +/// +/// Node-local and never replicated: the metadata primary collects it from the +/// other nodes' metadata replies and hands it back out in its probes, so a +/// node answers for partitions it does not store. +#[derive(Debug)] +pub struct PartitionDirectory { + entries: Mutex<BTreeMap<u64, Recorded>>, + /// Nodes the metadata primary hears from, bit `n` for node `n`, or every + /// bit without that evidence. Reports leave the reads with their in-sync + /// nodes narrowed to these, so a primary that died does not keep claiming + /// its members in sync with its last report. + ready_nodes: AtomicU32, + /// Locked after `entries` whenever both are held. + changes: Mutex<Changes>, +} + +#[derive(Debug, Clone, Copy)] +struct Recorded { + report: PartitionStatusEntry, + received_at: Instant, + /// The metadata primary found no copy newer than its stale cutoff, so the + /// reads list no member in sync until the primary reports again. + stale: bool, +} + +/// The order in which reports last changed what a route or an in-sync list +/// reads, so a node hears a new primary first rather than after a whole +/// cycle through every partition. +#[derive(Debug, Default)] +struct Changes { + last: u64, + by_seq: BTreeMap<u64, u64>, + by_namespace: HashMap<u64, u64>, +} + +impl Changes { + fn mark(&mut self, namespace: u64) { + self.last += 1; + if let Some(previous) = self.by_namespace.insert(namespace, self.last) { + self.by_seq.remove(&previous); + } + self.by_seq.insert(self.last, namespace); + } + + fn forget(&mut self, namespace: u64) { + if let Some(seq) = self.by_namespace.remove(&namespace) { + self.by_seq.remove(&seq); + } + } +} + +impl Default for PartitionDirectory { + fn default() -> Self { + Self { + entries: Mutex::default(), + ready_nodes: AtomicU32::new(u32::MAX), + changes: Mutex::default(), + } + } +} + +impl PartitionDirectory { + /// Narrow the in-sync nodes of every report read from here on to + /// `ready_nodes`, or stop narrowing them with `u32::MAX`. + pub fn set_ready_nodes(&self, ready_nodes: u32) { + self.ready_nodes.store(ready_nodes, Ordering::Relaxed); + } + + fn narrowed(&self, recorded: &Recorded) -> PartitionStatusEntry { + let in_sync_nodes = if recorded.stale { + 0 + } else { + recorded.report.in_sync_nodes & self.ready_nodes.load(Ordering::Relaxed) + }; + PartitionStatusEntry { + in_sync_nodes, + ..recorded.report + } + } + + /// Keep the newest report of each partition, ordered by incarnation, then + /// view, then the primary's commit, received at `now`. Returns the reports + /// it kept. + pub fn record( + &self, + reports: impl IntoIterator<Item = PartitionStatusEntry>, + now: Instant, + ) -> Vec<PartitionStatusEntry> { + let mut entries = self.lock(); + let mut changes = self.changes(); + let mut accepted = Vec::new(); + for report in reports { + let known = entries.get(&report.namespace); + let replace = known.is_none_or(|known| { + (report.created_revision, report.view, report.commit) + >= ( + known.report.created_revision, + known.report.view, + known.report.commit, + ) + }); + if replace { + let moved = known.is_none_or(|known| { + known.stale + || ( + known.report.created_revision, + known.report.view, + known.report.in_sync_nodes, + ) != (report.created_revision, report.view, report.in_sync_nodes) + }); + entries.insert( + report.namespace, + Recorded { + report, + received_at: now, + stale: false, + }, + ); + if moved { + changes.mark(report.namespace); + } + accepted.push(report); + } + } + drop(changes); + drop(entries); + accepted + } + + /// The newest change, 0 before the first. + #[must_use] + pub fn last_change(&self) -> u64 { + self.changes().last + } + + /// Count the held reports of `namespaces` as the newest changes. + pub fn mark_changed(&self, namespaces: impl IntoIterator<Item = u64>) { + let entries = self.lock(); + let mut changes = self.changes(); + for namespace in namespaces { + if entries.contains_key(&namespace) { + changes.mark(namespace); + } + } + drop(changes); + drop(entries); + } + + /// How many reports this node holds, of every incarnation not yet swept. + #[must_use] + pub fn report_count(&self) -> usize { + self.lock().len() + } + + /// Up to `limit` reports whose incarnation, view or in-sync nodes changed + /// after change `after`, oldest change first, with the last change they + /// reach, the next call's `after`. + #[must_use] + pub fn changed_after( + &self, + after: u64, + limit: usize, + mut include: impl FnMut(&PartitionStatusEntry) -> bool, + ) -> (Vec<PartitionStatusEntry>, u64) { + let entries = self.lock(); + let changes = self.changes(); + let mut last = after; + let mut batch = Vec::new(); + for (&seq, namespace) in changes.by_seq.range(after.saturating_add(1)..) { + if batch.len() == limit { + break; + } + last = seq; + if let Some(recorded) = entries + .get(namespace) + .filter(|recorded| include(&recorded.report)) + { + batch.push(self.narrowed(recorded)); + } + } + drop(changes); + drop(entries); + (batch, last) + } + + /// The latest report of `namespace`'s `created_revision` incarnation. + #[must_use] + pub fn get( + &self, + namespace: IggyNamespace, + created_revision: u64, + ) -> Option<PartitionStatusEntry> { + self.lock() + .get(&namespace.inner()) + .filter(|recorded| recorded.report.created_revision == created_revision) + .map(|recorded| self.narrowed(recorded)) + } + + /// Like [`Self::get`], after marking the report stale when it was received + /// before `stale_before`. Only a newer report clears the mark. A report + /// turning stale counts as a change, so the other nodes hear of it first. + pub fn expire_and_get( + &self, + namespace: IggyNamespace, + created_revision: u64, + stale_before: Option<Instant>, + ) -> Option<PartitionStatusEntry> { + let mut entries = self.lock(); + let recorded = entries + .get_mut(&namespace.inner()) + .filter(|recorded| recorded.report.created_revision == created_revision)?; + if !recorded.stale && stale_before.is_some_and(|before| recorded.received_at < before) { + recorded.stale = true; + self.changes().mark(namespace.inner()); + } + let status = self.narrowed(recorded); + drop(entries); + Some(status) + } + + /// Up to `limit` reports in namespace order after `after`, wrapping to the + /// start, so repeated calls cycle through every partition. + #[must_use] + pub fn batch(&self, after: Option<u64>, limit: usize) -> Vec<PartitionStatusEntry> { + self.scan(after, limit, usize::MAX, |_| true).0 + } + + /// Up to `limit` reports `include` accepts, visiting at most `scan_max` + /// reports in namespace order after `after` and wrapping to the start once. + /// Returns them with the last report visited, the next call's `after`. + pub fn scan( + &self, + after: Option<u64>, + limit: usize, + scan_max: usize, + mut include: impl FnMut(&PartitionStatusEntry) -> bool, + ) -> (Vec<PartitionStatusEntry>, Option<u64>) { + let entries = self.lock(); + let start = after.map_or(Bound::Unbounded, Bound::Excluded); + let end = after.map_or(Bound::Excluded(0), Bound::Included); + let mut batch = Vec::new(); + let mut last = after; + for (namespace, recorded) in entries + .range((start, Bound::Unbounded)) + .chain(entries.range((Bound::Unbounded, end))) + .take(scan_max.min(entries.len())) + { + if batch.len() == limit { + break; + } + last = Some(*namespace); + if include(&recorded.report) { + batch.push(self.narrowed(recorded)); + } + } + drop(entries); + (batch, last) + } + + /// Drop the reports `keep` rejects among at most `limit` after `after`. + /// Returns where the next chunk starts, `None` once the sweep reached the + /// end, so repeated calls eventually visit every report. + pub fn retain_chunk( + &self, + after: Option<u64>, + limit: usize, + mut keep: impl FnMut(&PartitionStatusEntry) -> bool, + ) -> Option<u64> { + let mut entries = self.lock(); + let start = after.map_or(Bound::Unbounded, Bound::Excluded); + let mut visited = 0; + let mut last = None; + let mut rejected = Vec::new(); + for (namespace, recorded) in entries.range((start, Bound::Unbounded)).take(limit) { + visited += 1; + last = Some(*namespace); + if !keep(&recorded.report) { + rejected.push(*namespace); + } + } + let mut changes = self.changes(); + for namespace in rejected { + entries.remove(&namespace); + changes.forget(namespace); + } + drop(changes); + drop(entries); + (visited == limit).then_some(last).flatten() + } + + fn lock(&self) -> std::sync::MutexGuard<'_, BTreeMap<u64, Recorded>> { + self.entries + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn changes(&self) -> std::sync::MutexGuard<'_, Changes> { + self.changes + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + fn report(namespace: u64, created_revision: u64, view: u32) -> PartitionStatusEntry { + PartitionStatusEntry { + namespace, + created_revision, + view, + commit: 0, + current_offset: u64::from(view), + messages_count: 0, + size_bytes: 0, + segments_count: 1, + in_sync_nodes: 0, + } + } + + #[test] + fn given_reports_when_their_view_or_in_sync_nodes_change_should_list_them_in_change_order() { + let directory = PartitionDirectory::default(); + directory.record( + [report(1, 1, 0), report(2, 1, 0), report(3, 1, 0)], + Instant::now(), + ); + let (all, seen) = directory.changed_after(0, 8, |_| true); + assert_eq!(all.len(), 3); + + let mut counters_only = report(1, 1, 0); + counters_only.messages_count = 5; + counters_only.commit = 2; + directory.record([report(3, 1, 1), counters_only], Instant::now()); + let mut in_sync = report(2, 1, 0); + in_sync.in_sync_nodes = 0b11; + directory.record([in_sync], Instant::now()); + let (changed, last) = directory.changed_after(seen, 8, |_| true); + assert_eq!( + changed + .iter() + .map(|entry| entry.namespace) + .collect::<Vec<_>>(), + [3, 2], + "a new view or in-sync set is a change, new counters are not" + ); + assert_eq!(directory.changed_after(last, 8, |_| true).0, []); + + directory.mark_changed([1, 99]); + assert_eq!( + directory + .changed_after(last, 8, |_| true) + .0 + .iter() + .map(|entry| entry.namespace) + .collect::<Vec<_>>(), + [1], + "a marked report goes out again, a namespace without one is no change" + ); + assert_eq!(directory.last_change(), last + 1); + + directory.retain_chunk(None, 8, |entry| entry.namespace != 3); + assert_eq!( + directory + .changed_after(0, 8, |_| true) + .0 + .iter() + .map(|entry| entry.namespace) + .collect::<Vec<_>>(), + [2, 1], + "a retired report leaves the change log" + ); + } + + #[test] + fn given_reports_when_recording_should_keep_the_newest_incarnation_and_view() { + let directory = PartitionDirectory::default(); + let namespace = IggyNamespace::from_raw(7); + directory.record([report(7, 1, 4), report(7, 1, 2)], Instant::now()); + assert_eq!(directory.get(namespace, 1).unwrap().view, 4); + + let mut refreshed = report(7, 1, 4); + refreshed.messages_count = 9; + refreshed.commit = 3; + assert_eq!(directory.record([refreshed], Instant::now()), [refreshed]); + assert_eq!(directory.get(namespace, 1).unwrap().messages_count, 9); + let mut reordered = refreshed; + reordered.commit = 2; + reordered.messages_count = 1; + assert_eq!( + directory.record([reordered], Instant::now()), + [], + "an older commit of the same view must not regress the counters" + ); + assert_eq!(directory.get(namespace, 1).unwrap().messages_count, 9); + + directory.record([report(7, 2, 0)], Instant::now()); + assert!( + directory.get(namespace, 1).is_none(), + "a newer incarnation replaces it" + ); + directory.record([report(7, 1, 9)], Instant::now()); + assert_eq!(directory.get(namespace, 2).unwrap().view, 0); + } + + #[test] + fn given_a_cursor_when_batching_should_cycle_through_every_report() { + let directory = PartitionDirectory::default(); + directory.record( + (1..=5).map(|namespace| report(namespace, 1, 0)), + Instant::now(), + ); + let namespaces = |batch: Vec<PartitionStatusEntry>| { + batch + .iter() + .map(|entry| entry.namespace) + .collect::<Vec<_>>() + }; + assert_eq!(namespaces(directory.batch(None, 2)), [1, 2]); + assert_eq!(namespaces(directory.batch(Some(2), 2)), [3, 4]); + assert_eq!(namespaces(directory.batch(Some(4), 2)), [5, 1]); + assert_eq!(namespaces(directory.batch(Some(5), 10)), [1, 2, 3, 4, 5]); + } + + #[test] + fn given_a_chunked_retain_when_cycling_should_visit_every_report_eventually() { + let directory = PartitionDirectory::default(); + directory.record( + (1..=5).map(|namespace| report(namespace, 1, 0)), + Instant::now(), + ); + let odd = |entry: &PartitionStatusEntry| entry.namespace % 2 == 1; + assert_eq!(directory.retain_chunk(None, 2, odd), Some(2)); + assert_eq!(directory.retain_chunk(Some(2), 2, odd), Some(4)); + assert_eq!(directory.retain_chunk(Some(4), 2, odd), None); + let namespaces = |batch: Vec<PartitionStatusEntry>| { + batch + .iter() + .map(|entry| entry.namespace) + .collect::<Vec<_>>() + }; + assert_eq!(namespaces(directory.batch(None, 10)), [1, 3, 5]); + assert_eq!( + directory.scan(Some(1), 1, 10, |entry| entry.namespace != 3), + (vec![report(5, 1, 0)], Some(5)), + "a filtered scan still advances past what it skips" + ); + assert_eq!(directory.scan(Some(5), 10, 1, |_| true).1, Some(1)); + } + + #[test] + fn given_a_report_received_before_the_cutoff_when_expiring_should_list_no_member_in_sync_until_the_next_one() + { + let directory = PartitionDirectory::default(); + let namespace = IggyNamespace::from_raw(7); + let received = Instant::now(); + let later = received + Duration::from_secs(1); + let mut healthy = report(7, 1, 0); + healthy.in_sync_nodes = 0b11; + directory.record([healthy], received); + let seen = directory.last_change(); + let in_sync = |stale_before| { + directory + .expire_and_get(namespace, 1, stale_before) + .unwrap() + .in_sync_nodes + }; + assert_eq!(in_sync(Some(received)), 0b11, "received at the cutoff"); + assert_eq!(in_sync(None), 0b11); + assert_eq!(in_sync(Some(later)), 0); + assert_eq!(directory.get(namespace, 1).unwrap().in_sync_nodes, 0); + assert_eq!( + directory.changed_after(seen, 8, |_| true).0, + [PartitionStatusEntry { + in_sync_nodes: 0, + ..healthy + }], + "the other nodes hear that no member is in sync" + ); + let seen = directory.last_change(); + assert_eq!(in_sync(None), 0, "only a new report clears the mark"); + + directory.record([healthy], later); + assert_eq!(directory.get(namespace, 1).unwrap().in_sync_nodes, 0b11); + assert_eq!( + directory.changed_after(seen, 8, |_| true).0, + [healthy], + "the same report arriving again is a change once it was stale" + ); + } +}
diff --git a/core/metadata/src/stm/result.rs b/core/metadata/src/stm/result.rs index 86b14e4..b5e619b 100644 --- a/core/metadata/src/stm/result.rs +++ b/core/metadata/src/stm/result.rs
@@ -191,6 +191,7 @@ result_enum!(CreateTopicResult { StreamNotFound = 1009, NameAlreadyExists = 2013, + InvalidReplicationFactor = 2018, InvalidPartitionsCount = 2019, TooManyTopics = 2021, PartitionIdSpaceExhausted = 3013, @@ -216,6 +217,7 @@ result_enum!(CreatePartitionsResult { StreamNotFound = 1009, TopicNotFound = 2010, + InvalidReplicationFactor = 2018, InvalidPartitionsCount = 2019, PartitionResizeDisabled = 2023, PartitionIdSpaceExhausted = 3013,
diff --git a/core/metadata/src/stm/snapshot.rs b/core/metadata/src/stm/snapshot.rs index fa42390..03313ad 100644 --- a/core/metadata/src/stm/snapshot.rs +++ b/core/metadata/src/stm/snapshot.rs
@@ -67,13 +67,19 @@ /// /// Version 14: `TopicSnapshot` gained `created_op`, the identity of a topic /// incarnation, and the snapshot carries the metadata change log. -pub const SNAPSHOT_FORMAT_VERSION: u32 = 14; +/// +/// Version 15: `PartitionSnapshot` gained `members` and `retirement_debt_op`, +/// and each pending lifecycle target `retired_members`, as trailing, defaulted +/// fields. +pub const SNAPSHOT_FORMAT_VERSION: u32 = 15; /// Oldest format version [`MetadataSnapshot::decode`] still reads. /// -/// Session protection is mandatory and its nested layout changed. Older -/// snapshots cannot prove the current retry contract and are refused. -pub const MIN_READABLE_SNAPSHOT_FORMAT_VERSION: u32 = SNAPSHOT_FORMAT_VERSION; +/// Session protection is mandatory and its nested layout changed in version +/// 14, so older snapshots cannot prove the current retry contract and are +/// refused. Version 14 reads with every partition on every node and no +/// retirement debt. +pub const MIN_READABLE_SNAPSHOT_FORMAT_VERSION: u32 = 14; /// The release that wrote a snapshot: the packed `iggy_binary_protocol` semver of /// this build. [`iggy_binary_protocol::ProtocolVersion`] documents the packing and @@ -371,6 +377,7 @@ pub(crate) fn validate(&self) -> Result<(), SnapshotError> { if let Some(streams) = &self.streams { streams.validate_ownership_history(self.sequence_number)?; + streams.validate_partition_members()?; } self.change_log.validate(self.sequence_number) } @@ -652,7 +659,7 @@ // operator's boot reading one field's bytes as another's. Changing either // number is the reminder to change the other. const FIELD_COUNT: u32 = 8; - const PINNED_VERSION: u32 = 14; + const PINNED_VERSION: u32 = 15; let encoded = MetadataSnapshot::new(0).encode().unwrap(); let mut cursor = encoded.as_slice(); @@ -668,6 +675,7 @@ } #[test] + #[allow(clippy::too_many_lines)] fn nested_snapshot_shapes_are_pinned_too() { // The top-level count above is blind to everything under it, which is how // `TopicSnapshot` once swapped a removed field for an added one and kept @@ -684,7 +692,8 @@ const CLIENT_TABLE_FIELD_COUNT: u32 = 2; // Version 5 appended `created_view` the same way. Version 10 dropped // `purge_generation` from the middle, which no default can cover. - const PARTITION_FIELD_COUNT: u32 = 6; + // `members` and `retirement_debt_op` are appended with defaults. + const PARTITION_FIELD_COUNT: u32 = 8; let client_table = consensus::ClientTableSnapshot { slots: Vec::new(), @@ -728,6 +737,8 @@ created_revision: 0, deleted_up_to_offset: 0, created_view: 0, + members: Vec::new(), + retirement_debt_op: 0, }; let encoded = rmp_serde::to_vec(&partition).unwrap(); assert_eq!( @@ -851,7 +862,7 @@ #[test] fn decode_refuses_snapshots_without_durable_session_protection() { - for version in 0..SNAPSHOT_FORMAT_VERSION { + for version in 0..MIN_READABLE_SNAPSHOT_FORMAT_VERSION { let mut snapshot = MetadataSnapshot::new(9); snapshot.version = version; let bytes = snapshot.encode().unwrap(); @@ -874,6 +885,36 @@ } #[test] + fn given_format_fourteen_partition_when_decoding_should_default_members_and_debt() { + #[derive(Serialize)] + struct FormatFourteenPartition { + id: usize, + consensus_group_id: u64, + created_at: IggyTimestamp, + created_revision: u64, + deleted_up_to_offset: u64, + created_view: u32, + } + let mut snapshot = MetadataSnapshot::new(9); + snapshot.version = MIN_READABLE_SNAPSHOT_FORMAT_VERSION; + assert!(MetadataSnapshot::decode(&snapshot.encode().unwrap()).is_ok()); + + let encoded = rmp_serde::to_vec(&FormatFourteenPartition { + id: 2, + consensus_group_id: 3, + created_at: IggyTimestamp::from(4), + created_revision: 5, + deleted_up_to_offset: 6, + created_view: 7, + }) + .unwrap(); + let decoded: PartitionSnapshot = rmp_serde::from_slice(&encoded).unwrap(); + assert_eq!((decoded.id, decoded.created_view), (2, 7)); + assert!(decoded.members.is_empty(), "stored on every node"); + assert_eq!(decoded.retirement_debt_op, 0); + } + + #[test] fn given_format_fourteen_snapshot_when_restoring_should_preserve_options_and_created_op() { const PREVIOUS_FORMAT: u32 = 13; let mut options = ResourceOptions::new(); @@ -1006,6 +1047,8 @@ created_revision: 0, deleted_up_to_offset: 0, created_view: 0, + members: vec![2, 0, 1], + retirement_debt_op: 9, }], consumer_groups: Vec::new(), // Nonzero and distinct from every id above so the @@ -1043,6 +1086,7 @@ let (_, topic) = &stream.topics[0]; assert_eq!(topic.partitions.len(), 1); assert_eq!(topic.partitions[0].consensus_group_id, 33); + assert_eq!(topic.partitions[0].retirement_debt_op, 9); // Never-reuse counter: `#[serde(default)]` would silently restore 0 if // the field were dropped from the wire format, so pin its survival. assert_eq!(topic.next_consumer_group_id, 5);
diff --git a/core/metadata/src/stm/stream.rs b/core/metadata/src/stm/stream.rs index b9e01e1..da29275 100644 --- a/core/metadata/src/stm/stream.rs +++ b/core/metadata/src/stm/stream.rs
@@ -26,21 +26,24 @@ use crate::stm::lifecycle::{ CompleteLifecycleRequest, LifecycleAction, LifecycleIntent, LifecycleScope, }; +use crate::stm::partition_directory::PartitionDirectory; use crate::stm::result::{ ApplyReply, CreatePartitionsResult, CreateStreamResult, CreateTopicResult, DeletePartitionsResult, DeleteStreamResult, DeleteTopicResult, DeletedResource, TruncatePartitionResult, UpdateStreamResult, UpdateTopicResult, }; -use crate::stm::snapshot::Snapshotable; +use crate::stm::snapshot::{SnapshotError, Snapshotable}; use crate::stm::{ApplyContext, StateHandler}; use crate::{collect_handlers, define_state, impl_fill_restore}; use ahash::{AHashMap, AHashSet}; use bytes::{BufMut, Bytes, BytesMut}; +use consensus::{ReplicaSet, ReplicaSetError}; use iggy_binary_protocol::codec::{WireDecode, WireEncode}; use iggy_binary_protocol::primitives::metadata_changes::{ WireMetadataChange, WireStreamBody, WireStreamDeletedBody, WireStreamKey, WireTopicDeletedBody, }; use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment; +use iggy_binary_protocol::primitives::partition_status::PartitionStatusEntry; use iggy_binary_protocol::requests::consumer_groups::{ CreateConsumerGroupRequest, DeleteConsumerGroupRequest, }; @@ -57,6 +60,7 @@ // their cfg the import is dead and `-p <crate>` clippy (which skips the // simulator feature) rejects it. use iggy_binary_protocol::primitives::partition_history::{ConsumerGroupOwner, PartitionContext}; +use iggy_binary_protocol::requests::system::RetirementDebt; #[cfg(any(test, feature = "simulator"))] use iggy_binary_protocol::requests::topics::CreateTopicRequest; use iggy_binary_protocol::requests::topics::{ @@ -83,6 +87,7 @@ use server_common::sharding::{IggyNamespace, MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS}; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Instant; /// Removals [`RemovedPartitions`] keeps. A reader that falls further behind /// rebuilds from the tree, which a delete of this many partitions costs anyway. @@ -105,6 +110,13 @@ /// view every group started in before it existed. #[serde(default)] pub created_view: u32, + /// Node ids storing the partition, in member order. Empty for a partition + /// every node stores, the only layout before replication factors. + #[serde(default)] + pub members: Vec<u8>, + /// `#[serde(default)]` so snapshots predating retirement debt restore at 0. + #[serde(default)] + pub retirement_debt_op: u64, } #[derive(Debug, Clone)] @@ -127,6 +139,13 @@ /// `TruncatePartition` (the resolved form of a client `DeleteSegments`). /// `0` means nothing has been trimmed. pub deleted_up_to_offset: u64, + /// The nodes storing this partition. `None` for one created before + /// replication factors, which every node stores. + pub members: Option<ReplicaSet>, + /// Newest `FinalizeSession` op that finalized a session this incarnation + /// had not proven retired, `0` for none. The partition pays it by + /// committing a retirement whose `paid_through` reaches it. + pub retirement_debt_op: u64, } impl Partition { @@ -145,6 +164,180 @@ created_revision, created_view, deleted_up_to_offset: 0, + members: None, + retirement_debt_op: 0, + } + } + + #[must_use] + pub const fn with_members(mut self, members: ReplicaSet) -> Self { + self.members = Some(members); + self + } + + /// The nodes storing this partition in a cluster of `nodes` nodes. + /// + /// # Panics + /// If `nodes` is zero or above the replica limit. + #[must_use] + pub fn replica_set(&self, nodes: u8) -> ReplicaSet { + self.members.unwrap_or_else(|| ReplicaSet::all(nodes)) + } + + /// Node id of the primary `view` assigns, in a cluster of `nodes` nodes. + #[must_use] + pub fn primary_node(&self, view: u32, nodes: u8) -> Option<u8> { + primary_of(self.members, view, nodes) + } +} + +/// Whether `status` reports the committed incarnation of its partition. +fn committed_report(inner: &StreamsInner, status: &PartitionStatusEntry) -> bool { + let namespace = IggyNamespace::from_raw(status.namespace); + inner + .items + .get(namespace.stream_id()) + .and_then(|stream| stream.topics.get(namespace.topic_id())) + .and_then(|topic| find_partition(&topic.partitions, namespace.partition_id())) + .is_some_and(|partition| partition.created_revision == status.created_revision) +} + +/// Partitions short of their factor and partitions that cannot commit, as +/// [`Streams::partition_replication`] counts them. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct PartitionReplication { + pub under_replicated: usize, + pub below_write_quorum: usize, +} + +/// When each committed partition, by namespace and incarnation, was first +/// found without a report, kept by the metadata primary from one +/// [`Streams::partition_replication`] scan to the next. +#[derive(Debug, Default)] +pub struct UnreportedPartitions { + since: AHashMap<(u64, u64), Instant>, +} + +/// Node id of the primary `view` assigns among `members`, or among nodes +/// `0..nodes` when there is no member list. +#[must_use] +pub fn primary_of(members: Option<ReplicaSet>, view: u32, nodes: u8) -> Option<u8> { + match members { + Some(members) => members.node(u8::try_from(view % u32::from(members.len())).ok()?), + None if nodes > 0 => u8::try_from(view % u32::from(nodes)).ok(), + None => None, + } +} + +/// Decode the member list of every committed assignment. Empty means every node. +fn assignment_members( + assignments: &[CreatedPartitionAssignment], +) -> Result<Vec<Option<ReplicaSet>>, ReplicaSetError> { + assignments + .iter() + .map(|assignment| { + (!assignment.members.is_empty()) + .then(|| ReplicaSet::new(&assignment.members)) + .transpose() + }) + .collect() +} + +/// The assignments' member lists, when each lists as many members as `topic`'s +/// replication factor, and none for a topic created without one. Admission +/// reads the factor from a possibly older view of the topic. +fn members_for_topic( + topic: &Topic, + assignments: &[CreatedPartitionAssignment], +) -> Option<Vec<Option<ReplicaSet>>> { + let factor = TopicCreateOptions::from_resource_options(&topic.options) + .replication_factor + .map_or(0, |factor| factor as usize); + assignments + .iter() + .all(|assignment| assignment.members.len() == factor) + .then(|| assignment_members(assignments).ok()) + .flatten() +} + +impl TryFrom<PartitionSnapshot> for Partition { + type Error = ReplicaSetError; + + fn try_from(snapshot: PartitionSnapshot) -> Result<Self, Self::Error> { + Ok(Self { + id: snapshot.id, + consensus_group_id: snapshot.consensus_group_id, + created_at: snapshot.created_at, + created_revision: snapshot.created_revision, + created_view: snapshot.created_view, + deleted_up_to_offset: snapshot.deleted_up_to_offset, + members: (!snapshot.members.is_empty()) + .then(|| ReplicaSet::new(&snapshot.members)) + .transpose()?, + retirement_debt_op: snapshot.retirement_debt_op, + }) + } +} + +/// Committed partitions counted the ways placement reads them, so it needs no +/// walk. Rebuilt on restore. +#[derive(Debug, Clone, Default)] +pub(crate) struct PlacementIndex { + /// Partitions without a member list, by created view. + pub(crate) created_views: BTreeMap<u32, u64>, + /// Partitions with a member list, by the node their created view names. + pub(crate) created_primaries: BTreeMap<u8, u64>, + /// Partitions with a member list, once for every node storing them. + pub(crate) stored_replicas: BTreeMap<u8, u64>, +} + +impl PlacementIndex { + fn build(items: &IdSlab<Stream>) -> Self { + let mut index = Self::default(); + for partition in items + .iter() + .flat_map(|(_, stream)| stream.topics.iter()) + .flat_map(|(_, topic)| topic.partitions.iter()) + { + index.insert(partition); + } + index + } + + fn insert(&mut self, partition: &Partition) { + let Some(members) = partition.members else { + *self + .created_views + .entry(partition.created_view) + .or_default() += 1; + return; + }; + if let Some(primary) = primary_of(Some(members), partition.created_view, members.len()) { + *self.created_primaries.entry(primary).or_default() += 1; + } + for node in members.nodes() { + *self.stored_replicas.entry(*node).or_default() += 1; + } + } + + fn remove(&mut self, partition: &Partition) { + fn decrement<K: Ord>(counts: &mut BTreeMap<K, u64>, key: &K) { + if let Some(count) = counts.get_mut(key) { + *count -= 1; + if *count == 0 { + counts.remove(key); + } + } + } + let Some(members) = partition.members else { + decrement(&mut self.created_views, &partition.created_view); + return; + }; + if let Some(primary) = primary_of(Some(members), partition.created_view, members.len()) { + decrement(&mut self.created_primaries, &primary); + } + for node in members.nodes() { + decrement(&mut self.stored_replicas, node); } } } @@ -423,11 +616,15 @@ /// the next `publish()` still sees the partition listed and can mint the entry /// back. What it does buy is that the resurrected entry carries an id the /// evicting op named, so the deferred `absorb_second` run evicts it again. +/// +/// It also carries the [`PartitionDirectory`]: the same node-local, cross-shard +/// lifetime, for partitions this node hears about but does not store. #[derive(Debug, Default)] pub struct StatsRegistry { streams: std::sync::Mutex<AHashMap<usize, Arc<StreamStats>>>, topics: std::sync::Mutex<AHashMap<(usize, usize), Arc<TopicStats>>>, partitions: std::sync::Mutex<AHashMap<(usize, usize, usize), PartitionEntry>>, + directory: PartitionDirectory, } /// Shared partition counters plus the identity of the partition they count. @@ -445,6 +642,11 @@ } impl StatsRegistry { + #[must_use] + pub const fn directory(&self) -> &PartitionDirectory { + &self.directory + } + fn stream(&self, id: usize) -> Arc<StreamStats> { self.streams .lock() @@ -791,6 +993,11 @@ // Retirement proofs survive truncation, but not a changed set of // partition incarnations. pub namespace_revision: u64, + // Newest op that recorded a retirement debt. Rebuilt on restore. + pub retirement_debt_revision: u64, + // Debts the newest commits recorded, so a debt reader absorbs only new + // ones without a walk. A restore starts it empty. + pub(crate) recent_retirement_debts: RecentRetirementDebts, pub lifecycle_intents: BTreeMap<u64, LifecycleIntent>, // The keys of `lifecycle_intents` ordered by stream and topic, so a // blocker query visits only its own scope. Rebuilt on restore. @@ -802,9 +1009,7 @@ pub(crate) pending_revocations: BTreeMap<(usize, usize, u64, usize), ConsumerGroupOwnershipTransition>, // Locations are (stream, topic, group, member); never persisted. pub(crate) consumer_group_members: AHashMap<u128, Vec<(usize, usize, u64, usize)>>, - // Committed partitions per created view, so placement counts them - // without a walk. Rebuilt on restore. - pub(crate) created_views: BTreeMap<u32, u64>, + pub(crate) placement_index: PlacementIndex, // Partitions the newest commits removed, so placement retires what it // observed of them without a walk. A restore starts it empty. pub(crate) removed_partitions: RemovedPartitions, @@ -815,6 +1020,44 @@ } } +/// A partition owing a retirement: `(namespace, created_revision, debt_op)`. +pub type OwedRetirement = (IggyNamespace, u64, u64); + +/// Debt entries [`RecentRetirementDebts`] keeps. A reader that falls further +/// behind rescans the tree. +pub(crate) const RECENT_RETIREMENT_DEBTS_RETAINED: usize = 65_536; + +/// Retirement debts that recent commits recorded, oldest first, each stamped +/// with the op that recorded it. +#[derive(Debug, Clone, Default)] +pub(crate) struct RecentRetirementDebts { + entries: VecDeque<(u64, RetirementDebt)>, + /// Newest op whose entries are gone, evicted or never recorded. + forgotten_through: u64, +} + +impl RecentRetirementDebts { + /// Every debt recorded after `op`, or `None` when some of them are no + /// longer recorded. + fn since(&self, op: u64) -> Option<impl Iterator<Item = &(u64, RetirementDebt)>> { + (self.forgotten_through <= op).then(|| { + let first = self + .entries + .partition_point(|(recorded, _)| *recorded <= op); + self.entries.range(first..) + }) + } + + fn push(&mut self, op: u64, debt: RetirementDebt) { + if self.entries.len() == RECENT_RETIREMENT_DEBTS_RETAINED + && let Some((evicted, _)) = self.entries.pop_front() + { + self.forgotten_through = evicted; + } + self.entries.push_back((op, debt)); + } +} + /// Partitions that recent commits removed, oldest first, each stamped with /// the `namespace_revision` its commit set. Only the newest entries stay, so a /// reader that last looked before [`Self::since`] can answer must start over. @@ -1021,7 +1264,41 @@ CompleteLifecycle, RewrapStreamKeys, } - internal { RefreshConsumerGroupSession, RemoveConsumerGroupMembers } + internal { RefreshConsumerGroupSession, RemoveConsumerGroupMembers, RecordRetirementDebt } +} + +/// The retirement debts of one committed `FinalizeSession`. Never decoded from +/// the wire: the op body carries them. +#[derive(Debug, Clone)] +pub struct RecordRetirementDebtRequest { + pub debts: Vec<RetirementDebt>, +} + +impl StateHandler for RecordRetirementDebtRequest { + type State = StreamsInner; + fn apply(&self, state: &mut StreamsInner, _timestamp: IggyTimestamp) -> ApplyReply { + let op = state.apply_context.metadata_op; + for debt in &self.debts { + let namespace = IggyNamespace::from_raw(debt.namespace); + let Some(partition) = state + .items + .get_mut(namespace.stream_id()) + .and_then(|stream| stream.topics.get_mut(namespace.topic_id())) + .and_then(|topic| { + find_partition_mut(&mut topic.partitions, namespace.partition_id()) + }) + else { + continue; + }; + // A recreated incarnation never inherits an older one's debt. + if partition.created_revision == debt.created_revision { + partition.retirement_debt_op = partition.retirement_debt_op.max(op); + state.retirement_debt_revision = state.retirement_debt_revision.max(op); + state.recent_retirement_debts.push(op, *debt); + } + } + ApplyReply::ok(Bytes::new()) + } } impl StreamsInner { @@ -1058,12 +1335,7 @@ /// that commit set `namespace_revision`. fn retire_partitions(&mut self, stream_id: usize, topic_id: usize, partitions: &[Partition]) { for partition in partitions { - if let Some(count) = self.created_views.get_mut(&partition.created_view) { - *count -= 1; - if *count == 0 { - self.created_views.remove(&partition.created_view); - } - } + self.placement_index.remove(partition); self.removed_partitions.push(RemovedPartition { namespace_revision: self.namespace_revision, namespace: IggyNamespace::new(stream_id, topic_id, partition.id), @@ -2029,6 +2301,399 @@ self.with_committed_partition(namespace, |partition| partition.created_view) } + /// Take in partition reports from their primaries, received at `now`. + /// Counters of a partition `self_node` does not store follow its report; + /// a stored one keeps its own. + pub fn record_partition_statuses( + &self, + statuses: &[PartitionStatusEntry], + self_node: u8, + nodes: u8, + now: Instant, + ) { + self.inner.read(|inner| { + let registry = &inner.stats_registry; + for status in registry.directory().record(statuses.iter().copied(), now) { + let namespace = IggyNamespace::from_raw(status.namespace); + let Some(topic) = inner + .items + .get(namespace.stream_id()) + .and_then(|stream| stream.topics.get(namespace.topic_id())) + else { + continue; + }; + let Some(partition) = find_partition(&topic.partitions, namespace.partition_id()) + else { + continue; + }; + if partition.created_revision != status.created_revision + || partition.replica_set(nodes).member(self_node).is_some() + { + continue; + } + registry + .partition( + namespace.stream_id(), + namespace.topic_id(), + partition, + topic.stats.clone(), + ) + .store_reported( + status.size_bytes, + status.messages_count, + status.segments_count, + status.current_offset, + ); + } + }); + } + + /// The newest directory change, so a cursor starting there sends only + /// later ones. + #[must_use] + pub fn partition_status_last_change(&self) -> u64 { + self.inner + .read(|inner| inner.stats_registry.directory().last_change()) + } + + /// Count the reports of `namespaces` as changed after every earlier + /// change, so a cursor that already passed them sends them again. + pub fn mark_partition_statuses_changed(&self, namespaces: &[u64]) { + self.inner.read(|inner| { + inner + .stats_registry + .directory() + .mark_changed(namespaces.iter().copied()); + }); + } + + /// How many partition reports this node holds. + #[must_use] + pub fn partition_status_count(&self) -> usize { + self.inner + .read(|inner| inner.stats_registry.directory().report_count()) + } + + /// Up to `limit` directory reports that still name a committed incarnation + /// and changed their incarnation, view or in-sync nodes after change + /// `after`, with the last change they reach as the next call's `after`. + #[must_use] + pub fn partition_status_changes( + &self, + after: u64, + limit: usize, + ) -> (Vec<PartitionStatusEntry>, u64) { + self.inner.read(|inner| { + inner + .stats_registry + .directory() + .changed_after(after, limit, |status| committed_report(inner, status)) + }) + } + + /// Up to `limit` directory reports after `after` that still name a + /// committed incarnation. A peer storing a partition reads its counters + /// locally but still takes its report, for the in-sync nodes. Visits at + /// most `scan_max` reports and returns the last one visited as the next + /// cursor. + #[must_use] + pub fn partition_status_batch( + &self, + after: Option<u64>, + limit: usize, + scan_max: usize, + ) -> (Vec<PartitionStatusEntry>, Option<u64>) { + self.inner.read(|inner| { + inner + .stats_registry + .directory() + .scan(after, limit, scan_max, |status| { + committed_report(inner, status) + }) + }) + } + + /// Narrow the in-sync nodes the directory's reports carry to + /// `ready_nodes`, the nodes the metadata primary hears from, or stop + /// narrowing them with `u32::MAX`. + pub fn set_partition_directory_ready_nodes(&self, ready_nodes: u32) { + self.inner.read(|inner| { + inner + .stats_registry + .directory() + .set_ready_nodes(ready_nodes); + }); + } + + /// The directory's latest report of `namespace`, if it still names the + /// committed incarnation. + #[must_use] + pub fn partition_status(&self, namespace: IggyNamespace) -> Option<PartitionStatusEntry> { + self.inner.read(|inner| { + let created_revision = inner + .items + .get(namespace.stream_id()) + .and_then(|stream| stream.topics.get(namespace.topic_id())) + .and_then(|topic| find_partition(&topic.partitions, namespace.partition_id()))? + .created_revision; + inner + .stats_registry + .directory() + .get(namespace, created_revision) + }) + } + + /// Forget, among at most `limit` reports after `after`, those of + /// partitions no longer committed in the incarnation they describe. + /// Returns where the next chunk starts, `None` after the last one. + pub fn retain_partition_directory(&self, after: Option<u64>, limit: usize) -> Option<u64> { + self.inner.read(|inner| { + inner + .stats_registry + .directory() + .retain_chunk(after, limit, |status| { + let namespace = IggyNamespace::from_raw(status.namespace); + inner + .items + .get(namespace.stream_id()) + .and_then(|stream| stream.topics.get(namespace.topic_id())) + .and_then(|topic| { + find_partition(&topic.partitions, namespace.partition_id()) + }) + .is_some_and(|partition| { + partition.created_revision == status.created_revision + }) + }) + }) + } + + /// Record what a committed `FinalizeSession` at `metadata_op` left owed. + /// Applied on every replica as a side effect of that op. + pub fn record_retirement_debt(&self, debts: &[RetirementDebt], metadata_op: u64) { + if debts.is_empty() { + return; + } + let command = StreamsCommand::RecordRetirementDebt( + RecordRetirementDebtRequest { + debts: debts.to_vec(), + }, + IggyTimestamp::default(), + ApplyContext { + metadata_op, + ..ApplyContext::default() + }, + ); + if let Err(error) = self.inner.try_apply(command) { + tracing::error!(%error, "record_retirement_debt dispatched to reader-only Streams STM"); + } + } + + /// Every committed partition `node` stores that owes a retirement, as + /// `(namespace, created_revision, retirement_debt_op)`, with the newest + /// debt op. A walk: prefer [`Self::retirement_debts_since`]. + #[must_use] + pub fn retirement_debts_stored_by(&self, node: u8, nodes: u8) -> (Vec<OwedRetirement>, u64) { + self.inner.read(|inner| { + let mut debts = Vec::new(); + for (stream_id, stream) in &inner.items { + for (topic_id, topic) in &stream.topics { + for partition in &topic.partitions { + if partition.retirement_debt_op != 0 + && partition.replica_set(nodes).member(node).is_some() + { + debts.push(( + IggyNamespace::new(stream_id, topic_id, partition.id), + partition.created_revision, + partition.retirement_debt_op, + )); + } + } + } + } + (debts, inner.retirement_debt_revision) + }) + } + + /// The debts of committed partitions `node` stores that ops after + /// `seen_through` recorded, with the newest debt op, or `None` when they + /// are no longer all recorded. + #[must_use] + pub fn retirement_debts_since( + &self, + seen_through: u64, + node: u8, + nodes: u8, + ) -> Option<(Vec<OwedRetirement>, u64)> { + self.inner.read(|inner| { + let debts = inner + .recent_retirement_debts + .since(seen_through)? + .filter_map(|(op, debt)| { + let namespace = IggyNamespace::from_raw(debt.namespace); + let partition = find_partition( + &inner + .items + .get(namespace.stream_id())? + .topics + .get(namespace.topic_id())? + .partitions, + namespace.partition_id(), + )?; + (partition.created_revision == debt.created_revision + && partition.replica_set(nodes).member(node).is_some()) + .then_some((namespace, debt.created_revision, *op)) + }) + .collect(); + Some((debts, inner.retirement_debt_revision)) + }) + } + + /// The committed partition's retirement debt op if it is still the + /// `created_revision` incarnation. + #[must_use] + pub fn retirement_debt_for( + &self, + namespace: IggyNamespace, + created_revision: u64, + ) -> Option<u64> { + self.with_committed_partition(namespace, |partition| { + (partition.created_revision == created_revision).then_some(partition.retirement_debt_op) + }) + .flatten() + } + + /// A committed partition naming a member outside a cluster of `nodes` + /// nodes, with that member: state the cluster cannot serve. + #[must_use] + pub fn member_outside_cluster(&self, nodes: u8) -> Option<(IggyNamespace, u8)> { + self.inner.read(|inner| { + inner.items.iter().find_map(|(stream_id, stream)| { + stream.topics.iter().find_map(|(topic_id, topic)| { + topic.partitions.iter().find_map(|partition| { + let node = partition + .members? + .nodes() + .iter() + .copied() + .find(|node| *node >= nodes)?; + Some((IggyNamespace::new(stream_id, topic_id, partition.id), node)) + }) + }) + }) + }) + } + + /// The committed partition's member list and created view, with the view + /// its primary last reported for that incarnation, from one read. + #[must_use] + pub fn partition_route_inputs( + &self, + namespace: IggyNamespace, + ) -> Option<(Option<ReplicaSet>, u32, Option<u32>)> { + self.inner.read(|inner| { + let partition = find_partition( + &inner + .items + .get(namespace.stream_id())? + .topics + .get(namespace.topic_id())? + .partitions, + namespace.partition_id(), + )?; + let reported = inner + .stats_registry + .directory() + .get(namespace, partition.created_revision) + .map(|status| status.view); + Some((partition.members, partition.created_view, reported)) + }) + } + + /// How many committed partitions run short of their factor, and how many + /// cannot commit, counted once for the whole cluster at `now`. A partition + /// whose primary, as last reported or created before any report, is `live` + /// counts the members of its last report that are live too. One whose + /// primary is not live counts its live members, which must elect a primary + /// before anything commits. + /// + /// A primary reports only while it leads, so a report received before + /// `stale_before` lists no member in sync, here and in every read of it + /// on this node. A partition with no report counts the same way once it + /// was first found without one before `stale_before`, and is left out + /// until then. + #[must_use] + pub fn partition_replication( + &self, + live: &[bool], + now: Instant, + stale_before: Option<Instant>, + unreported: &mut UnreportedPartitions, + ) -> PartitionReplication { + let nodes = u8::try_from(live.len()).unwrap_or(u8::MAX); + let is_live = |node: u8| live.get(usize::from(node)).copied().unwrap_or(false); + let mut previous = std::mem::take(&mut unreported.since); + self.inner.read(|inner| { + let directory = inner.stats_registry.directory(); + let mut counts = PartitionReplication::default(); + for (stream_id, stream) in &inner.items { + for (topic_id, topic) in &stream.topics { + for partition in &topic.partitions { + let members = partition.replica_set(nodes); + let namespace = IggyNamespace::new(stream_id, topic_id, partition.id); + let report = directory.expire_and_get( + namespace, + partition.created_revision, + stale_before, + ); + let unreported_long = report.is_none() && { + let key = (namespace.inner(), partition.created_revision); + let since = previous.remove(&key).unwrap_or(now); + unreported.since.insert(key, since); + stale_before.is_some_and(|before| since < before) + }; + let view = report.map_or(partition.created_view, |status| status.view); + let (holding, quorum) = + if partition.primary_node(view, nodes).is_some_and(is_live) { + let in_sync_nodes = match report { + Some(report) => report.in_sync_nodes, + None if unreported_long => 0, + None => continue, + }; + let in_sync = members + .nodes() + .iter() + .filter(|&&node| { + 1_u32 + .checked_shl(u32::from(node)) + .is_some_and(|bit| in_sync_nodes & bit != 0) + }) + .count(); + (in_sync, consensus::quorum_replication(members.len())) + } else { + let live_members = members + .nodes() + .iter() + .filter(|&&node| is_live(node)) + .count(); + (live_members, consensus::quorum_view_change(members.len())) + }; + counts.under_replicated += + usize::from(holding < usize::from(members.len())); + counts.below_write_quorum += usize::from(holding < quorum); + } + } + } + counts + }) + } + + /// The committed partition's member list, `Some(None)` when every node + /// stores it. + #[must_use] + pub fn members_for_namespace(&self, namespace: IggyNamespace) -> Option<Option<ReplicaSet>> { + self.with_committed_partition(namespace, |partition| partition.members) + } + fn with_committed_partition<T>( &self, namespace: IggyNamespace, @@ -2104,6 +2769,12 @@ } else { Vec::new() }; + let replication_factor = partitions + .first() + .filter(|partition| !partition.members.is_empty()) + .map(|partition| { + u32::try_from(partition.members.len()).expect("member count fits u32") + }); self.inner .try_apply(StreamsCommand::CreateTopicWithAssignments( CreateTopicWithAssignmentsRequest { @@ -2116,6 +2787,7 @@ options: iggy_common::TopicCreateOptions { durability, consumer_offset_durability: durability, + replication_factor, ..Default::default() } .to_wire() @@ -2178,6 +2850,24 @@ created_view: u32, durability: iggy_common::Durability, ) { + self.seed_namespace_on(namespace, consensus_group_id, created_view, durability, &[]); + } + + /// [`Self::seed_namespace_with_durability`] storing the partitions on + /// `members`, in member order, with a topic whose replication factor is + /// their count. Empty `members` seeds the every-node layout. + /// + /// # Panics + /// As [`Self::seed_namespace`]. + #[cfg(any(test, feature = "simulator"))] + pub fn seed_namespace_on( + &self, + namespace: IggyNamespace, + consensus_group_id: u64, + created_view: u32, + durability: iggy_common::Durability, + members: &[u8], + ) { let stream_slab = namespace.stream_id(); let topic_slab = namespace.topic_id(); let partition_id = @@ -2208,6 +2898,7 @@ initial_view: created_view, partition_id: id, consensus_group_id: sibling_group_id(id), + members: members.to_vec(), }) .collect(); self.seed_stream_slabs(stream_slab); @@ -2233,6 +2924,7 @@ initial_view: created_view, partition_id: id - base, consensus_group_id: sibling_group_id(id), + members: members.to_vec(), }) .collect::<Vec<_>>(); self.inner @@ -2267,18 +2959,6 @@ } } -fn count_created_views(items: &IdSlab<Stream>) -> BTreeMap<u32, u64> { - let mut created_views = BTreeMap::new(); - for partition in items - .iter() - .flat_map(|(_, stream)| stream.topics.iter()) - .flat_map(|(_, topic)| topic.partitions.iter()) - { - *created_views.entry(partition.created_view).or_default() += 1; - } - created_views -} - /// Whether the next insert's slab key still fits the packed namespace layout. /// /// Takes `vacant_key()` rather than `len()` because the KEY is what gets packed @@ -2308,6 +2988,20 @@ .find(|partition| partition.id == partition_id) } +fn find_partition_mut(partitions: &mut [Partition], partition_id: usize) -> Option<&mut Partition> { + let index = if partitions + .get(partition_id) + .is_some_and(|partition| partition.id == partition_id) + { + partition_id + } else { + partitions + .iter() + .position(|partition| partition.id == partition_id)? + }; + partitions.get_mut(index) +} + /// Range and distinctness for the ABSOLUTE partition ids on a topic create, /// `None` when the vector is well formed. /// @@ -2592,6 +3286,10 @@ } } + let Ok(members) = assignment_members(&self.partitions) else { + return ApplyReply::err(CreateTopicResult::InvalidReplicationFactor); + }; + // Past validation: this commit adds partitions, so bump the // monotonic revision and stamp every new partition with it. let new_revision = state.revision.wrapping_add(1); @@ -2636,7 +3334,7 @@ let inserted = stream.topics.insert(topic); debug_assert_eq!(inserted, topic_id, "vacant_key must match the insert slot"); if let Some(topic) = stream.topics.get_mut(inserted) { - for partition in &self.partitions { + for (partition, members) in self.partitions.iter().zip(members) { let partition = Partition { id: partition.partition_id as usize, consensus_group_id: partition.consensus_group_id, @@ -2644,11 +3342,10 @@ created_revision: new_revision, created_view: partition.initial_view, deleted_up_to_offset: 0, + members, + retirement_debt_op: 0, }; - *state - .created_views - .entry(partition.created_view) - .or_default() += 1; + state.placement_index.insert(&partition); topic.partitions.push(partition); } } @@ -2735,6 +3432,13 @@ owner_generation: 0, metadata_op, }, + // A new topic always lists its members. Nothing is in sync + // before the partition's primary first reports. + members: p + .members + .map(|members| members.nodes().to_vec()) + .unwrap_or_default(), + in_sync_members: Vec::new(), }) }) .collect::<Result<Vec<PartitionResponse>, _>>() @@ -3035,7 +3739,7 @@ // the revision bump. Validate every id transition before mutating // topic.partitions; mid-batch overflow + retry would otherwise // re-base over a partial set and mint duplicate ids. - let resolved: Vec<usize> = { + let (resolved, members) = { let Some(stream) = state.items.get_mut(stream_id) else { return ApplyReply::err(CreatePartitionsResult::StreamNotFound); }; @@ -3045,6 +3749,11 @@ if topic.resolved.partition_resize_policy == PartitionResizePolicy::Fixed { return ApplyReply::err(CreatePartitionsResult::PartitionResizeDisabled); } + // Not `TransientNotAccepted`: a committed refusal is cached against + // the request id, so a retry would replay it instead of admitting. + let Some(members) = members_for_topic(topic, &self.partitions) else { + return ApplyReply::err(CreatePartitionsResult::InvalidReplicationFactor); + }; let base_partition_id = topic .partitions @@ -3082,7 +3791,7 @@ } resolved.push(resolved_id_usize); } - resolved + (resolved, members) }; let new_revision = state.revision.wrapping_add(1); @@ -3095,19 +3804,23 @@ let Some(topic) = stream.topics.get_mut(topic_id) else { return ApplyReply::err(CreatePartitionsResult::TopicNotFound); }; - for (resolved_id_usize, partition) in resolved.into_iter().zip(self.partitions.iter()) { - *state - .created_views - .entry(partition.initial_view) - .or_default() += 1; - topic.partitions.push(Partition { + for ((resolved_id_usize, partition), members) in resolved + .into_iter() + .zip(self.partitions.iter()) + .zip(members) + { + let partition = Partition { id: resolved_id_usize, consensus_group_id: partition.consensus_group_id, created_at: timestamp, created_revision: new_revision, created_view: partition.initial_view, deleted_up_to_offset: 0, - }); + members, + retirement_debt_op: 0, + }; + state.placement_index.insert(&partition); + topic.partitions.push(partition); } // Added partitions are unassigned until the groups rebalance. topic.rebalance_consumer_groups(state.apply_context.metadata_op, timestamp); @@ -3274,6 +3987,11 @@ created_revision: p.created_revision, created_view: p.created_view, deleted_up_to_offset: p.deleted_up_to_offset, + members: p + .members + .map(|members| members.nodes().to_vec()) + .unwrap_or_default(), + retirement_debt_op: p.retirement_debt_op, }) .collect(), consumer_groups: topic @@ -3321,6 +4039,7 @@ snapshot: Self::Snapshot, ) -> Result<Self, crate::stm::snapshot::SnapshotError> { snapshot.validate_ownership_history(u64::MAX)?; + snapshot.validate_partition_members()?; // Boot: no live registry exists yet, so mint one. Safe because // `new_from_empty` clones this single inner onto the other left-right // buffer rather than building a second one. @@ -3333,6 +4052,26 @@ } } +impl StreamsSnapshot { + pub(crate) fn validate_partition_members(&self) -> Result<(), SnapshotError> { + let valid = self + .items + .iter() + .flat_map(|(_, stream)| stream.topics.iter()) + .flat_map(|(_, topic)| topic.partitions.iter()) + .all(|partition| { + partition.members.is_empty() || ReplicaSet::new(&partition.members).is_ok() + }); + if valid { + Ok(()) + } else { + Err(SnapshotError::InvalidState( + "partition member list is empty, oversized or repeats a node", + )) + } + } +} + impl StreamsInner { /// Rebuild from a snapshot section IN PLACE, keeping the live stats /// registry. @@ -3372,6 +4111,7 @@ /// stream that never held it, clamps, and raises the rollup-underflow alarm /// on every boot with no real divergence behind it. A snapshot's totals are /// the DONOR's, over children the receiver kept the whole time. + #[allow(clippy::too_many_lines)] pub(crate) fn inner_from_snapshot( snapshot: StreamsSnapshot, stats_registry: Arc<StatsRegistry>, @@ -3399,13 +4139,10 @@ partitions: topic_snap .partitions .into_iter() - .map(|p| Partition { - id: p.id, - consensus_group_id: p.consensus_group_id, - created_at: p.created_at, - created_revision: p.created_revision, - created_view: p.created_view, - deleted_up_to_offset: p.deleted_up_to_offset, + .map(|partition| { + Partition::try_from(partition).expect( + "StreamsSnapshot::validate_partition_members runs before restore", + ) }) .collect(), // Not snapshotted (see `TopicSnapshot`): start fresh. @@ -3462,19 +4199,31 @@ .values() .map(LifecycleIntent::scope) .collect(); - let created_views = count_created_views(&items); + let placement_index = PlacementIndex::build(&items); + let retirement_debt_revision = items + .iter() + .flat_map(|(_, stream)| stream.topics.iter()) + .flat_map(|(_, topic)| topic.partitions.iter()) + .map(|partition| partition.retirement_debt_op) + .max() + .unwrap_or(0); let mut inner = Self { index, items, revision: snapshot.revision, namespace_revision: snapshot.namespace_revision, + retirement_debt_revision, + recent_retirement_debts: RecentRetirementDebts { + entries: VecDeque::new(), + forgotten_through: retirement_debt_revision, + }, lifecycle_intents, lifecycle_scopes, finalizing_lifecycle: None, // Recomputed from the restored groups just below. pending_revocations: BTreeMap::new(), consumer_group_members: AHashMap::new(), - created_views, + placement_index, removed_partitions: RemovedPartitions { entries: VecDeque::new(), forgotten_through: snapshot.namespace_revision, @@ -3513,6 +4262,7 @@ topic_option_keys, }; use std::str::FromStr; + use std::time::Duration; #[test] fn truncate_partition_request_round_trips() { @@ -3584,6 +4334,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], }; let reply = StateHandler::apply(&request, &mut inner, IggyTimestamp::from(1)); @@ -3648,11 +4399,13 @@ partition_id: 0, consensus_group_id: 1, initial_view: 7, + members: Vec::new(), }, CreatedPartitionAssignment { partition_id: 1, consensus_group_id: 2, initial_view: 8, + members: Vec::new(), }, ], }; @@ -3672,11 +4425,13 @@ partition_id: 0, consensus_group_id: 3, initial_view: 9, + members: Vec::new(), }, CreatedPartitionAssignment { partition_id: 1, consensus_group_id: 4, initial_view: 10, + members: Vec::new(), }, ], }; @@ -3758,6 +4513,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], }; assert!(explicit.max_topic_size.is_some(), "sentinel was sent"); @@ -4096,6 +4852,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], }; assert_eq!( @@ -4188,6 +4945,7 @@ partition_id: 0, consensus_group_id: 1, initial_view: 0, + members: Vec::new(), }], }; @@ -4298,6 +5056,7 @@ partition_id: 0, consensus_group_id: 1, initial_view: 0, + members: Vec::new(), }], }; assert_eq!( @@ -4374,6 +5133,7 @@ partition_id: 0, consensus_group_id: 1, initial_view: 0, + members: Vec::new(), }], }; assert_eq!( @@ -4422,6 +5182,7 @@ partition_id: 0, consensus_group_id: 1, initial_view: 0, + members: Vec::new(), }], }; assert_eq!( @@ -4492,6 +5253,7 @@ partition_id: 0, consensus_group_id: 1, initial_view: 0, + members: Vec::new(), }], }; assert_eq!( @@ -4605,11 +5367,13 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 0, partition_id: 1, consensus_group_id: 2, + members: Vec::new(), }, ], }; @@ -4660,6 +5424,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 3, + members: Vec::new(), }], }; let delete = DeletePartitionsRequest { @@ -4736,11 +5501,13 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 0, partition_id: 1, consensus_group_id: 2, + members: Vec::new(), }, ], }; @@ -4806,11 +5573,13 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 0, partition_id: 1, consensus_group_id: 2, + members: Vec::new(), }, ], }; @@ -4836,11 +5605,13 @@ initial_view: 0, partition_id: 0, consensus_group_id: 10, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 0, partition_id: 1, consensus_group_id: 11, + members: Vec::new(), }, ], }; @@ -4857,11 +5628,13 @@ initial_view: 0, partition_id: 0, consensus_group_id: 12, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 0, partition_id: 1, consensus_group_id: 13, + members: Vec::new(), }, ], }; @@ -4895,11 +5668,13 @@ initial_view: 0, partition_id: 0, consensus_group_id: 100, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 0, partition_id: 1, consensus_group_id: 101, + members: Vec::new(), }, ], }; @@ -4931,11 +5706,13 @@ initial_view: 0, partition_id: 0, consensus_group_id: 50, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 0, partition_id: 1, consensus_group_id: 51, + members: Vec::new(), }, ], }; @@ -4954,11 +5731,13 @@ initial_view: 0, partition_id: 0, consensus_group_id: 60, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 0, partition_id: 1, consensus_group_id: 61, + members: Vec::new(), }, ], }; @@ -4990,6 +5769,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], }; let apply = StateHandler::apply(&create_partitions, &mut inner, IggyTimestamp::now()); @@ -5031,6 +5811,7 @@ initial_view: 0, partition_id, consensus_group_id: 1, + members: Vec::new(), }) .collect(), }; @@ -5079,6 +5860,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], }; let _ = crate::stm::lifecycle::apply_with_lifecycle_completion( @@ -5360,11 +6142,13 @@ initial_view: 0, partition_id: 3, consensus_group_id: 1, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 0, partition_id: 4, consensus_group_id: 2, + members: Vec::new(), }, ], }; @@ -5436,6 +6220,7 @@ initial_view: 0, partition_id, consensus_group_id: u64::from(partition_id) + 1, + members: Vec::new(), }) .collect(), }; @@ -5519,6 +6304,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], }; let _ = crate::stm::lifecycle::apply_with_lifecycle_completion( @@ -5605,6 +6391,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], }; let _ = StateHandler::apply(&create_topic, &mut inner, IggyTimestamp::now()); @@ -5680,6 +6467,7 @@ initial_view: 0, partition_id, consensus_group_id: u64::from(partition_id) + 1, + members: Vec::new(), }) .collect(), }; @@ -5977,6 +6765,7 @@ initial_view: 0, partition_id: u32::try_from(MAX_PARTITIONS - 2).unwrap(), consensus_group_id: 1, + members: Vec::new(), }], }; assert_eq!( @@ -5995,6 +6784,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 2, + members: Vec::new(), }], }; assert_eq!( @@ -6013,6 +6803,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 3, + members: Vec::new(), }], }; let reply = StateHandler::apply(&overflow, &mut inner, IggyTimestamp::now()); @@ -6042,6 +6833,7 @@ initial_view: 0, partition_id: u32::try_from(MAX_PARTITIONS).unwrap(), consensus_group_id: 1, + members: Vec::new(), }], }; let reply = StateHandler::apply(&request, &mut inner, IggyTimestamp::now()); @@ -6173,11 +6965,13 @@ initial_view: 0, partition_id: 7, consensus_group_id: 1, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 0, partition_id: 7, consensus_group_id: 2, + members: Vec::new(), }, ], }; @@ -6208,6 +7002,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], }; assert_eq!( @@ -6226,11 +7021,13 @@ initial_view: 0, partition_id: 0, consensus_group_id: 2, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 0, partition_id: 0, consensus_group_id: 3, + members: Vec::new(), }, ], }; @@ -6262,6 +7059,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], }; assert_eq!( @@ -6280,11 +7078,13 @@ initial_view: 0, partition_id: 0, consensus_group_id: 2, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 0, partition_id: 1, consensus_group_id: 3, + members: Vec::new(), }, ], }; @@ -6326,4 +7126,257 @@ assert_eq!(restored.items.vacant_key(), before); } + + #[test] + fn given_directory_reports_when_batching_for_a_peer_should_relay_committed_ones_narrowed_and_retain_in_chunks() + { + let streams = Streams::default(); + let stored_by_peer = IggyNamespace::new(0, 0, 0); + let elsewhere = IggyNamespace::new(1, 0, 0); + streams.seed_namespace_on( + stored_by_peer, + 1, + 0, + iggy_common::Durability::Persisted, + &[1], + ); + streams.seed_namespace_on(elsewhere, 2, 0, iggy_common::Durability::Persisted, &[2]); + let status = |namespace: IggyNamespace| PartitionStatusEntry { + namespace: namespace.inner(), + created_revision: streams.created_revision_for_namespace(namespace).unwrap(), + view: 0, + commit: 0, + current_offset: 0, + messages_count: 0, + size_bytes: 0, + segments_count: 1, + in_sync_nodes: 0, + }; + streams.record_partition_statuses( + &[status(stored_by_peer), status(elsewhere)], + 0, + 3, + Instant::now(), + ); + let (batch, cursor) = streams.partition_status_batch(None, 4, 4); + assert_eq!( + batch, + [status(stored_by_peer), status(elsewhere)], + "a peer storing a partition still takes its report, for the in-sync nodes" + ); + assert_eq!(cursor, Some(elsewhere.inner())); + streams.record_partition_statuses( + &[PartitionStatusEntry { + commit: 1, + in_sync_nodes: 0b111, + ..status(elsewhere) + }], + 0, + 3, + Instant::now(), + ); + streams.set_partition_directory_ready_nodes(0b011); + assert_eq!( + streams.partition_status(elsewhere).unwrap().in_sync_nodes, + 0b011, + "a node the metadata primary cannot hear is not in sync" + ); + streams.set_partition_directory_ready_nodes(u32::MAX); + + streams.record_partition_statuses( + &[PartitionStatusEntry { + created_revision: 99, + ..status(elsewhere) + }], + 0, + 3, + Instant::now(), + ); + assert_eq!( + streams.retain_partition_directory(None, 1), + Some(stored_by_peer.inner()) + ); + assert_eq!( + streams.retain_partition_directory(Some(stored_by_peer.inner()), 1), + Some(elsewhere.inner()) + ); + assert_eq!( + streams.retain_partition_directory(Some(elsewhere.inner()), 1), + None + ); + assert_eq!( + streams.partition_status_batch(None, 4, 4).0, + [status(stored_by_peer)], + "the stale incarnation's report is gone once the sweep reached it" + ); + } + + #[test] + fn given_members_down_when_counting_partitions_without_a_ready_quorum_should_leave_those_with_a_ready_primary() + { + let streams = Streams::default(); + let seed = |stream: usize, view: u32, members: &[u8]| { + let namespace = IggyNamespace::new(stream, 0, 0); + streams.seed_namespace_on( + namespace, + u64::try_from(stream + 1).unwrap(), + view, + iggy_common::Durability::Persisted, + members, + ); + namespace + }; + seed(0, 0, &[0, 1]); + seed(1, 1, &[0, 1]); + seed(2, 0, &[0, 1, 2]); + seed(3, 0, &[0]); + let moved = seed(4, 0, &[0, 1]); + seed(5, 0, &[]); + streams.record_partition_statuses( + &[PartitionStatusEntry { + namespace: moved.inner(), + created_revision: streams.created_revision_for_namespace(moved).unwrap(), + view: 1, + commit: 0, + current_offset: 0, + messages_count: 0, + size_bytes: 0, + segments_count: 1, + in_sync_nodes: 0b011, + }], + 2, + 3, + Instant::now(), + ); + let counts = |live: &[bool]| { + streams.partition_replication( + live, + Instant::now(), + None, + &mut UnreportedPartitions::default(), + ) + }; + assert_eq!( + counts(&[false, true, true]).below_write_quorum, + 2, + "only the pair led by the down node and the single copy on it" + ); + assert_eq!(counts(&[true; 3]).below_write_quorum, 0); + } + + /// A primary reports only while it leads, so a pair whose members are + /// both up but neither leads, as after both came back on empty storage, + /// keeps the last healthy report until it goes stale. + #[test] + fn given_reports_no_primary_refreshed_when_counting_should_count_no_member_in_sync() { + let streams = Streams::default(); + let reported = IggyNamespace::new(0, 0, 0); + let silent = IggyNamespace::new(1, 0, 0); + streams.seed_namespace_on(reported, 1, 0, iggy_common::Durability::Persisted, &[0, 1]); + streams.seed_namespace_on(silent, 2, 0, iggy_common::Durability::Persisted, &[1, 2]); + let healthy = PartitionStatusEntry { + namespace: reported.inner(), + created_revision: streams.created_revision_for_namespace(reported).unwrap(), + view: 0, + commit: 0, + current_offset: 0, + messages_count: 0, + size_bytes: 0, + segments_count: 1, + in_sync_nodes: 0b011, + }; + let received = Instant::now(); + let cutoff = received + Duration::from_secs(1); + let later = received + Duration::from_secs(10); + streams.record_partition_statuses(&[healthy], 2, 3, received); + let mut unreported = UnreportedPartitions::default(); + let mut counts = |now: Instant, stale_before: Option<Instant>| { + streams.partition_replication(&[true; 3], now, stale_before, &mut unreported) + }; + assert_eq!( + counts(received, None), + PartitionReplication::default(), + "a partition yet to report is left out" + ); + assert_eq!( + counts(later, Some(received)), + PartitionReplication::default(), + "both were seen at the cutoff" + ); + assert_eq!( + counts(later, Some(cutoff)), + PartitionReplication { + under_replicated: 2, + below_write_quorum: 2, + } + ); + assert_eq!( + streams.partition_status(reported).unwrap().in_sync_nodes, + 0, + "GetTopic lists no member in sync either" + ); + + streams.record_partition_statuses(&[healthy], 2, 3, later); + assert_eq!( + counts(later, Some(cutoff)), + PartitionReplication { + under_replicated: 1, + below_write_quorum: 1, + }, + "a primary that reports again holds its members" + ); + assert_eq!( + streams.partition_status(reported).unwrap().in_sync_nodes, + 0b011 + ); + } + + /// A partition without its primary needs a view-change quorum to elect, + /// three of four, while a replication quorum is only two. + #[test] + fn given_four_members_without_their_primary_when_two_remain_should_count_them_as_unable_to_elect() + { + let streams: Streams = StreamsInner::new().into(); + streams.seed_namespace_on( + IggyNamespace::new(0, 0, 0), + 1, + 0, + iggy_common::Durability::Persisted, + &[0, 1, 2, 3], + ); + let counts = |live: &[bool]| { + streams.partition_replication( + live, + Instant::now(), + None, + &mut UnreportedPartitions::default(), + ) + }; + assert_eq!( + counts(&[false, false, true, true]), + PartitionReplication { + under_replicated: 1, + below_write_quorum: 1, + } + ); + assert_eq!( + counts(&[false, true, true, true]).below_write_quorum, + 0, + "three members elect a new primary" + ); + assert_eq!( + counts(&[true, false, false, true]), + PartitionReplication::default(), + "a live primary that has not reported yet is not counted" + ); + } + + #[test] + fn given_a_member_beyond_the_node_count_when_checking_the_cluster_should_name_it() { + let streams = Streams::default(); + let namespace = IggyNamespace::new(0, 0, 0); + streams.seed_namespace_on(namespace, 1, 0, iggy_common::Durability::Persisted, &[1, 4]); + assert_eq!(streams.member_outside_cluster(3), Some((namespace, 4))); + assert_eq!(streams.member_outside_cluster(5), None); + } }
diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index 35b02c6..9d7b556 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs
@@ -326,6 +326,12 @@ #[cfg(test)] offset_dir_sync_count: Cell<usize>, pub(crate) required_metadata_frontier: u64, + /// Newest metadata `FinalizeSession` op whose retirement debt a committed + /// payment covers. Replicated with the receipts it protects. + pub(crate) retirement_paid_through: u64, + /// Newest retirement debt op this replica learned from local metadata. + /// While it exceeds `retirement_paid_through`, session requests wait. + retirement_debt: Cell<u64>, /// Durable superblock for this partition's consensus group, recording /// `(view, log_view)` across a crash so this replica can never /// re-participate in a view older than one it advertised. `None` for @@ -981,6 +987,8 @@ #[cfg(test)] offset_dir_sync_count: Cell::new(0), required_metadata_frontier: 0, + retirement_paid_through: 0, + retirement_debt: Cell::new(0), superblock: None, superblock_lock: Rc::new(LocalGate::new()), superblock_intent: Cell::new(None), @@ -1011,14 +1019,15 @@ if single_replica { partition.log.journal().inner.set_repair_retention(false); } - partition - .consensus - .set_recovered_log_complete(partition.acknowledges_only_written_prepares()); + partition.refresh_recovered_log_complete(); partition } /// Record the metadata incarnation this partition was built for. pub fn set_created_revision(&mut self, created_revision: u64) { + if created_revision != self.created_revision() { + self.retirement_debt.set(0); + } self.consensus.set_partition_incarnation(created_revision); } @@ -1046,6 +1055,38 @@ && self.dedup.get_epoch(identity.client_id) != Some(identity.session))) } + /// Arms the session gate for `debt_op`. Only a normal primary with an + /// empty pipeline lists its retry slice: the gate then keeps every session + /// request out until the payment built from that list commits. + #[must_use] + pub fn observe_retirement_debt(&self, debt_op: u64) -> RetirementDebtStatus { + self.retirement_debt + .set(self.retirement_debt.get().max(debt_op)); + if self.retirement_paid_through >= debt_op { + return RetirementDebtStatus::Paid; + } + let consensus = &self.consensus; + let can_pay = self.fatal.is_none() + && !self.requires_state_transfer() + && consensus.is_primary() + && consensus.is_normal() + && !consensus.is_transferring() + && consensus.commit_min() == consensus.commit_max() + && consensus.pipeline_is_empty(); + RetirementDebtStatus::Unpaid(can_pay.then(|| { + self.dedup + .active_sessions() + .map(|(client_id, _, session)| (client_id, session)) + .collect() + })) + } + + /// Whether session requests must wait for a retirement debt payment. + #[must_use] + pub const fn owes_session_retirement(&self) -> bool { + self.retirement_debt.get() > self.retirement_paid_through + } + fn pending_session_retirement(&self, client_id: u128, session: u64) -> bool { let matches_identity = |body: &[u8]| { RetireSessionsRequest::decode_from(body).is_ok_and(|request| { @@ -1832,6 +1873,7 @@ } pub fn ensure_materialization_recovery(&self) { + self.refresh_recovered_log_complete(); if self.materialization_missing && (self.consensus.state_transfer_stage() == consensus::StateTransferStage::Idle || self.consensus.status() == consensus::Status::ViewChange) @@ -3032,6 +3074,7 @@ .then_some(self.dedup.capacity()), retry_protection: self.dedup.watermarks_sorted(), required_metadata_frontier: self.required_metadata_frontier, + retirement_paid_through: self.retirement_paid_through, consumer_offsets: self.retained_consumer_offsets(ConsumerKind::Consumer), consumer_group_offsets: self.retained_consumer_offsets(ConsumerKind::ConsumerGroup), external_group_offsets: self.retained_consumer_offsets(ConsumerKind::ExternalGroup), @@ -3068,6 +3111,7 @@ retry_capacity, retry_protection, required_metadata_frontier, + retirement_paid_through, durable_offset, write_offset, offset_space, @@ -3094,6 +3138,7 @@ ); } self.required_metadata_frontier = required_metadata_frontier; + self.retirement_paid_through = retirement_paid_through; for (consumer_id, offset) in consumer_offsets { self.consumer_offsets.pin().insert( consumer_id as usize, @@ -3727,8 +3772,16 @@ /// Unset fields keep the shard-wide configured values. pub fn set_runtime_options(&mut self, runtime_options: TopicRuntimeOptions) { self.runtime_options = runtime_options; - self.consensus - .set_recovered_log_complete(self.acknowledges_only_written_prepares()); + self.refresh_recovered_log_complete(); + } + + /// A replica whose materialization is missing lost the prepares it + /// acknowledged, whatever the durability, so it answers view changes with + /// the empty vote rather than staying silent until a transfer. + pub(crate) fn refresh_recovered_log_complete(&self) { + self.consensus.set_recovered_log_complete( + self.acknowledges_only_written_prepares() && !self.materialization_missing, + ); } /// Whether every prepare this replica acknowledges is on stable storage @@ -4472,6 +4525,12 @@ { return Err(IggyError::TransientNotAccepted); } + // A payment starts only from an empty pipeline. A consumer committing + // on every read would otherwise keep a prepare queued at each pay read + // and hold the partition's sessions off until it drained its backlog. + if self.owes_session_retirement() { + return Err(IggyError::TransientNotAccepted); + } if self.pending_owner_install.get().is_some_and(|pending| { kind == ConsumerKind::ConsumerGroup && pending.group_id == u64::from(consumer_id) }) || self.pending_history_transition.get().is_some() @@ -5968,8 +6027,11 @@ // A fenced backup refuses prepares while the primary keeps // committing, so its drain stalls until the grant expires. && self.consensus.is_primary() - && request.target < self.consensus.replica_count() - && request.target != self.consensus.primary_index(self.consensus.view()) + && self + .consensus + .members() + .member(request.target) + .is_some_and(|target| target != self.consensus.primary_index(self.consensus.view())) && self.consensus.is_normal() && !self.consensus.has_ceded_primaryship() && !self.consensus.is_transferring() @@ -7578,6 +7640,19 @@ } else { None }; + // Checked before dedup: a finalized session's cached receipt must + // not be served either until the debt is paid. + if !is_auto_commit_client(client_id) && self.owes_session_retirement() { + Self::send_partition_deny_or_log( + consensus, + message.header(), + IggyError::TransientNotAccepted.as_code(), + "partition owes session retirement", + reply.take(), + ) + .await; + return; + } if !is_auto_commit_client(client_id) && message.header().metadata_watermark < self.required_metadata_frontier { @@ -11344,6 +11419,12 @@ .required_metadata_frontier .max(identity.metadata_watermark); } + // A finalized session absent from the slice still must not + // start writing here, so the frontier covers the whole debt. + self.retirement_paid_through = + self.retirement_paid_through.max(retirement.paid_through); + self.required_metadata_frontier = + self.required_metadata_frontier.max(retirement.paid_through); } let reply = match self.commit_receipt( &entry, @@ -13831,6 +13912,15 @@ /// each with an awaited reply send. const PROMOTION_DENIALS_MAX: usize = 4; +/// Answer of [`IggyPartition::observe_retirement_debt`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RetirementDebtStatus { + /// A committed payment covers the debt. + Paid, + /// The retry slice as `(client_id, session)` when this replica can pay now. + Unpaid(Option<Vec<(u128, u64)>>), +} + /// What one call to [`IggyPartition::remove_sealed_segments_up_to`] reclaimed. /// /// `budget_spent` reports that the pass stopped on @@ -15830,6 +15920,8 @@ .segment_checkpoint() .is_none() ); + assert!(!partition.consensus().recovery_election_allowed()); + assert!(!partition.consensus().recovered_log_complete()); let prepare = checksummed_segment_prepare(1, 0, 0, b"transferred"); let mut staged = Vec::new(); if materialized { @@ -15860,6 +15952,7 @@ groups: Vec::new(), external_groups: Vec::new(), dedup: Vec::new(), + retirement_paid_through: 0, }; partition .install_state_transfer(&repair_config(), 1, staged, &offsets.encode()) @@ -15883,6 +15976,15 @@ assert!(partition.log.messages_writers().iter().all(Option::is_none)); assert!(partition.log.index_writers().last().unwrap().is_some()); assert_eq!(partition.consensus().commit_min(), 1); + assert!(!partition.requires_state_transfer()); + assert!( + partition.consensus().recovery_election_allowed(), + "{durability:?}: the installed copy may vote and lead again" + ); + assert_eq!( + partition.consensus().recovered_log_complete(), + durability.is_persisted() + ); } } } @@ -15931,6 +16033,7 @@ groups: Vec::new(), external_groups: Vec::new(), dedup: Vec::new(), + retirement_paid_through: 0, }; partition .install_state_transfer(&repair_config(), 3, staged, &offsets.encode()) @@ -15981,6 +16084,7 @@ groups: Vec::new(), external_groups: Vec::new(), dedup: Vec::new(), + retirement_paid_through: 0, }; assert!(matches!( partition @@ -16207,6 +16311,7 @@ owners: Vec::new(), history_transition: None, owner_retirement: None, + retirement_paid_through: 0, }; let result = partition @@ -18622,6 +18727,122 @@ } } + fn retirement_payment_request( + namespace: IggyNamespace, + identities: Vec<SessionIdentity>, + paid_through: u64, + ) -> Message<RoutedRequestHeader> { + let body = RetireSessionsRequest { + identities, + paid_through, + } + .to_bytes(); + let size = size_of::<RoutedRequestHeader>() + body.len(); + let mut request = Message::<RoutedRequestHeader>::new(size).transmute_header( + |_, header: &mut RoutedRequestHeader| { + header.command = Command::Request; + header.operation = Operation::RetireSession; + header.cluster = TEST_CLUSTER; + header.client = message_bus::AUTO_COMMIT_CLIENT_ID; + header.session = 1; + header.request = paid_through; + header.metadata_watermark = paid_through; + header.group = namespace.inner(); + header.size = u32::try_from(size).unwrap(); + }, + ); + request.as_mut_slice()[size_of::<RoutedRequestHeader>()..].copy_from_slice(&body); + request + } + + #[compio::test] + async fn given_retirement_debt_when_session_requests_arrive_should_refuse_them_until_the_payment_commits() + { + const DEBT_OP: u64 = 40; + const FINALIZED_CLIENT: u128 = 1; + const NEW_CLIENT: u128 = 2; + let (mut partition, _replies) = recording_partition_at(0, 3); + let namespace = partition.namespace(); + let send = |client_id: u128, metadata_watermark: u64| { + let mut send = checksumless_send_request(namespace, 1).transmute_header( + |header, next: &mut RoutedRequestHeader| { + *next = header; + next.client = client_id; + next.metadata_watermark = metadata_watermark; + }, + ); + send.as_mut_slice()[size_of::<RoutedRequestHeader>()..] + .copy_from_slice(&build_segment_record(namespace, 0)); + send + }; + partition.on_request(send(FINALIZED_CLIENT, 0), None).await; + partition.consensus.advance_commit_max(1); + partition.commit_journal(&repair_config()).await; + + assert_eq!( + partition.observe_retirement_debt(DEBT_OP), + RetirementDebtStatus::Unpaid(Some(vec![(FINALIZED_CLIENT, 1)])) + ); + // The first request is the committed one's retry, a cached hit. + for request in [send(FINALIZED_CLIENT, 0), send(NEW_CLIENT, DEBT_OP)] { + let (sender, receiver) = consensus::oneshot_channel(); + partition.on_request(request, Some(sender)).await; + assert_eq!( + receiver.await.unwrap().header().status, + IggyError::TransientNotAccepted.as_code() + ); + } + assert_eq!(partition.consensus.sequencer().current_sequence(), 1); + + partition + .on_request( + retirement_payment_request( + namespace, + vec![SessionIdentity { + client_id: FINALIZED_CLIENT, + session: 1, + metadata_watermark: DEBT_OP, + }], + DEBT_OP, + ), + None, + ) + .await; + assert!(partition.owes_session_retirement(), "prepared is not paid"); + partition.consensus.advance_commit_max(2); + partition.commit_journal(&repair_config()).await; + assert!(partition.fatal.is_none(), "{:?}", partition.fatal); + assert!(!partition.owes_session_retirement()); + assert_eq!( + partition.observe_retirement_debt(DEBT_OP), + RetirementDebtStatus::Paid + ); + assert!(!partition.dedup.contains(FINALIZED_CLIENT)); + assert_eq!(partition.required_metadata_frontier, DEBT_OP); + + let (sender, receiver) = consensus::oneshot_channel(); + partition + .on_request(send(FINALIZED_CLIENT, 0), Some(sender)) + .await; + assert_eq!( + receiver.await.unwrap().header().status, + IggyError::TransientNotAccepted.as_code(), + "the paid frontier keeps a finalized session's stale request out" + ); + partition.on_request(send(NEW_CLIENT, DEBT_OP), None).await; + assert_eq!(partition.consensus.sequencer().current_sequence(), 3); + + let wire = crate::state_transfer::ConsumerOffsetsWire::decode( + &crate::state_transfer::ConsumerOffsetsWire { + retirement_paid_through: partition.retirement_paid_through, + ..Default::default() + } + .encode(), + ) + .unwrap(); + assert_eq!(wire.retirement_paid_through, DEBT_OP); + } + /// Pins the barrier count of a burst of ended sessions: the sweep sends the /// oldest `MAX_SESSIONS_PER_RETIREMENT` identities in one barrier, so one /// partition commits `ceil(K / MAX_SESSIONS_PER_RETIREMENT)` barriers for K @@ -19352,6 +19573,7 @@ owners: Vec::new(), history_transition: None, owner_retirement: None, + retirement_paid_through: 0, }; let installed = backup .install_state_transfer(&repair_config(), 1, vec![staged], &offsets.encode()) @@ -20264,6 +20486,7 @@ dedup: partition.dedup.watermarks_sorted(), owners: Vec::new(), history_transition: None, + retirement_paid_through: 0, }; partition .install_state_transfer(&config, 2, vec![staged], &offsets.encode()) @@ -23917,6 +24140,7 @@ groups: Vec::new(), external_groups: Vec::new(), dedup: Vec::new(), + retirement_paid_through: 0, }; let outcome = partition .install_state_transfer(&repair_config(), 12, Vec::new(), &offer.encode()) @@ -25032,6 +25256,7 @@ groups: vec![(9, 3)], external_groups: Vec::new(), dedup: Vec::new(), + retirement_paid_through: 0, }; partition .install_state_transfer(&repair_config(), 12, Vec::new(), &wire.encode()) @@ -26794,6 +27019,7 @@ groups: vec![(8, 5)], external_groups: Vec::new(), dedup: Vec::new(), + retirement_paid_through: 0, }; partition .install_state_transfer(&repair_config(), 1, Vec::new(), &wire.encode()) @@ -29494,6 +29720,32 @@ } } + /// A retirement debt is paid only from an empty pipeline, so an automatic + /// commit waits for the payment like a session request does. + #[test] + fn given_retirement_debt_when_auto_commit_poll_completes_should_reject_without_progress() { + const DEBT_OP: u64 = 40; + for owed in [false, true] { + let (mut partition, _) = recording_partition_at(0, 3); + let consumer = PollingConsumer::Consumer(7, 0); + if owed { + assert!(matches!( + partition.observe_retirement_debt(DEBT_OP), + RetirementDebtStatus::Unpaid(Some(_)) + )); + } + let read_result = poll_read_result(&partition, consumer, true, Some(9)); + let completion = partition.complete_poll(read_result); + if owed { + assert!(matches!(completion, Err(IggyError::TransientNotAccepted))); + assert_eq!(partition.get_consumer_offset(consumer), None); + assert_eq!(partition.consensus.pipeline_len(), 0); + } else { + assert!(completion.unwrap().replication.is_some()); + } + } + } + /// `on_replicate` refuses prepares during shutdown, retirement and a /// transition. An automatic commit admitted then would hold a pipeline /// slot that no journal backs, and every later op would wait behind it. @@ -30468,6 +30720,7 @@ groups: Vec::new(), external_groups: Vec::new(), dedup: Vec::new(), + retirement_paid_through: 0, }; let installed_commit_operation = 12; partition @@ -30653,6 +30906,7 @@ groups: Vec::new(), external_groups: Vec::new(), dedup: Vec::new(), + retirement_paid_through: 0, }; let offered_commit_operation = 12; let install_error = partition @@ -36142,6 +36396,7 @@ groups: Vec::new(), external_groups: Vec::new(), dedup: Vec::new(), + retirement_paid_through: 0, }; let refused = partition .install_state_transfer(&repair_config(), 12, Vec::new(), &behind.encode()) @@ -36197,6 +36452,7 @@ groups: Vec::new(), external_groups: Vec::new(), dedup: Vec::new(), + retirement_paid_through: 0, }; let refused = partition .install_state_transfer(&repair_config(), 12, Vec::new(), &stale.encode()) @@ -37414,6 +37670,7 @@ owners: Vec::new(), history_transition: None, owner_retirement: None, + retirement_paid_through: 0, }; partition .install_state_transfer(&repair_config(), INSTALLED_OPS, staged, &offsets.encode())
diff --git a/core/partitions/src/lib.rs b/core/partitions/src/lib.rs index 085d06f..601a6ee 100644 --- a/core/partitions/src/lib.rs +++ b/core/partitions/src/lib.rs
@@ -62,7 +62,7 @@ pub use iggy_index::IggyIndex; pub use iggy_index_reader::IggyIndexReader; pub use iggy_index_writer::IggyIndexWriter; -pub use iggy_partition::{IggyPartition, SegmentRemoval}; +pub use iggy_partition::{IggyPartition, RetirementDebtStatus, SegmentRemoval}; pub use iggy_partitions::IggyPartitions; pub(crate) use io::PartitionIoVerdict; pub use io::{ @@ -144,6 +144,7 @@ pub retry_capacity: Option<usize>, pub retry_protection: Vec<consensus::DedupWatermark>, pub required_metadata_frontier: u64, + pub retirement_paid_through: u64, /// Offset counter the previous incarnation had proved durable. pub durable_offset: u64, /// Highest offset it had written, durable or not.
diff --git a/core/partitions/src/state_transfer.rs b/core/partitions/src/state_transfer.rs index bed2c2a..777baf8 100644 --- a/core/partitions/src/state_transfer.rs +++ b/core/partitions/src/state_transfer.rs
@@ -576,6 +576,9 @@ pub history_transition: Option<InstalledPartitionHistory>, pub owner_retirement: Option<iggy_binary_protocol::requests::partitions::RetireConsumerGroupOwnersRequest>, + /// Trailing and optional on decode, so artifacts written before + /// retirement debt read as `0`. + pub retirement_paid_through: u64, } impl ConsumerOffsetsWire { @@ -589,7 +592,8 @@ /// history_present u8 | [history fence, partition_op u64] | /// retirement_length u32 | retirement bytes | checksum_present u8 | /// prepare_checksum u128 | prepare_length u32 | checkpoint_prepare bytes | - /// XxHash3_64 trailer`. Little-endian throughout. + /// retirement_paid_through u64 | XxHash3_64 trailer`. Little-endian + /// throughout. #[must_use] pub fn encode(&self) -> Vec<u8> { let sections = self.offset_sections(); @@ -666,6 +670,7 @@ .to_le_bytes(), ); out.extend_from_slice(&self.checkpoint_prepare); + out.extend_from_slice(&self.retirement_paid_through.to_le_bytes()); debug_assert_eq!(out.len() + size_of::<u64>(), reserved, "encode reservation"); let trailer = state_artifact_checksum(&out); out.extend_from_slice(&trailer.to_le_bytes()); @@ -757,6 +762,11 @@ return Err(ConsumerOffsetsWireError::InvalidPrepareChecksum); } let checkpoint_prepare = cursor.take(prepare_length)?.to_vec(); + let retirement_paid_through = if cursor.remaining().len() >= size_of::<u64>() { + cursor.u64()? + } else { + 0 + }; if !cursor.remaining().is_empty() { // Distinct from `Truncated`: extra bytes point at a NEWER // encoder, and telling the operator the artifact is short would @@ -779,6 +789,7 @@ owners, history_transition, owner_retirement, + retirement_paid_through, }) } @@ -820,6 +831,7 @@ .as_ref() .map_or(0, WireEncode::encoded_size) + size_of::<u64>() + + size_of::<u64>() } fn decode_retirement_section( @@ -1252,6 +1264,7 @@ dedup_entry(11, 4, 90), dedup_entry(usize::MAX as u128 + 5, 9, 91), ], + retirement_paid_through: 0, } } @@ -1435,6 +1448,7 @@ groups: Vec::new(), external_groups: Vec::new(), dedup: Vec::new(), + retirement_paid_through: 0, }; let encoded = empty.encode(); assert_eq!( @@ -1521,9 +1535,10 @@ // forgotten, and a peer on another release then reads the artifact out // of place. One entry per section makes each length cover the header, // every count field and every entry stride. Changing a length is the - // reminder to change its version. - const WITHOUT_EXTERNAL_LEN: usize = 555; - const WITH_EXTERNAL_LEN: usize = 567; + // reminder to change its version. The trailing `retirement_paid_through` + // is optional on decode, so it kept version 8 readable without a bump. + const WITHOUT_EXTERNAL_LEN: usize = 563; + const WITH_EXTERNAL_LEN: usize = 575; const INCARNATION: u64 = 1; let with_external = ConsumerOffsetsWire { @@ -1563,6 +1578,7 @@ next_group_id: 2, live_group_ids: vec![1], }), + retirement_paid_through: 0, }; let without_external = ConsumerOffsetsWire { external_groups: Vec::new(), @@ -1671,6 +1687,7 @@ groups: Vec::new(), external_groups: Vec::new(), dedup: vec![dedup_entry(9, 1, 1), dedup_entry(4, 2, 2)], + retirement_paid_through: 0, }; assert_eq!( ConsumerOffsetsWire::decode(&unordered.encode()), @@ -1694,6 +1711,7 @@ groups: Vec::new(), external_groups: Vec::new(), dedup: vec![dedup_entry(0, 1, 1), dedup_entry(4, 2, 2)], + retirement_paid_through: 0, }; assert_eq!( ConsumerOffsetsWire::decode(&reserved.encode()), @@ -1831,6 +1849,7 @@ groups: Vec::new(), external_groups: Vec::new(), dedup: Vec::new(), + retirement_paid_through: 0, }; assert_eq!( ConsumerOffsetsWire::decode(&duplicate.encode()), @@ -1853,6 +1872,7 @@ groups: vec![(9, 1), (4, 2)], external_groups: Vec::new(), dedup: Vec::new(), + retirement_paid_through: 0, }; assert_eq!( ConsumerOffsetsWire::decode(&unordered.encode()), @@ -1879,6 +1899,20 @@ } #[test] + fn given_an_artifact_without_paid_through_when_decoded_should_read_it_as_zero() { + let paid = ConsumerOffsetsWire { + retirement_paid_through: 9, + ..table() + }; + let encoded = paid.encode(); + assert_eq!(ConsumerOffsetsWire::decode(&encoded), Ok(paid)); + let mut legacy = encoded[..encoded.len() - 2 * size_of::<u64>()].to_vec(); + let trailer = state_artifact_checksum(&legacy); + legacy.extend_from_slice(&trailer.to_le_bytes()); + assert_eq!(ConsumerOffsetsWire::decode(&legacy), Ok(table())); + } + + #[test] fn given_offset_count_above_transfer_ceiling_when_validated_should_reject() { assert!(validate_consumer_offset_transfer_count(ConsumerKind::Consumer, 4, 4).is_ok()); assert!(matches!( @@ -3620,6 +3654,7 @@ owners: self.consumer_group_owners.values().copied().collect(), history_transition: self.history_transition, owner_retirement: self.owner_retirement.clone(), + retirement_paid_through: self.retirement_paid_through, }) } @@ -4298,6 +4333,12 @@ if let (Some(persistence), Some(drain)) = (&self.persistence, &install.drain) { persistence.finish_drain(drain); } + // The installed copy is what a primary of this view committed, so + // this member may vote and lead again. Nothing else lifts the bar + // a member that came back without its data was held under. + if install.failure.is_none() { + self.consensus().set_recovery_election_allowed(true); + } let outcome = install.failure.take().map_or_else( || { Ok(PartitionInstallOutcome { @@ -4651,6 +4692,7 @@ InstallFilePhase::FinishBackup => InstallPhase::ClearMissing, InstallFilePhase::ClearMissing => { self.materialization_missing = false; + self.refresh_recovered_log_complete(); InstallPhase::Done } _ => unreachable!("transfer file result matches captured phase"), @@ -4878,6 +4920,9 @@ self.required_metadata_frontier = self .required_metadata_frontier .max(install.offsets_wire.required_metadata_frontier); + self.retirement_paid_through = self + .retirement_paid_through + .max(install.offsets_wire.retirement_paid_through); self.consumer_group_owners = install .offsets_wire .owners @@ -5974,6 +6019,7 @@ owners: self.consumer_group_owners.values().copied().collect(), history_transition: self.history_transition, owner_retirement: self.owner_retirement.clone(), + retirement_paid_through: self.retirement_paid_through, ..Default::default() }; Ok(retry_checkpoint_bytes( @@ -6062,6 +6108,9 @@ self.required_metadata_frontier = self .required_metadata_frontier .max(wire.required_metadata_frontier); + self.retirement_paid_through = self + .retirement_paid_through + .max(wire.retirement_paid_through); if let Err(error) = self.reclaim_retry_checkpoints(through_op).await { tracing::warn!(%error, namespace_raw = self.consensus().group(), "cannot reclaim obsolete receipt checkpoints during recovery"); }
diff --git a/core/sdk/Cargo.toml b/core/sdk/Cargo.toml index 0304d73..7932033 100644 --- a/core/sdk/Cargo.toml +++ b/core/sdk/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy" -version = "0.11.1-edge.6" +version = "0.11.1-edge.7" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2024" rust-version.workspace = true
diff --git a/core/sdk/README.md b/core/sdk/README.md index 5eab426..d5e2fd0 100644 --- a/core/sdk/README.md +++ b/core/sdk/README.md
@@ -53,7 +53,7 @@ Mixed versions and rolling upgrades are unsupported. Binary clients share one logical session across replica links using BindSession (15) and discover partition primaries with GetPartitionRoute (105). The minimum binary protocol -version is 0.12.1. Commands 14, 103, 104, 107, 123 and 124 are retired. HTTP clients +version is 0.12.2. Commands 14, 103, 104, 107, 123 and 124 are retired. HTTP clients use server-side forwarding. ```bash
diff --git a/core/sdk/src/http/http_client.rs b/core/sdk/src/http/http_client.rs index e132e79..3f51a7e 100644 --- a/core/sdk/src/http/http_client.rs +++ b/core/sdk/src/http/http_client.rs
@@ -856,8 +856,9 @@ /// Decodes the refusals the client acts on: a producer never repeats `RequestTooOld`, a send /// refreshes its context once on `HistoryUnavailable`, `LifecycleBusy` is retried, -/// non-admission and capacity refusals permit a retry without risking duplicate progress, and a -/// poll refused for its byte limit returns `InvalidSizeBytes`. +/// non-admission and capacity refusals permit a retry without risking duplicate progress, a +/// poll refused for its byte limit returns `InvalidSizeBytes`, and a topic refused for a factor +/// above the node count returns `InvalidReplicationFactor`, as over the binary transports. fn typed_refusal(body: &str) -> Option<IggyError> { let id = error_id(body)?; [ @@ -867,6 +868,7 @@ IggyError::TransientNotAccepted, IggyError::DeferredPollCapacityExceeded, IggyError::InvalidSizeBytes, + IggyError::InvalidReplicationFactor, ] .into_iter() .find(|error| error.as_code() == id) @@ -889,7 +891,7 @@ mod tests { use super::*; use crate::http::scripted_server::{Recorded, Reply, ScriptedServer}; - use crate::prelude::{Identifier, StreamClient}; + use crate::prelude::{Identifier, StreamClient, TopicClient, TopicCreateOptions}; use std::iter; /// The number of `LifecycleBusy` refusals scripted for a request retried until the deadline, @@ -1062,6 +1064,7 @@ IggyError::TransientNotAccepted, IggyError::DeferredPollCapacityExceeded, IggyError::InvalidSizeBytes, + IggyError::InvalidReplicationFactor, ] { let body = format!(r#"{{"id":{},"reason":"unavailable"}}"#, error.as_code()); assert_eq!(typed_refusal(&body), Some(error)); @@ -1071,6 +1074,30 @@ } } + #[tokio::test] + async fn given_factor_above_node_count_when_creating_topic_should_return_typed_error() { + let server = ScriptedServer::start(vec![Reply::error( + 400, + &IggyError::InvalidReplicationFactor, + )]) + .await; + let options = TopicCreateOptions { + replication_factor: Some(2), + ..Default::default() + }; + + let created = server + .client() + .create_topic(&stream_id(), "topic", &options) + .await; + + assert!( + matches!(created, Err(IggyError::InvalidReplicationFactor)), + "{created:?}" + ); + assert_eq!(server.requests().len(), 1); + } + use iggy_common::{Consumer, PollMessages, PollingStrategy}; use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; use tokio::net::TcpListener;
diff --git a/core/sdk/src/http/scripted_server.rs b/core/sdk/src/http/scripted_server.rs index 8e001aa..1bb0803 100644 --- a/core/sdk/src/http/scripted_server.rs +++ b/core/sdk/src/http/scripted_server.rs
@@ -270,6 +270,8 @@ size: IggyByteSize::default(), messages_count: 0, context, + members: Vec::new(), + in_sync_members: Vec::new(), }) .collect::<Vec<_>>(); serde_json::to_string(&TopicDetails {
diff --git a/core/sdk/src/links/fake_cluster.rs b/core/sdk/src/links/fake_cluster.rs index 1e05e89..2a52ca7 100644 --- a/core/sdk/src/links/fake_cluster.rs +++ b/core/sdk/src/links/fake_cluster.rs
@@ -499,6 +499,8 @@ owner_generation: 0, metadata_op: 11, }, + members: vec![0], + in_sync_members: vec![0], }) .collect(), }
diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml index 1a4982b..c4e5841 100644 --- a/core/server/Cargo.toml +++ b/core/server/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "server" -version = "0.9.1-edge.5" +version = "0.9.1-edge.7" edition = "2024" license = "Apache-2.0" publish = false
diff --git a/core/server/README.md b/core/server/README.md index 1a95423..2ed8d2b 100644 --- a/core/server/README.md +++ b/core/server/README.md
@@ -57,6 +57,21 @@ WAL entries. This boundary also rejects `IGGY-BOUND-GRANTS-4` and `IGGY-BOUND-GRANTS-5` directories from upstream and earlier development builds, unchanged before replay. +Replication factors change what replicas send each other: partition member +lists, partition status on metadata probes and replies, retirement debt on session +finalization and the paid-through mark on session retirement. Peers compare a +replica wire revision at the handshake, so a build from before that change and a +build after it refuse each other even when both report the same version, for +example two consecutive `:edge` images. Stop every node and start them all on the +new build. Their `IGGY-METADATA-CHANGE-LOG-1` data directories are kept: older +metadata snapshots and journal entries read with every partition on every node, +and older builds refuse the newer snapshot format instead of misreading it. +Topics created without `replication_factor` still go on every node, so clients +that never send the key, which includes every SDK, connector, benchmark and MCP +release before the option, keep the old layout. Set `[cluster] +default_replication_factor` to a number to give those topics fewer copies. +Topic responses carry each partition's member list and in-sync members, a client +wire change that SDKs from before this release cannot decode. No in-place upgrade, storage conversion tool or retained-data migration is provided. Use an offline cutover with logical re-ingestion: @@ -322,11 +337,122 @@ Set the option through `TopicCreateOptions::partition_resize_policy`, HTTP's `options` map, or the CLI's `--set partition_resize_policy=fixed`. +### Replication factor + +A topic's create-only `replication_factor` is the number of nodes that store +each of its partitions. It applies to every partition of the topic, including +ones added later, and takes 1 to 32 but no more than the cluster's node count. +Any other number is refused with `InvalidReplicationFactor` (2018), and a value +that is not a number with `InvalidOptionValue` (4042). Omitted, it is the +server's `[cluster] default_replication_factor`: `"all"` by default, which stores +each partition on every node, or a number from 1 to 32, lowered to the node count +when larger, so `"3"` keeps three copies on clusters of any size, like Kafka's +`default.replication.factor`. The metadata primary resolves it when it admits the +create, so set the same value on every node. `DescribeOptions` reports the value +that node resolves to, and a topic created without the key stores it as derived. +Metadata is always on every node. + +`GetTopic` lists, for each partition, its `members`, the node ids that store it, +and its `in_sync_members`, the members its primary last reported in sync, which +the metadata primary narrows to the nodes answering its probes. A member missing +from the second list is down or behind. Before restarting a node, check that +every partition it is a member of keeps, without it, enough in-sync members to +elect a primary: 2 of 2 or 3, 3 of 4 or 5, 4 of 6, and 5 of 7. A restart of a +node that leads a partition needs that many, not just a write quorum. +The in-sync list is empty until a partition's primary first reports, about a +second after it is created, and from shortly after its primary stops leading it +until a member leads it again. A primary reports only while it leads, and the +metadata primary drops a report it has not heard again for 6 seconds, 2 more for +every 512 partitions past the first 512 in the cluster. + +At create the metadata primary picks each partition's nodes: the primary is the +ready node leading the fewest partitions, and the other members are the nodes +storing the fewest, each in a rack the partition does not use yet while one has +a ready node, like Kafka's `broker.rack`. A node names its rack, or availability +zone, with `rack` in its `[[cluster.nodes]]` entry. Set it on every node or on +none: a node starts with a configuration that sets it on only some. Each rack +then holds about an equal share of the copies, so a rack with fewer nodes puts +more on each of them, and with a factor equal to the number of racks a rack of +one node stores every partition. A node that is down is picked only when too few +are up. The assignment never moves. While a member is down its partitions run +with fewer copies, and the member catches up from the others when it comes back. +A new metadata primary refuses creates with `TransientNotAccepted` (58) until +every node has answered its probe or missed it, so the node whose failure +started the view is not picked. + +The metadata primary reports in `/metrics`, on its first shard, the cluster's +`partitions_under_replicated`, partitions with a member out of sync, and +`partitions_below_write_quorum`, partitions that cannot commit. For a partition +whose primary answers its probes, it counts the members of the primary's last +report that answer too: a member is out of the report while the primary cannot +reach it or while it has trailed the commit point for 30 seconds, Kafka's +`replica.lag.time.max.ms` default. A dropped report, or none that long after the +partition is created, counts no member, as when every member is up but none can +lead, such as both members of a factor-2 partition back on empty storage. For a +partition whose primary does not answer, it counts the members that answer, +which must be enough to elect a primary before anything commits. Every other +node reports 0, so, like Kafka's `UnderReplicatedPartitions`, summing the series +over every node gives the cluster's total. + +A write commits on a quorum of a partition's members: 2 of 3, 3 of 5, and at +most 3 for wider groups. A factor of 2 keeps two copies but stops accepting +writes while either member is down, like Kafka with `min.insync.replicas=2`, +because the two members have no third party to decide between them. A factor +of 1 keeps one copy, and the default `replicated` durability acknowledges a +write before that copy reaches disk; set `durability=persisted` on such a topic +to acknowledge only written data. + +Deleting a topic or stream waits while one of its partitions lacks a quorum of +its members, and completes once they return, like Kafka's pending deletes, or +once enough of them came back without their copy that the rest cannot elect a +primary: 1 of 2, 2 of 3 or 4, and 3 of 5. A pending consumer group deletion on +the topic does not hold the delete back: the delete takes it over and removes +the group with the topic. + +Logins keep working while a partition lacks a quorum of its members. A session +that ends then cannot be retired on that partition, so the partition owes the +retirement instead of holding the session's slot in the client table. The +partition's next primary pays it before serving sessions again, and refuses +their requests and polls that commit their offset on read with +`TransientNotAccepted` (58) until it has. `/metrics` +reports `partitions_owing_retirement`, the partitions still refusing sessions +for that reason, and `retirement_debts_paid`, the payments a node made. + +A node that comes back with empty storage refuses to serve a factor-1 +partition created before it, because that data was only on this node and +serving it again would reuse its offsets. A member of a wider partition copies +the data from a member that kept it instead. Until then it never leads, and it +votes in an election only to let every member report, which elects a member that +kept the data once fewer than a write quorum of members lost theirs. So a +2-member partition elects its other member when one member comes back empty, as +replacing one disk at a time with `--fresh` does, and a 3-member partition with +one member down and another back on empty storage waits for the one that is +down. With `durability` and `consumer_offset_durability` both `persisted`, if a +write quorum of members lost their storage, none of them serves the partition +rather than restart it at offset 0 or without ops that only they held, like Kafka +with `unclean.leader.election.enable` off. Delete and recreate the topic to +accept the loss: the delete completes once the members that lost their copy +leave too few to elect a primary, and frees the name. The default `replicated` +durability acknowledges a write before it reaches the disk, so a restart of +every member can lose acknowledged writes, and members that lost their storage +are counted the same way: once every member reports, the partition elects on the +logs that are left, like Kafka with `unclean.leader.election.enable` on. + +A node that does not store a partition answers its route lookups from what the +partition's primary last reported, or from the partition's initial primary +before any report, and it refuses sends and polls for it with `NotPrimary`, so +clients go to the primary. The metadata primary relays a report that names a +new primary or changes the in-sync members in its next probe, about a second +after it hears it, and cycles through the others 512 per probe. Its topic and +stream counters follow those reports too, so they trail the members. + +Topics created before replication factors keep every partition on every node. + ## Resource options Streams, topics and users take their settings from one option catalog. The binary `DescribeOptions` command (13), `GET /options/{scope}` over HTTP (`stream`, `topic` or `user`) and `iggy options <scope>` list every key a client can see with its kind, default, description and five flags: updatable, resettable, secret, read-only and critical. -Most keys are fixed at create. A topic's `compression_algorithm`, `segment_size`, `durability`, `consumer_offset_durability`, `partition_resize_policy` and save thresholds, and a stream's `encryption_key`, are refused by an update with `OptionNotUpdatable` (4103). `compression_algorithm` is create-only because the server never applies it and stored batches carry no codec marker. +Most keys are fixed at create. A topic's `compression_algorithm`, `segment_size`, `durability`, `consumer_offset_durability`, `partition_resize_policy`, `replication_factor` and save thresholds, and a stream's `encryption_key`, are refused by an update with `OptionNotUpdatable` (4103). `compression_algorithm` is create-only because the server never applies it and stored batches carry no codec marker. The updatable keys are `name` for streams and topics, `username` and `status` for users, and a topic's `message_expiry` and `max_topic_size`. Only those two topic keys can be reset to their default, any other reset is refused with `OptionNotResettable` (4104). Read-only keys, such as the stream key projections below, are refused with `ReadOnlyOption` (4105). @@ -390,7 +516,8 @@ This release changes protocol and storage formats. Servers and compatible clients deploy together; mixed versions and rolling upgrades are unsupported. -Peers verify protocol, release and storage-format identity before admission. +Peers verify protocol, replica wire, release and storage-format identity before +admission. Executable packaging does not affect that identity, so stripping or rebuilding the same compatible release does not by itself prevent a replica from joining. @@ -398,7 +525,7 @@ REST has no version handshake, so a 0.11 HTTP client is not refused at login. A create body that sends an option beside `options`, such as a top-level `message_expiry`, and an update body with a nested member, such as the `{"name": ..., "options": {}}` stream update older SDKs send, are refused with `InvalidCommand` (3), and the error's `field` names the member. -Metadata snapshot format 13 is both the current and the oldest readable format. A data directory holding an older snapshot is refused, and journals written by older builds hold operations this build no longer knows. Start this release from fresh storage on every replica. +Metadata snapshot format 15 is the current format and 14 the oldest readable one. A data directory holding an older snapshot is refused, and journals written by older builds hold operations this build no longer knows. Start this release from fresh storage on every replica. The protocol number is independent of server and SDK package versions. Its current and minimum values are maintained manually for stable releases. The
diff --git a/core/server/config.toml b/core/server/config.toml index b9627fd..d199b52 100644 --- a/core/server/config.toml +++ b/core/server/config.toml
@@ -591,6 +591,15 @@ # a transient disk hiccup cannot kill the process. superblock_wedged_fatal_timeout = "2m" +# Replication factor of a topic created without one ("all" or an integer). +# "all" stores each of its partitions on every node, the layout from before +# per-topic replication factors, so clients that never send the option keep it. +# An integer from 1 to 32 stores each partition on that many nodes, lowered to +# the node count when larger, so "3" keeps three copies on clusters of any size. +# Read by the metadata primary when it admits the create, which stores the +# resolved factor with the topic: set the same value on every node. +default_replication_factor = "all" + # Superblock compatibility: version 2 atomically records adopted view/head/header # identity separately from WAL body certification. Legacy version 1 records with # 66/74-byte payloads remain readable. After a version 2 write, an older binary @@ -734,6 +743,7 @@ # ip = "10.0.1.5" # replica plane, literal IP only # advertised_address = "203.0.113.10" # catch-all for unmatched clients # replica_id = 0 +# rack = "eu-west-1a" # copies of a partition go to different racks # ports = { tcp = 8090, http = 3000, tcp_replica = 9090 } # # [[cluster.nodes.advertised_addresses]]
diff --git a/core/server/src/boot/mod.rs b/core/server/src/boot/mod.rs index 238260c..988522e 100644 --- a/core/server/src/boot/mod.rs +++ b/core/server/src/boot/mod.rs
@@ -67,6 +67,7 @@ use message_bus::transports::tls::install_default_crypto_provider; use message_bus::{IggyMessageBus, ReplicaOwnerTable}; use metadata::change_log::{ChangeLogCapacities, MetadataChangeLog}; +use metadata::impls::metadata::StreamsFrontend; use metadata::impls::recovery::recover; use metadata::{AppliedFrontier, ReplicaIdentity}; use server_common::bootstrap::create_directories; @@ -604,6 +605,33 @@ ) .await .map_err(ServerError::MetadataRecovery)?; + let fresh_storage = recovered.last_journaled_op.unwrap_or_else(|| { + recovered + .snapshot + .as_ref() + .map_or(0, metadata::stm::snapshot::Snapshot::sequence_number) + }) == 0 + && recovered.recovered_state.is_none(); + if let Some((namespace, node)) = recovered + .mux_stm + .streams() + .member_outside_cluster(topology.replica_count) + { + return Err(ServerError::PartitionMemberOutsideCluster { + namespace_raw: namespace.inner(), + node, + nodes: topology.replica_count, + }); + } + // Peers may already hold partitions this node stored before + // it lost its storage; see `load_partition_or_fence`. + if fresh_storage && topology.replica_count > 1 { + crate::partition_helpers::record_storage_epoch( + &config.get_system_path(), + iggy_common::IggyTimestamp::now().as_micros(), + ) + .await?; + } ( Rc::new(recovered.mux_stm), Rc::new(recovered.change_log), @@ -715,6 +743,14 @@ // session (and trip its empty-table assert). metadata.set_clients_table_max(config.metadata.clients_table_max); metadata.set_partitions_max(config.metadata.partitions_max); + metadata.set_default_replication_factor(config.cluster.default_replication_factor.factor()); + let mut racks = vec![None; config.cluster.nodes.len()]; + for node in &config.cluster.nodes { + if let Some(rack) = racks.get_mut(usize::from(node.replica_id)) { + rack.clone_from(&node.rack); + } + } + metadata.set_node_racks(&racks); metadata.set_max_tokens_per_user(config.personal_access_token.max_tokens_per_user); // Reinstall the sessions recovery restored from the checkpoint and the WAL // suffix, so a rebooted node dedups retries and admits continuations from
diff --git a/core/server/src/boot/recovery.rs b/core/server/src/boot/recovery.rs index 5044794..2ee2026 100644 --- a/core/server/src/boot/recovery.rs +++ b/core/server/src/boot/recovery.rs
@@ -29,8 +29,8 @@ }; use configs::server::ServerConfig; use consensus::{ - ClientTable, HistoryRecovery, JoinMode, LocalPipeline, PipelineEntry, Sequencer, VsrConsensus, - VsrRestore, VsrState, + ClientTable, HistoryRecovery, JoinMode, LocalPipeline, PipelineEntry, ReplicaSet, Sequencer, + VsrConsensus, VsrRestore, VsrState, }; use iggy_common::{ EncryptorKind, IggyByteSize, IggyError, PartitionStats, ServerKeyContext, StreamEncryption, @@ -208,6 +208,13 @@ let mut encryption = None; for (topic_id, topic) in &stream.topics { for partition in &topic.partitions { + if partition + .replica_set(topology.replica_count) + .member(topology.self_replica_id) + .is_none() + { + continue; + } let namespace = IggyNamespace::new(stream.id, topic_id, partition.id); let owning_shard = calculate_shard_assignment(&namespace, u32::from(total_shards)); @@ -285,7 +292,7 @@ topic_runtime, topology.cluster_id, topology.self_replica_id, - topology.replica_count, + partition_metadata.replica_set(topology.replica_count), Rc::clone(&bus), &partitions, ) @@ -782,7 +789,7 @@ let consensus = VsrConsensus::restored( topology.cluster_id, topology.self_replica_id, - replica_count, + ReplicaSet::all(replica_count), server_common::sharding::METADATA_GROUP, bus, // Request queue keeps the stock 2x ratio over the prepare queue @@ -981,6 +988,7 @@ partition_id: 0, consensus_group_id: 1, initial_view: 0, + members: Vec::new(), }], } .to_bytes(),
diff --git a/core/server/src/boot/topology.rs b/core/server/src/boot/topology.rs index 5a98a2e..4b617ff 100644 --- a/core/server/src/boot/topology.rs +++ b/core/server/src/boot/topology.rs
@@ -462,6 +462,7 @@ http: Option<u16>, ) -> configs::cluster::ClusterNodeConfig { configs::cluster::ClusterNodeConfig { + rack: None, name: "node".to_owned(), ip: ip.to_owned(), advertised_address: None,
diff --git a/core/server/src/cluster_meta.rs b/core/server/src/cluster_meta.rs index a4ae7b3..78fe236 100644 --- a/core/server/src/cluster_meta.rs +++ b/core/server/src/cluster_meta.rs
@@ -269,6 +269,7 @@ fn node_config(advertised_address: Option<String>) -> ClusterNodeConfig { ClusterNodeConfig { + rack: None, name: "node-0".to_owned(), ip: "10.0.0.1".to_owned(), advertised_address,
diff --git a/core/server/src/consumer_group.rs b/core/server/src/consumer_group.rs index df0d05c..ce11771 100644 --- a/core/server/src/consumer_group.rs +++ b/core/server/src/consumer_group.rs
@@ -305,6 +305,7 @@ initial_view: 0, partition_id, consensus_group_id: u64::from(partition_id) + 1, + members: Vec::new(), }) .collect(), }
diff --git a/core/server/src/consumer_group/lease.rs b/core/server/src/consumer_group/lease.rs index 113b385..91c00d9 100644 --- a/core/server/src/consumer_group/lease.rs +++ b/core/server/src/consumer_group/lease.rs
@@ -24,8 +24,12 @@ use std::rc::{Rc, Weak}; use std::time::{Duration, Instant}; -use iggy_binary_protocol::requests::system::SessionIdentity; -use iggy_binary_protocol::{Command, ConsumerSession, ConsumerSessionHeartbeatHeader, WireDecode}; +use bytes::{BufMut, BytesMut}; +use iggy_binary_protocol::requests::system::{RetirementDebt, SessionIdentity}; +use iggy_binary_protocol::{ + Command, ConsumerSession, ConsumerSessionHeartbeatHeader, MAX_CONSUMER_SESSIONS_PER_HEARTBEAT, + WireDecode, WireEncode, +}; use server_common::Message; use server_common::sharding::IggyNamespace; use tracing::{debug, warn}; @@ -54,14 +58,29 @@ } } +/// What a reporter could not prove retired among the partitions it stores. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub(super) struct Withheld { + /// Member sets none of whose partitions it could prove. + pub(super) masks: Vec<u32>, + pub(super) partitions: Vec<RetirementDebt>, +} + +/// A reporter's withheld vouches. `None` vouches for none of its partitions. +pub(super) type Unprovable = Option<Rc<Withheld>>; + pub(super) struct RetirementProgress { pub(super) identity: SessionIdentity, pub(super) revision: u64, pub(super) cursor: usize, - pub(super) complete: bool, + /// Swept namespaces not yet proven: created revision, first attempt, and + /// whether a retry followed it. + pub(super) pending: BTreeMap<IggyNamespace, (u64, Instant, bool)>, + pub(super) unprovable: BTreeSet<RetirementDebt>, pub(super) reported: bool, last_report: Option<Instant>, - pub(super) reporters: BTreeSet<u8>, + pub(super) reporters: BTreeMap<u8, Unprovable>, + pub(super) debt_settleable_since: Option<Instant>, } impl RetirementProgress { @@ -79,6 +98,32 @@ type RetirementNamespaces = Rc<[(IggyNamespace, u64)]>; +/// Committed partitions grouped by member set, for counting vouching reporters +/// once per set instead of once per partition. +#[derive(Default)] +pub(super) struct RetirementGroups { + pub(super) sets: Vec<MemberSet>, + pub(super) by_partition: BTreeMap<RetirementDebt, usize>, +} + +pub(super) struct MemberSet { + pub(super) mask: u32, + pub(super) quorum: u32, + pub(super) partitions: Vec<RetirementDebt>, +} + +/// Unpaid retirement debts of partitions this node stores, and the newest +/// debt op absorbed into them. +#[derive(Default)] +pub(super) struct RetirementDebts { + pub(super) seen_through: Option<u64>, + /// `(created_revision, debt_op)` by namespace. + pub(super) unpaid: BTreeMap<IggyNamespace, (u64, u64)>, + /// Debts confirmed paid, so a rescan does not visit them again. + pub(super) paid: BTreeMap<IggyNamespace, (u64, u64)>, + pub(super) cursor: Option<IggyNamespace>, +} + /// Bounded by the session registry, independent of heartbeat volume. #[derive(Default)] pub struct ConsumerGroupLiveness { @@ -92,6 +137,17 @@ pub(super) retirement: Option<RetirementProgress>, pub(super) retirement_namespaces: Option<(u64, RetirementNamespaces)>, pub(super) retirement_fences: BTreeMap<IggyNamespace, u64>, + /// Namespaces the last completed sweep could not prove: the next sweep + /// gives each of them one attempt instead of the full wait. + pub(super) retirement_suspects: BTreeSet<IggyNamespace>, + /// When each suspect last got a retirement barrier. Without one a suspect + /// that regained its quorum never proves a session again. + pub(super) retirement_suspect_barriers: BTreeMap<IggyNamespace, Instant>, + pub(super) retirement_groups: Option<(u64, Rc<RetirementGroups>)>, + /// Nodes missing from the reports of the last finalization, which the + /// next one does not wait for before recording debt. + pub(super) retirement_absent: u32, + pub(super) retirement_debts: RetirementDebts, } impl ConsumerGroupLiveness { @@ -171,20 +227,24 @@ identity, revision, cursor: 0, - complete: true, + pending: BTreeMap::new(), + unprovable: BTreeSet::new(), reported: false, last_report: None, - reporters: BTreeSet::new(), + reporters: BTreeMap::new(), + debt_settleable_since: None, }); } else if let Some(progress) = &mut self.retirement && progress.revision != revision { progress.revision = revision; progress.cursor = 0; - progress.complete = true; + progress.pending.clear(); + progress.unprovable.clear(); progress.reported = false; progress.last_report = None; progress.reporters.clear(); + progress.debt_settleable_since = None; } } @@ -218,6 +278,26 @@ if primary_view != Some(header.view) || header.cluster != cluster { return; } + if header.command == Command::SessionRetirementProgress { + if header.incomplete != 0 { + return; + } + let Some((session, unprovable)) = decode_retirement_progress(message.body()) else { + debug!( + replica = header.replica, + "invalid session retirement progress body" + ); + return; + }; + if let Some(progress) = &mut self.retirement + && header.namespace_revision == progress.revision + && session.client_id == progress.identity.client_id + && session.session == progress.identity.session + { + progress.reporters.insert(header.replica, unprovable); + } + return; + } // Validate the whole batch before refreshing anything. let (sessions, remainder) = message .body() @@ -239,24 +319,6 @@ ); return; } - if header.command == Command::SessionRetirementProgress { - if header.incomplete != 0 { - return; - } - if let Some(progress) = &mut self.retirement - && header.namespace_revision == progress.revision - { - for bytes in sessions { - if let Ok((session, _)) = ConsumerSession::decode(bytes) - && session.client_id == progress.identity.client_id - && session.session == progress.identity.session - { - progress.reporters.insert(header.replica); - } - } - } - return; - } if header.incomplete != 0 { self.defer_expiry(now, timeout, header.replica); } @@ -286,6 +348,76 @@ } } +const WITHHELD_PARTITION: u64 = 1; +const WITHHELD_MEMBER_SET: u64 = 2; +const WITHHELD_ALL: u64 = 3; + +/// A retirement progress body: the session, then one 24-byte record per +/// withheld vouch, `namespace u64 | created_revision u64 | tag u64`, or +/// `mask u64 | 0 | tag` for a member set. The records keep the body whole +/// heartbeat sessions; a list that does not fit is sent as one record +/// withholding every vouch. +pub(super) fn encode_retirement_progress( + session: ConsumerSession, + withheld: Option<&Withheld>, + body_max: usize, +) -> BytesMut { + let records = withheld.map_or(1, |withheld| { + withheld.masks.len() + withheld.partitions.len() + }); + let fits = withheld.is_some() + && records < MAX_CONSUMER_SESSIONS_PER_HEARTBEAT + && (records + 1) * ConsumerSession::ENCODED_SIZE <= body_max; + let mut body = BytesMut::with_capacity((records + 1) * ConsumerSession::ENCODED_SIZE); + session.encode(&mut body); + let mut record = |first: u64, second: u64, tag: u64| { + body.put_u64_le(first); + body.put_u64_le(second); + body.put_u64_le(tag); + }; + match withheld { + Some(withheld) if fits => { + for &mask in &withheld.masks { + record(u64::from(mask), 0, WITHHELD_MEMBER_SET); + } + for debt in &withheld.partitions { + record(debt.namespace, debt.created_revision, WITHHELD_PARTITION); + } + } + _ => record(0, 0, WITHHELD_ALL), + } + body +} + +fn decode_retirement_progress(body: &[u8]) -> Option<(ConsumerSession, Unprovable)> { + let (records, remainder) = body.as_chunks::<{ ConsumerSession::ENCODED_SIZE }>(); + let (session, records) = records.split_first()?; + if !remainder.is_empty() { + return None; + } + let (session, _) = ConsumerSession::decode(session).ok()?; + let mut withheld = Withheld::default(); + for record in records { + let (first, rest) = record.split_first_chunk::<8>()?; + let (second, tag) = rest.split_first_chunk::<8>()?; + let (first, second) = (u64::from_le_bytes(*first), u64::from_le_bytes(*second)); + match u64::from_le_bytes(tag.try_into().ok()?) { + WITHHELD_PARTITION => withheld.partitions.push(RetirementDebt { + namespace: first, + created_revision: second, + }), + WITHHELD_MEMBER_SET if second == 0 => { + withheld.masks.push(u32::try_from(first).ok()?); + } + WITHHELD_ALL if records.len() == 1 && first == 0 && second == 0 => { + return Some((session, None)); + } + _ => return None, + } + } + Some((session, Some(Rc::new(withheld)))) +} + #[cfg(test)] mod tests { use super::{ConsumerGroupLiveness, SessionActivity};
diff --git a/core/server/src/consumer_group/liveness.rs b/core/server/src/consumer_group/liveness.rs index d8e9ea0..ee965b2 100644 --- a/core/server/src/consumer_group/liveness.rs +++ b/core/server/src/consumer_group/liveness.rs
@@ -19,38 +19,60 @@ //! primary, which renews leases and commits a Logout for each expired member. use std::cell::RefCell; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; +use std::ops::Bound; use std::pin::pin; use std::rc::Rc; use std::time::{Duration, Instant}; use bytes::BytesMut; -use consensus::{Consensus, MetadataHandle}; +use consensus::{Consensus, MetadataHandle, VsrConsensus}; +use futures::StreamExt; use futures::future::{Either, select}; use iggy_binary_protocol::requests::system::{ - MAX_SESSIONS_PER_RETIREMENT, RetireSessionsRequest, SessionIdentity, + FinalizeSessionRequest, MAX_SESSIONS_PER_RETIREMENT, RetireSessionsRequest, RetirementDebt, + SessionIdentity, }; use iggy_binary_protocol::{ - Command, ConsensusHeader, ConsumerSession, ConsumerSessionHeartbeatHeader, HEADER_SIZE, - MAX_CONSUMER_SESSIONS_PER_HEARTBEAT, PrepareHeader, WireEncode, + Command, ConsensusHeader, ConsumerSession, ConsumerSessionHeartbeatHeader, GenericHeader, + HEADER_SIZE, MAX_CONSUMER_SESSIONS_PER_HEARTBEAT, Operation, PrepareHeader, ReplyHeader, + RoutedRequestHeader, WireEncode, }; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; use message_bus::SendError; use message_bus::lifecycle::ShutdownToken; use metadata::impls::metadata::StreamsFrontend; +use metadata::stm::stream::Streams; +use partitions::RetirementDebtStatus; use server_common::Message; -use server_common::sharding::CONSUMER_SESSION_REPORT_TIMEOUT; +use server_common::sharding::{CONSUMER_SESSION_REPORT_TIMEOUT, IggyNamespace}; use shard::Receiver; -use tracing::{info, trace}; +use tracing::{debug, info, trace, warn}; -use crate::consumer_group::lease::ConsumerGroupLiveness; +use crate::consumer_group::lease::{ + ConsumerGroupLiveness, MemberSet, RetirementGroups, RetirementProgress, Unprovable, Withheld, + encode_retirement_progress, +}; use crate::shell::{ShellBus, ShellShard}; const MAX_LOGOUTS_PER_PASS: usize = 256; const MAX_RETIREMENTS_PER_PASS: usize = 128; const HEARTBEAT_RETRY_INTERVAL: Duration = Duration::from_millis(1); const RETIREMENT_SWEEP_INTERVAL: Duration = Duration::from_millis(100); +/// Retirement reads and barriers one sweep keeps in flight. One shard's inbox +/// serves every other request too. +const RETIREMENT_ATTEMPTS_IN_FLIGHT: usize = 8; +/// How long a stored partition may stay unproven, after its retry, before +/// this node reports it unprovable. +const UNPROVABLE_AFTER: Duration = Duration::from_secs(2); +/// How often a suspect namespace gets a retirement barrier it does not wait +/// for. Only a barrier moves a partition's metadata frontier past sessions +/// that ended after its last debt payment. +const SUSPECT_BARRIER_INTERVAL: Duration = Duration::from_secs(1); +/// How long the metadata primary waits for an expected reporter before it +/// finalizes with retirement debt. +const RETIREMENT_DEBT_GRACE: Duration = Duration::from_secs(2); #[derive(Debug, PartialEq, Eq)] enum Pass { @@ -83,7 +105,8 @@ .borrow() .ended_sessions() .next() - .is_some(); + .is_some() + || !liveness.borrow().retirement_debts.unpaid.is_empty(); let report_wait = next_report.saturating_duration_since(shard.bus.monotonic_now()); let wait = if pending_retirement { report_wait.min(RETIREMENT_SWEEP_INTERVAL) @@ -119,9 +142,246 @@ } else { retire_sessions(&shard, &liveness, &stop, &shutdown, interval, &system_path).await; } + pay_retirement_debts(&shard, &liveness, &stop, &shutdown).await; } } +/// Pay, for the partitions this node leads, the retirement debts committed +/// finalizations recorded. Reading a debt also arms the session gate of every +/// local replica, so a backup that becomes primary already refuses sessions. +async fn pay_retirement_debts<B, MJ, S, SB>( + shard: &Rc<ShellShard<B, MJ, S, SB>>, + liveness: &RefCell<ConsumerGroupLiveness>, + stop: &Receiver<()>, + shutdown: &ShutdownToken, +) where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let metadata = shard.plane.metadata(); + let Some(consensus) = metadata.consensus.as_ref() else { + return; + }; + absorb_retirement_debts( + metadata.mux_stm.streams(), + liveness, + consensus.replica(), + consensus.replica_count(), + ); + let batch: Vec<(IggyNamespace, u64, u64)> = { + let mut tracker = liveness.borrow_mut(); + let debts = &mut tracker.retirement_debts; + let rotated: Box<dyn Iterator<Item = _>> = match debts.cursor { + Some(after) => Box::new( + debts + .unpaid + .range((Bound::Excluded(after), Bound::Unbounded)) + .chain(debts.unpaid.range(..=after)), + ), + None => Box::new(debts.unpaid.iter()), + }; + let batch: Vec<_> = rotated + .take(MAX_RETIREMENTS_PER_PASS) + .map(|(namespace, (created_revision, debt_op))| { + (*namespace, *created_revision, *debt_op) + }) + .collect(); + debts.cursor = batch.last().map(|(namespace, _, _)| *namespace); + batch + }; + if batch.is_empty() || stop.try_recv().is_ok() || shutdown.is_triggered() { + return; + } + let mut running = futures::stream::iter(batch.iter().map( + |&(namespace, created_revision, debt_op)| async move { + let settled = pay_retirement_debt(shard, namespace, created_revision, debt_op).await; + (namespace, created_revision, debt_op, settled) + }, + )) + .buffer_unordered(RETIREMENT_ATTEMPTS_IN_FLIGHT); + let mut deadline = pin!(shard.bus.sleep(CONSUMER_SESSION_REPORT_TIMEOUT)); + while let Either::Left((Some((namespace, created_revision, debt_op, settled)), _)) = + select(running.next(), deadline.as_mut()).await + { + let Some(settled) = settled else { + continue; + }; + let mut tracker = liveness.borrow_mut(); + let debts = &mut tracker.retirement_debts; + if debts.unpaid.get(&namespace) == Some(&(created_revision, debt_op)) { + debts.unpaid.remove(&namespace); + if settled == DebtSettlement::Paid { + debts.paid.insert(namespace, (created_revision, debt_op)); + } else { + debts.paid.remove(&namespace); + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DebtSettlement { + Paid, + Gone, +} + +/// Merge the debts recorded since the last call into the unpaid set, +/// rescanning committed metadata when the recent ones are no longer kept. +fn absorb_retirement_debts( + streams: &Streams, + liveness: &RefCell<ConsumerGroupLiveness>, + node: u8, + nodes: u8, +) { + let mut tracker = liveness.borrow_mut(); + let debts = &mut tracker.retirement_debts; + let recent = debts + .seen_through + .and_then(|seen_through| streams.retirement_debts_since(seen_through, node, nodes)); + let (entries, newest) = recent.unwrap_or_else(|| { + debts.unpaid.clear(); + streams.retirement_debts_stored_by(node, nodes) + }); + for (namespace, created_revision, debt_op) in entries { + if debts + .paid + .get(&namespace) + .is_some_and(|&(paid_revision, paid_op)| { + paid_revision == created_revision && paid_op >= debt_op + }) + { + continue; + } + let entry = debts + .unpaid + .entry(namespace) + .or_insert((created_revision, debt_op)); + if entry.0 < created_revision || (entry.0 == created_revision && entry.1 < debt_op) { + *entry = (created_revision, debt_op); + } + } + debts.seen_through = Some(newest); +} + +/// `Some(Paid)` once a committed payment covers `debt_op`, `Some(Gone)` when +/// the incarnation no longer exists, `None` while it is still owed. Only the +/// node whose replica is a normal primary with an idle pipeline pays. +async fn pay_retirement_debt<B, MJ, S, SB>( + shard: &Rc<ShellShard<B, MJ, S, SB>>, + namespace: IggyNamespace, + created_revision: u64, + debt_op: u64, +) -> Option<DebtSettlement> +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let read = || { + shard.partition_read( + namespace, + shard::PartitionRead::RetirementDebt { + created_revision, + debt_op, + }, + ) + }; + let metadata = shard.plane.metadata(); + let sessions = match read().await { + Some(shard::PartitionReadReply::RetirementDebt(RetirementDebtStatus::Paid)) => { + return Some(DebtSettlement::Paid); + } + Some(shard::PartitionReadReply::RetirementDebt(RetirementDebtStatus::Unpaid(Some( + sessions, + )))) => sessions, + Some(shard::PartitionReadReply::NotFound) => { + return metadata + .mux_stm + .streams() + .retirement_debt_for(namespace, created_revision) + .is_none() + .then_some(DebtSettlement::Gone); + } + _ => return None, + }; + let consensus = metadata.consensus.as_ref()?; + let applied = consensus.commit_min(); + if applied < debt_op { + return None; + } + // A session absent from the table at `applied` but registered at or + // below it was finalized; newer ones are left alone. + let mut identities: Vec<SessionIdentity> = { + let table = metadata.client_table.borrow(); + sessions + .into_iter() + .filter(|&(client_id, session)| { + session <= applied && table.session_epoch(client_id) != Some(session) + }) + .map(|(client_id, session)| SessionIdentity { + client_id, + session, + metadata_watermark: debt_op.max(session), + }) + .collect() + }; + identities.sort_unstable_by_key(|identity| identity.client_id); + let batch_limit = MAX_SESSIONS_PER_RETIREMENT.min( + shard + .bus_max_message_size() + .saturating_sub(HEADER_SIZE + size_of::<u64>()) + / SessionIdentity::ENCODED_SIZE, + ); + if batch_limit == 0 { + return None; + } + let batches: Vec<&[SessionIdentity]> = if identities.is_empty() { + vec![&[]] + } else { + identities.chunks(batch_limit).collect() + }; + let last = batches.len() - 1; + // Each batch must commit before the next: the last one's `paid_through` + // vouches for every earlier batch. + for (index, identities) in batches.into_iter().enumerate() { + let body = RetireSessionsRequest { + identities: identities.to_vec(), + paid_through: if index == last { debt_op } else { 0 }, + } + .to_bytes(); + let request = + retire_sessions_frame(consensus.cluster(), namespace, debt_op, applied, &body); + let ticket = shard.partition_submit(namespace, request).ok()?; + let reply = shard.await_partition_submit(ticket).await?; + if !reply_committed(&reply) { + return None; + } + } + let paid = matches!( + read().await, + Some(shard::PartitionReadReply::RetirementDebt( + RetirementDebtStatus::Paid + )) + ); + if paid { + shard.metrics().record_retirement_debt_paid(); + } + paid.then_some(DebtSettlement::Paid) +} + +fn reply_committed(reply: &Message<GenericHeader>) -> bool { + reply + .as_slice() + .get(..size_of::<ReplyHeader>()) + .and_then(|bytes| bytemuck::checked::try_from_bytes::<ReplyHeader>(bytes).ok()) + .is_some_and(|header| header.status == 0) +} + async fn report_and_expire<B, MJ, S, SB>( shard: &Rc<ShellShard<B, MJ, S, SB>>, liveness: &RefCell<ConsumerGroupLiveness>, @@ -372,206 +632,34 @@ liveness .borrow_mut() .reconcile_retirement(Some(identity), revision); - let remote_quorum = consensus.is_primary() - && liveness - .borrow() - .retirement - .as_ref() - .is_some_and(|progress| progress.reporters.len() >= consensus.quorum_replication()); - if remote_quorum { - // The metadata primary need not belong to the common reporting quorum. - // Every reporter already proved retirement in every allocated group. - if let Err(error) = metadata - .submit_session_finalization(identity, revision) - .await - { - trace!( - ?error, - client_id = identity.client_id, - "session finalization deferred" - ); - } + // The metadata primary need not belong to the common reporting quorum. + if consensus.is_primary() + && let Some(debt) = settle_retirement(shard, liveness, revision) + { + finalize_session(shard, identity, revision, debt).await; return; } - let (cursor, reported) = { - let tracker = liveness.borrow(); - let Some(progress) = &tracker.retirement else { - return; - }; - (progress.cursor, progress.reported) + let Some(reported) = liveness + .borrow() + .retirement + .as_ref() + .map(|progress| progress.reported) + else { + return; }; - if !reported { - let mut identities: Vec<_> = metadata.client_table.borrow().ended_sessions().collect(); - identities - .sort_unstable_by_key(|identity| (identity.metadata_watermark, identity.client_id)); - let batch_limit = MAX_SESSIONS_PER_RETIREMENT.min( - shard.bus_max_message_size().saturating_sub(HEADER_SIZE) - / SessionIdentity::ENCODED_SIZE, - ); - if batch_limit == 0 { - return; - } - identities.truncate(batch_limit); - let body = RetireSessionsRequest { identities }.to_bytes(); - let namespaces = { - let mut tracker = liveness.borrow_mut(); - if tracker - .retirement_namespaces - .as_ref() - .is_none_or(|(cached_revision, _)| *cached_revision != revision) - { - tracker.retirement_fences.clear(); - tracker.retirement_namespaces = Some(( - revision, - metadata.mux_stm.streams().read(|inner| { - let mut namespaces = Vec::new(); - for (_, stream) in &inner.items { - for (topic_id, topic) in &stream.topics { - for partition in &topic.partitions { - namespaces.push(( - server_common::sharding::IggyNamespace::new( - stream.id, - topic_id, - partition.id, - ), - partition.created_revision, - )); - } - } - } - Rc::from(namespaces) - }), - )); - } - let Some((_, namespaces)) = &tracker.retirement_namespaces else { - return; - }; - Rc::clone(namespaces) - }; - // Repeated barriers are idempotent, so an unfinished sweep can resume - // without delaying activity reports past their existing budget. - let completed = shard::bus_timeout(&shard.bus, CONSUMER_SESSION_REPORT_TIMEOUT, async { - for (namespace, created_revision) in namespaces - .iter() - .skip(cursor) - .take(MAX_RETIREMENTS_PER_PASS) - { - if stop.try_recv().is_ok() || shutdown.is_triggered() { - return false; - } - if consensus.view() != view || !consensus.is_normal() { - return false; - } - let status = shard - .partition_read( - *namespace, - shard::PartitionRead::SessionRetired { - identity, - created_revision: *created_revision, - }, - ) - .await; - let retired = match status { - Some(shard::PartitionReadReply::SessionRetired(true)) => true, - Some(shard::PartitionReadReply::SessionRetirementFailed { - created_revision: failed_revision, - }) if failed_revision == *created_revision => { - if liveness.borrow().retirement_fences.get(namespace) - == Some(created_revision) - { - true - } else { - match crate::partition_helpers::record_partition_retirement_fence( - system_path, - *namespace, - *created_revision, - ) - .await - { - Ok(()) => { - liveness - .borrow_mut() - .retirement_fences - .insert(*namespace, *created_revision); - true - } - Err(error) => { - tracing::warn!(%error, namespace_raw = namespace.inner(), - "retirement waits for durable partition fence"); - false - } - } - } - } - _ => false, - }; - if !retired { - let size = HEADER_SIZE + body.len(); - let mut request = - Message::<iggy_binary_protocol::RoutedRequestHeader>::new(size) - .transmute_header( - |_, header: &mut iggy_binary_protocol::RoutedRequestHeader| { - header.command = Command::Request; - header.operation = - iggy_binary_protocol::Operation::RetireSession; - header.cluster = consensus.cluster(); - header.client = message_bus::AUTO_COMMIT_CLIENT_ID; - header.session = 1; - header.request = identity.metadata_watermark; - header.metadata_watermark = consensus.commit_min(); - header.group = namespace.inner(); - header.size = - u32::try_from(size).expect("retirement frame fits u32"); - }, - ); - request.as_mut_slice()[HEADER_SIZE..].copy_from_slice(&body); - if let Ok(ticket) = shard.partition_submit(*namespace, request) { - let _ = shard.await_partition_submit(ticket).await; - } - } - let retired = retired - || matches!( - shard - .partition_read( - *namespace, - shard::PartitionRead::SessionRetired { - identity, - created_revision: *created_revision, - } - ) - .await, - Some(shard::PartitionReadReply::SessionRetired(true)) - ); - let mut tracker = liveness.borrow_mut(); - let Some(progress) = &mut tracker.retirement else { - return false; - }; - if progress.identity != identity || progress.revision != revision { - return false; - } - progress.complete &= retired; - progress.cursor += 1; - } - let mut tracker = liveness.borrow_mut(); - let Some(progress) = &mut tracker.retirement else { - return false; - }; - if progress.cursor < namespaces.len() { - return false; - } - if !progress.complete { - progress.cursor = 0; - progress.complete = true; - return false; - } - progress.reported = true; - true - }) + if !reported + && !sweep_retirement( + shard, + liveness, + stop, + shutdown, + identity, + revision, + system_path, + ) .await - .unwrap_or(false); - if !completed { - return; - } + { + return; } if consensus.view() != view || !consensus.is_normal() @@ -584,65 +672,691 @@ { return; } + let groups = retirement_groups(shard, liveness, revision); + let withheld = { + let tracker = liveness.borrow(); + let Some(progress) = &tracker.retirement else { + return; + }; + withheld_vouches(&groups, &progress.unprovable) + }; if consensus.is_primary() { - let ready = { + let debt = { let mut tracker = liveness.borrow_mut(); let Some(progress) = &mut tracker.retirement else { return; }; - progress.reporters.insert(consensus.replica()); - progress.reporters.len() >= consensus.quorum_replication() + progress + .reporters + .insert(consensus.replica(), Some(Rc::new(withheld))); + drop(tracker); + settle_retirement(shard, liveness, revision) }; - if ready - && let Err(error) = metadata - .submit_session_finalization(identity, revision) - .await - { - trace!( - ?error, - client_id = identity.client_id, - "session finalization deferred" - ); + if let Some(debt) = debt { + finalize_session(shard, identity, revision, debt).await; } - } else { - if !liveness - .borrow_mut() - .retirement - .as_mut() - .is_some_and(|progress| { - // The primary can receive this report before it starts the sweep. - progress.report_due( - shard.bus.monotonic_now(), - report_interval.min(RETIREMENT_SWEEP_INTERVAL), - ) - }) - { - return; - } - let message = heartbeat_message( - consensus.cluster(), - view, - consensus.replica(), - &[ConsumerSession { - client_id: identity.client_id, - session: identity.session, - }], - false, - ) + return; + } + if !liveness + .borrow_mut() + .retirement + .as_mut() + .is_some_and(|progress| { + // The primary can receive this report before it starts the sweep. + progress.report_due( + shard.bus.monotonic_now(), + report_interval.min(RETIREMENT_SWEEP_INTERVAL), + ) + }) + { + return; + } + let body = encode_retirement_progress( + ConsumerSession { + client_id: identity.client_id, + session: identity.session, + }, + Some(&withheld), + shard.bus_max_message_size().saturating_sub(HEADER_SIZE), + ); + let message = heartbeat_frame(consensus.cluster(), view, consensus.replica(), &body, false) .transmute_header(|old, header| { *header = old; header.command = Command::SessionRetirementProgress; header.namespace_revision = revision; header.seal(); }); - if let Err(error) = shard - .bus - .send_to_replica(consensus.primary_index(view), message.into_frozen()) - .await - { - trace!(?error, "session retirement report deferred"); + if let Err(error) = shard + .bus + .send_to_replica(consensus.primary_index(view), message.into_frozen()) + .await + { + trace!(?error, "session retirement report deferred"); + } +} + +/// One bounded pass of this node's sweep for `identity`. Every stored +/// namespace is tried once, then the unproven ones are retried until each is +/// proven or has stayed unproven for [`UNPROVABLE_AFTER`]. Returns whether the +/// sweep is complete. +#[allow(clippy::too_many_lines, clippy::too_many_arguments)] +async fn sweep_retirement<B, MJ, S, SB>( + shard: &Rc<ShellShard<B, MJ, S, SB>>, + liveness: &RefCell<ConsumerGroupLiveness>, + stop: &Receiver<()>, + shutdown: &ShutdownToken, + identity: SessionIdentity, + revision: u64, + system_path: &str, +) -> bool +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let metadata = shard.plane.metadata(); + let Some(consensus) = metadata.consensus.as_ref() else { + return false; + }; + let view = consensus.view(); + let mut identities: Vec<_> = metadata.client_table.borrow().ended_sessions().collect(); + identities.sort_unstable_by_key(|identity| (identity.metadata_watermark, identity.client_id)); + let batch_limit = MAX_SESSIONS_PER_RETIREMENT.min( + shard.bus_max_message_size().saturating_sub(HEADER_SIZE) / SessionIdentity::ENCODED_SIZE, + ); + if batch_limit == 0 { + return false; + } + identities.truncate(batch_limit); + let body = RetireSessionsRequest { + identities, + paid_through: 0, + } + .to_bytes(); + let namespaces = stored_namespaces(shard, liveness, revision); + let (cursor, attempts) = { + let tracker = liveness.borrow(); + let Some(progress) = &tracker.retirement else { + return false; + }; + let now = shard.bus.monotonic_now(); + let barrier = |namespace: &IggyNamespace| { + if !tracker.retirement_suspects.contains(namespace) { + Barrier::Await + } else if tracker + .retirement_suspect_barriers + .get(namespace) + .is_none_or(|sent| now.saturating_duration_since(*sent) >= SUSPECT_BARRIER_INTERVAL) + { + Barrier::Send + } else { + Barrier::Skip + } + }; + let attempts: Vec<(IggyNamespace, u64, Barrier)> = if progress.cursor < namespaces.len() { + namespaces + .iter() + .skip(progress.cursor) + .take(MAX_RETIREMENTS_PER_PASS) + .map(|&(namespace, created_revision)| { + (namespace, created_revision, barrier(&namespace)) + }) + .collect() + } else { + progress + .pending + .iter() + .take(MAX_RETIREMENTS_PER_PASS) + .map(|(namespace, (created_revision, _, _))| { + (*namespace, *created_revision, barrier(namespace)) + }) + .collect() + }; + (progress.cursor, attempts) + }; + { + let now = shard.bus.monotonic_now(); + let mut tracker = liveness.borrow_mut(); + for (namespace, _, barrier) in &attempts { + if *barrier == Barrier::Send { + tracker.retirement_suspect_barriers.insert(*namespace, now); + } } } + if stop.try_recv().is_ok() + || shutdown.is_triggered() + || consensus.view() != view + || !consensus.is_normal() + { + return false; + } + // A group without a live member quorum stalls only its own attempt: the + // pass deadline settles every attempt that answered in time. + let mut running = futures::stream::iter(attempts.iter().map( + |&(namespace, created_revision, barrier)| { + let body = &body[..]; + async move { + let retired = prove_retired( + shard, + liveness, + identity, + (barrier, body), + namespace, + created_revision, + system_path, + ) + .await; + (namespace, retired) + } + }, + )) + .buffer_unordered(RETIREMENT_ATTEMPTS_IN_FLIGHT); + let mut proven = BTreeSet::new(); + { + let mut deadline = pin!(shard.bus.sleep(CONSUMER_SESSION_REPORT_TIMEOUT)); + while let Either::Left((Some((namespace, retired)), _)) = + select(running.next(), deadline.as_mut()).await + { + if retired { + proven.insert(namespace); + } + } + } + drop(running); + if consensus.view() != view || !consensus.is_normal() { + return false; + } + let now = shard.bus.monotonic_now(); + let mut tracker = liveness.borrow_mut(); + let ConsumerGroupLiveness { + retirement, + retirement_suspects, + retirement_suspect_barriers, + .. + } = &mut *tracker; + let Some(progress) = retirement else { + return false; + }; + if progress.identity != identity || progress.revision != revision { + return false; + } + let first_round = cursor < namespaces.len(); + for (namespace, created_revision, _) in &attempts { + if proven.contains(namespace) { + progress.pending.remove(namespace); + } else if first_round { + progress + .pending + .insert(*namespace, (*created_revision, now, false)); + } else if let Some((_, _, retried)) = progress.pending.get_mut(namespace) { + *retried = true; + } + } + if first_round { + progress.cursor = cursor + attempts.len(); + } + if progress.cursor < namespaces.len() { + return false; + } + let RetirementProgress { + pending, + unprovable, + .. + } = progress; + pending.retain(|namespace, (created_revision, first_attempt, retried)| { + let given_up = *retried + && (retirement_suspects.contains(namespace) + || now.saturating_duration_since(*first_attempt) >= UNPROVABLE_AFTER); + if given_up { + unprovable.insert(RetirementDebt { + namespace: namespace.inner(), + created_revision: *created_revision, + }); + } + !given_up + }); + if !pending.is_empty() { + return false; + } + progress.reported = true; + *retirement_suspects = progress + .unprovable + .iter() + .map(|debt| IggyNamespace::from_raw(debt.namespace)) + .collect(); + retirement_suspect_barriers.retain(|namespace, _| retirement_suspects.contains(namespace)); + if !progress.unprovable.is_empty() { + debug!( + client_id = identity.client_id, + unprovable = progress.unprovable.len(), + "session retirement could not be proven on partitions without a live member quorum" + ); + } + true +} + +/// The namespaces this node stores, cached per namespace revision. Each node +/// vouches only for the copies it stores. +fn stored_namespaces<B, MJ, S, SB>( + shard: &Rc<ShellShard<B, MJ, S, SB>>, + liveness: &RefCell<ConsumerGroupLiveness>, + revision: u64, +) -> Rc<[(IggyNamespace, u64)]> +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let metadata = shard.plane.metadata(); + let mut tracker = liveness.borrow_mut(); + if let Some((cached_revision, namespaces)) = &tracker.retirement_namespaces + && *cached_revision == revision + { + return Rc::clone(namespaces); + } + tracker.retirement_fences.clear(); + tracker.retirement_suspects.clear(); + tracker.retirement_suspect_barriers.clear(); + let (node, nodes) = metadata.consensus.as_ref().map_or((0, 1), |consensus| { + (consensus.replica(), consensus.replica_count()) + }); + let namespaces: Rc<[(IggyNamespace, u64)]> = metadata.mux_stm.streams().read(|inner| { + let mut namespaces = Vec::new(); + for (_, stream) in &inner.items { + for (topic_id, topic) in &stream.topics { + for partition in topic + .partitions + .iter() + .filter(|partition| partition.replica_set(nodes).member(node).is_some()) + { + namespaces.push(( + IggyNamespace::new(stream.id, topic_id, partition.id), + partition.created_revision, + )); + } + } + } + Rc::from(namespaces) + }); + tracker.retirement_namespaces = Some((revision, Rc::clone(&namespaces))); + namespaces +} + +/// Whether this node's copy of `namespace` holds `identity` retired: an +/// applied barrier, or a durable fence over a failed incarnation. A suspect +/// never waits on its barrier, so a group that cannot commit never holds a slot. +async fn prove_retired<B, MJ, S, SB>( + shard: &Rc<ShellShard<B, MJ, S, SB>>, + liveness: &RefCell<ConsumerGroupLiveness>, + identity: SessionIdentity, + (barrier, body): (Barrier, &[u8]), + namespace: IggyNamespace, + created_revision: u64, + system_path: &str, +) -> bool +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let read = || { + shard.partition_read( + namespace, + shard::PartitionRead::SessionRetired { + identity, + created_revision, + }, + ) + }; + match read().await { + Some(shard::PartitionReadReply::SessionRetired(true)) => return true, + Some(shard::PartitionReadReply::SessionRetirementFailed { + created_revision: failed_revision, + }) if failed_revision == created_revision => { + if liveness.borrow().retirement_fences.get(&namespace) == Some(&created_revision) { + return true; + } + return match crate::partition_helpers::record_partition_retirement_fence( + system_path, + namespace, + created_revision, + ) + .await + { + Ok(()) => { + liveness + .borrow_mut() + .retirement_fences + .insert(namespace, created_revision); + true + } + Err(error) => { + tracing::warn!(%error, namespace_raw = namespace.inner(), + "retirement waits for durable partition fence"); + false + } + }; + } + _ => {} + } + let Some(consensus) = shard.plane.metadata().consensus.as_ref() else { + return false; + }; + if barrier == Barrier::Skip { + return false; + } + let request = retire_sessions_frame( + consensus.cluster(), + namespace, + identity.metadata_watermark, + consensus.commit_min(), + body, + ); + let Ok(ticket) = shard.partition_submit(namespace, request) else { + return false; + }; + if barrier == Barrier::Send { + // The retry's read proves it once the barrier commits. + return false; + } + let _ = shard.await_partition_submit(ticket).await; + matches!( + read().await, + Some(shard::PartitionReadReply::SessionRetired(true)) + ) +} + +/// How a sweep attempt treats a namespace the plain read cannot prove. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Barrier { + /// Submit a retirement and wait for it before reading again. + Await, + /// A suspect due for a retirement it does not wait for. + Send, + /// A suspect that got one recently: read only. + Skip, +} + +/// A server-originated `RetireSession` request for `namespace`. +fn retire_sessions_frame( + cluster: u128, + namespace: IggyNamespace, + request: u64, + metadata_watermark: u64, + body: &[u8], +) -> Message<RoutedRequestHeader> { + let size = HEADER_SIZE + body.len(); + let mut message = Message::<RoutedRequestHeader>::new(size).transmute_header( + |_, header: &mut RoutedRequestHeader| { + header.command = Command::Request; + header.operation = Operation::RetireSession; + header.cluster = cluster; + header.client = message_bus::AUTO_COMMIT_CLIENT_ID; + header.session = 1; + header.request = request; + header.metadata_watermark = metadata_watermark; + header.group = namespace.inner(); + header.size = u32::try_from(size).expect("retirement frame fits u32"); + }, + ); + message.as_mut_slice()[HEADER_SIZE..].copy_from_slice(body); + message +} + +/// Settle the tracked retirement from the reports so far: the debt of every +/// partition whose members did not vouch for it, or `None` to keep waiting. +/// A partition goes into debt only once every node expected to report has, or +/// after [`RETIREMENT_DEBT_GRACE`]. +fn settle_retirement<B, MJ, S, SB>( + shard: &Rc<ShellShard<B, MJ, S, SB>>, + liveness: &RefCell<ConsumerGroupLiveness>, + revision: u64, +) -> Option<Vec<RetirementDebt>> +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let metadata = shard.plane.metadata(); + let nodes = metadata.consensus.as_ref()?.replica_count(); + let groups = retirement_groups(shard, liveness, revision); + let now = shard.bus.monotonic_now(); + let mut tracker = liveness.borrow_mut(); + let ConsumerGroupLiveness { + retirement, + retirement_absent, + .. + } = &mut *tracker; + let progress = retirement.as_mut()?; + let debt = retirement_settlement(&groups, &progress.reporters, nodes)?; + let reported = progress + .reporters + .keys() + .fold(0, |mask, &node| mask | node_bit(node)); + let expected = all_nodes(nodes) & !*retirement_absent; + if !debt.is_empty() && reported & expected != expected { + let since = *progress.debt_settleable_since.get_or_insert(now); + if now.saturating_duration_since(since) < RETIREMENT_DEBT_GRACE { + return None; + } + } + *retirement_absent = all_nodes(nodes) & !reported; + Some(debt) +} + +fn retirement_groups<B, MJ, S, SB>( + shard: &Rc<ShellShard<B, MJ, S, SB>>, + liveness: &RefCell<ConsumerGroupLiveness>, + revision: u64, +) -> Rc<RetirementGroups> +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let mut tracker = liveness.borrow_mut(); + if let Some((cached_revision, groups)) = &tracker.retirement_groups + && *cached_revision == revision + { + return Rc::clone(groups); + } + let metadata = shard.plane.metadata(); + let nodes = metadata + .consensus + .as_ref() + .map_or(1, VsrConsensus::replica_count); + let groups = Rc::new(build_retirement_groups(metadata.mux_stm.streams(), nodes)); + tracker.retirement_groups = Some((revision, Rc::clone(&groups))); + groups +} + +fn build_retirement_groups(streams: &Streams, nodes: u8) -> RetirementGroups { + streams.read(|inner| { + let mut groups = RetirementGroups::default(); + let mut by_mask = BTreeMap::new(); + for (stream_id, stream) in &inner.items { + for (topic_id, topic) in &stream.topics { + for partition in &topic.partitions { + let members = partition.replica_set(nodes); + let mask = members + .nodes() + .iter() + .fold(0, |mask, &node| mask | node_bit(node)); + let index = *by_mask.entry(mask).or_insert_with(|| { + groups.sets.push(MemberSet { + mask, + quorum: u32::try_from(consensus::quorum_replication(members.len())) + .unwrap_or(u32::MAX), + partitions: Vec::new(), + }); + groups.sets.len() - 1 + }); + let debt = RetirementDebt { + namespace: IggyNamespace::new(stream_id, topic_id, partition.id).inner(), + created_revision: partition.created_revision, + }; + groups.sets[index].partitions.push(debt); + groups.by_partition.insert(debt, index); + } + } + } + groups + }) +} + +/// `None` without a replication quorum of the cluster. Otherwise every +/// partition whose vouching members fall short of a replication quorum of its +/// member set: those were not proven retired and owe a retirement. +fn retirement_settlement( + groups: &RetirementGroups, + reporters: &BTreeMap<u8, Unprovable>, + nodes: u8, +) -> Option<Vec<RetirementDebt>> { + if reporters.len() < consensus::quorum_replication(nodes) { + return None; + } + let vouching = reporters + .iter() + .filter(|(_, withheld)| withheld.is_some()) + .fold(0, |mask, (&node, _)| mask | node_bit(node)); + let vouchers: Vec<u32> = groups + .sets + .iter() + .map(|set| { + let withheld_sets = reporters + .iter() + .filter(|(node, withheld)| { + set.mask & node_bit(**node) != 0 + && withheld + .as_ref() + .is_some_and(|withheld| withheld.masks.contains(&set.mask)) + }) + .count(); + (vouching & set.mask) + .count_ones() + .saturating_sub(u32::try_from(withheld_sets).unwrap_or(u32::MAX)) + }) + .collect(); + let mut debt = Vec::new(); + for (set, &vouchers) in groups.sets.iter().zip(&vouchers) { + if vouchers < set.quorum { + debt.extend_from_slice(&set.partitions); + } + } + let mut withheld: BTreeMap<RetirementDebt, (usize, u32)> = BTreeMap::new(); + for (&node, entries) in reporters { + for entry in entries + .iter() + .flat_map(|withheld| withheld.partitions.iter()) + { + if let Some(&index) = groups.by_partition.get(entry) + && vouchers[index] >= groups.sets[index].quorum + && groups.sets[index].mask & node_bit(node) != 0 + { + withheld.entry(*entry).or_insert((index, 0)).1 += 1; + } + } + } + for (entry, (index, count)) in withheld { + if vouchers[index].saturating_sub(count) < groups.sets[index].quorum { + debt.push(entry); + } + } + Some(debt) +} + +/// The vouches a node withholds for `unprovable`, as whole member sets where +/// it could prove none of a set's partitions. +fn withheld_vouches(groups: &RetirementGroups, unprovable: &BTreeSet<RetirementDebt>) -> Withheld { + let mut by_set: BTreeMap<usize, Vec<RetirementDebt>> = BTreeMap::new(); + let mut withheld = Withheld::default(); + for entry in unprovable { + match groups.by_partition.get(entry) { + Some(&index) => by_set.entry(index).or_default().push(*entry), + None => withheld.partitions.push(*entry), + } + } + for (index, entries) in by_set { + let set = &groups.sets[index]; + if entries.len() == set.partitions.len() { + withheld.masks.push(set.mask); + } else { + withheld.partitions.extend(entries); + } + } + withheld +} + +const fn node_bit(node: u8) -> u32 { + match 1_u32.checked_shl(node as u32) { + Some(bit) => bit, + None => 0, + } +} + +const fn all_nodes(nodes: u8) -> u32 { + match 1_u32.checked_shl(nodes as u32) { + Some(bit) => bit - 1, + None => u32::MAX, + } +} + +async fn finalize_session<B, MJ, S, SB>( + shard: &Rc<ShellShard<B, MJ, S, SB>>, + identity: SessionIdentity, + revision: u64, + debt: Vec<RetirementDebt>, +) where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let metadata = shard.plane.metadata(); + let encoded = FinalizeSessionRequest { + identity, + namespace_revision: revision, + debt: Vec::new(), + } + .encoded_size() + + size_of::<u32>() + + debt.len() * RetirementDebt::ENCODED_SIZE; + if HEADER_SIZE + encoded > shard.bus_max_message_size() { + warn!( + client_id = identity.client_id, + debt = debt.len(), + "session finalization deferred: its retirement debt exceeds one message" + ); + return; + } + if !debt.is_empty() { + debug!( + client_id = identity.client_id, + debt = debt.len(), + "finalizing session with partition retirement debt" + ); + } + if let Err(error) = metadata + .submit_session_finalization(identity, revision, debt) + .await + { + trace!( + ?error, + client_id = identity.client_id, + "session finalization deferred" + ); + } } async fn expire_sessions<B, MJ, S, SB>( @@ -723,7 +1437,6 @@ } } -#[allow(clippy::cast_possible_truncation)] fn heartbeat_message( cluster: u128, view: u32, @@ -735,6 +1448,17 @@ for session in sessions { session.encode(&mut body); } + heartbeat_frame(cluster, view, replica, &body, incomplete) +} + +#[allow(clippy::cast_possible_truncation)] +fn heartbeat_frame( + cluster: u128, + view: u32, + replica: u8, + body: &[u8], + incomplete: bool, +) -> Message<ConsumerSessionHeartbeatHeader> { let size = HEADER_SIZE + body.len(); let mut message = Message::<ConsumerSessionHeartbeatHeader>::new(size).transmute_header( |_, header: &mut ConsumerSessionHeartbeatHeader| { @@ -744,11 +1468,11 @@ header.replica = replica; header.incomplete = u8::from(incomplete); header.size = size as u32; - header.checksum_body = u128::from(iggy_common::calculate_checksum(&body)); + header.checksum_body = u128::from(iggy_common::calculate_checksum(body)); header.seal(); }, ); - message.as_mut_slice()[HEADER_SIZE..size].copy_from_slice(&body); + message.as_mut_slice()[HEADER_SIZE..size].copy_from_slice(body); message } @@ -932,6 +1656,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 2, + members: Vec::new(), }], }; let group = CreateConsumerGroupRequest { @@ -1208,6 +1933,7 @@ initial_view: 0, partition_id, consensus_group_id: u64::from(partition_id) + 1, + members: Vec::new(), }) .collect(), }; @@ -1509,7 +2235,9 @@ tracker.reconcile_retirement(Some(identity), revision); let progress = tracker.retirement.as_mut().unwrap(); progress.reported = true; - progress.reporters.extend([0, 1]); + progress + .reporters + .extend([(0, Some(Rc::default())), (1, Some(Rc::default()))]); } let (stop, receiver) = shard::channel(1); stop.try_send(()).unwrap(); @@ -1542,6 +2270,142 @@ } #[test] + fn given_a_partition_without_member_reports_when_finalizing_should_owe_debt_not_block() { + let streams = Streams::default(); + let stored = server_common::sharding::IggyNamespace::new(0, 0, 0); + let sibling = server_common::sharding::IggyNamespace::new(0, 0, 1); + let everywhere = server_common::sharding::IggyNamespace::new(1, 0, 0); + for (namespace, group) in [(stored, 1), (sibling, 2)] { + streams.seed_namespace_on( + namespace, + group, + 0, + iggy_common::Durability::Persisted, + &[1, 3, 4], + ); + } + streams.seed_namespace_on( + everywhere, + 3, + 0, + iggy_common::Durability::Persisted, + &[0, 1, 2, 3, 4], + ); + let debt = |namespace: server_common::sharding::IggyNamespace| RetirementDebt { + namespace: namespace.inner(), + created_revision: streams.created_revision_for_namespace(namespace).unwrap(), + }; + let groups = build_retirement_groups(&streams, 5); + let settle = |reporters: &[(u8, Unprovable)]| { + retirement_settlement(&groups, &reporters.iter().cloned().collect(), 5).map( + |mut debt| { + debt.sort_unstable(); + debt + }, + ) + }; + let vouches = |node: u8| (node, Some(Rc::<Withheld>::default())); + let withholds = |node: u8, unprovable: &[RetirementDebt]| { + ( + node, + Some(Rc::new(withheld_vouches( + &groups, + &unprovable.iter().copied().collect(), + ))), + ) + }; + let mut both = vec![debt(stored), debt(sibling)]; + both.sort_unstable(); + assert_eq!( + settle(&[vouches(0), vouches(1), vouches(2)]), + Some(both.clone()), + "a cluster quorum holding one member finalizes and leaves the partitions owing" + ); + assert_eq!( + settle(&[vouches(0), vouches(1), vouches(3)]), + Some(Vec::new()) + ); + assert_eq!( + settle(&[vouches(1), vouches(3)]), + None, + "the cluster-wide quorum still applies" + ); + assert_eq!( + settle(&[ + vouches(0), + withholds(1, &[debt(stored)]), + vouches(3), + (4, None) + ]), + Some(vec![debt(stored)]), + "an unprovable or withheld report is not a vouch" + ); + assert_eq!( + settle(&[ + vouches(0), + withholds(1, &[debt(stored), debt(sibling)]), + vouches(3), + ]), + Some(both), + "a withheld member set counts against every partition in it" + ); + assert_eq!( + settle(&[ + vouches(0), + vouches(1), + vouches(3), + withholds(4, &[debt(stored)]), + ]), + Some(Vec::new()), + "two of three members still vouch" + ); + } + + #[test] + fn given_withheld_vouches_when_reporting_progress_should_round_trip_or_withhold_all() { + let session = ConsumerSession { + client_id: CLIENT, + session: SESSION, + }; + let withheld = Withheld { + masks: vec![0b1010], + partitions: vec![RetirementDebt { + namespace: 5, + created_revision: 6, + }], + }; + let report = |body: &[u8]| { + heartbeat_frame(CLUSTER, VIEW, 1, body, false).transmute_header(|old, header| { + *header = old; + header.command = Command::SessionRetirementProgress; + header.namespace_revision = 1; + header.seal(); + }) + }; + let identity = iggy_binary_protocol::requests::system::SessionIdentity { + client_id: CLIENT, + session: SESSION, + metadata_watermark: 3, + }; + for (sent, body_max, expected) in [ + (Some(&withheld), 1024, Some(Rc::new(withheld.clone()))), + (Some(&Withheld::default()), 1024, Some(Rc::default())), + (Some(&withheld), 2 * ConsumerSession::ENCODED_SIZE, None), + (None, 1024, None), + ] { + let mut tracker = ConsumerGroupLiveness::default(); + tracker.observe_view(Some(VIEW)); + tracker.reconcile_retirement(Some(identity), 1); + let body = encode_retirement_progress(session, sent, body_max); + tracker.receive(CLUSTER, Some(VIEW), &report(&body), Instant::now(), TIMEOUT); + assert_eq!( + tracker.retirement.as_ref().unwrap().reporters.get(&1), + Some(&expected) + ); + } + } + + #[test] fn retirement_quorum_discards_reports_from_an_older_namespace_revision() { let identity = iggy_binary_protocol::requests::system::SessionIdentity { client_id: CLIENT, @@ -1618,7 +2482,9 @@ tracker.reconcile_retirement(Some(identity), revision); let progress = tracker.retirement.as_mut().unwrap(); progress.reported = true; - progress.reporters.insert(consensus.replica()); + progress + .reporters + .insert(consensus.replica(), Some(Rc::default())); } let (_stop, receiver) = shard::channel(1); let (_signal, shutdown) = Shutdown::new(); @@ -1832,7 +2698,7 @@ ); let progress = tracker.retirement.as_mut().unwrap(); progress.cursor = 1; - progress.reporters.insert(1); + progress.reporters.insert(1, Some(Rc::default())); let truncate = TruncatePartitionRequest { stream_id: WireIdentifier::numeric(0), topic_id: WireIdentifier::numeric(0), @@ -1866,7 +2732,7 @@ progress.cursor, 1, "segment deletion must not restart session retirement" ); - assert!(progress.reporters.contains(&1)); + assert!(progress.reporters.contains_key(&1)); } } @@ -1947,6 +2813,192 @@ } } + #[compio::test] + async fn given_a_recorded_debt_when_paying_should_retire_the_finalized_session_and_open_the_gate() + { + const STRANDED: u128 = CLIENT + 7; + const STRANDED_SESSION: u64 = 5; + const DEBT_OP: u64 = 9; + let (_directory, shard) = shard_with_members(1).await; + let metadata = shard.plane.metadata(); + let namespace = IggyNamespace::new(0, 0, 0); + let created_revision = insert_in_memory_partition(&shard, namespace); + let mut messages = server_common::send_messages::IggyMessages::with_capacity(1); + messages.push(server_common::send_messages::IggyMessage { + header: server_common::send_messages::IggyMessageHeader::default(), + payload: bytes::Bytes::from_static(b"stranded"), + user_headers: None, + }); + let batch = + server_common::send_messages::SendMessagesOwned::from_messages(namespace, &messages) + .unwrap(); + let mut body = vec![0; batch.header.total_size()]; + batch.header.encode_into(&mut body); + body[iggy_binary_protocol::batch::BATCH_HEADER_SIZE..].copy_from_slice(&batch.blob); + let send = crate::dispatch::test_support::request_message( + Operation::SendMessages, + STRANDED, + STRANDED_SESSION, + 1, + &body, + ) + .transmute_header(|original, header: &mut RoutedRequestHeader| { + *header = original; + header.cluster = 1; + header.group = namespace.inner(); + header.partition_incarnation = created_revision; + }); + super::super::tests::run_with_partition_message_pump(&shard, async { + let ticket = shard.partition_submit(namespace, send).unwrap(); + let reply = shard.await_partition_submit(ticket).await.unwrap(); + assert!(reply_committed(&reply), "the stranded session writes once"); + }) + .await; + let slice_holds = |shard: &TestShard| { + shard + .plane + .partitions() + .get_by_ns(&namespace) + .unwrap() + .observe_retirement_debt(0) + }; + assert_eq!( + slice_holds(&shard), + RetirementDebtStatus::Paid, + "no debt is owed before one is recorded" + ); + assert_eq!( + shard + .plane + .partitions() + .get_by_ns(&namespace) + .unwrap() + .observe_retirement_debt(1), + RetirementDebtStatus::Unpaid(Some(vec![(STRANDED, STRANDED_SESSION)])) + ); + metadata.mux_stm.streams().record_retirement_debt( + &[RetirementDebt { + namespace: namespace.inner(), + created_revision, + }], + DEBT_OP, + ); + + let tracker = RefCell::new(ConsumerGroupLiveness::default()); + let (_stop, receiver) = shard::channel(1); + let (_signal, shutdown) = Shutdown::new(); + super::super::tests::run_with_partition_message_pump( + &shard, + pay_retirement_debts(&shard, &tracker, &receiver, &shutdown), + ) + .await; + + let partitions = shard.plane.partitions(); + let partition = partitions.get_by_ns(&namespace).unwrap(); + assert!( + !partition.owes_session_retirement(), + "the payment committed and opened the gate" + ); + assert_eq!( + partition.observe_retirement_debt(DEBT_OP), + RetirementDebtStatus::Paid + ); + assert_eq!( + partition.observe_retirement_debt(DEBT_OP + 1), + RetirementDebtStatus::Unpaid(Some(Vec::new())), + "the finalized session left the retry slice" + ); + let tracker = tracker.borrow(); + assert!(tracker.retirement_debts.unpaid.is_empty()); + assert_eq!( + tracker.retirement_debts.paid.get(&namespace), + Some(&(created_revision, DEBT_OP)) + ); + } + + #[compio::test] + async fn given_a_suspect_partition_that_commits_when_sweeping_should_prove_without_debt() { + const PASSES_MAX: usize = 16; + let (directory, shard) = shard_with_members(1).await; + let metadata = shard.plane.metadata(); + let namespace = IggyNamespace::new(0, 0, 0); + let created_revision = insert_in_memory_partition(&shard, namespace); + assert!( + metadata + .client_table + .borrow_mut() + .end_session(CLIENT, 0, SESSION, SESSION) + ); + let revision = metadata + .mux_stm + .streams() + .read(|inner| inner.namespace_revision); + let tracker = RefCell::new(ConsumerGroupLiveness::default()); + // An earlier sweep found the partition without its member quorum. + stored_namespaces(&shard, &tracker, revision); + tracker.borrow_mut().retirement_suspects.insert(namespace); + let (_stop, receiver) = shard::channel(1); + let (_signal, shutdown) = Shutdown::new(); + let system_path = directory.path().to_str().unwrap(); + for _ in 0..PASSES_MAX { + if metadata.client_table.borrow().count() == 0 { + break; + } + super::super::tests::run_with_partition_message_pump( + &shard, + retire_sessions(&shard, &tracker, &receiver, &shutdown, TIMEOUT, system_path), + ) + .await; + } + + assert_eq!(metadata.client_table.borrow().count(), 0); + assert_eq!( + metadata + .mux_stm + .streams() + .retirement_debt_for(namespace, created_revision), + Some(0), + "a suspect that commits again proves the session instead of owing it" + ); + assert!(!tracker.borrow().retirement_suspects.contains(&namespace)); + } + + /// A single-replica in-memory partition at `namespace`, routed to shard 0. + fn insert_in_memory_partition(shard: &TestShard, namespace: IggyNamespace) -> u64 { + let created_revision = shard + .plane + .metadata() + .mux_stm + .streams() + .created_revision_for_namespace(namespace) + .unwrap(); + let consensus = VsrConsensus::new( + 1, + 0, + 1, + namespace.inner(), + shard.bus.clone(), + LocalPipeline::new(), + ); + consensus.init(); + let mut partition = partitions::IggyPartition::with_in_memory_storage( + std::sync::Arc::new(iggy_common::PartitionStats::default()), + consensus, + shard.plane.partitions().config().segment_size, + ); + partition.set_created_revision(created_revision); + shard.plane.partitions().insert(namespace, partition); + shard::shards_table::ShardsTable::insert( + shard.shards_table(), + namespace, + server_common::sharding::PartitionLocation::new( + server_common::sharding::ShardId::new(0), + created_revision, + ), + ); + created_revision + } + #[test] fn bounded_heartbeat_round_trips_through_replica_routing() { let sessions = vec![
diff --git a/core/server/src/dispatch/mod.rs b/core/server/src/dispatch/mod.rs index 3fc9c4e..2fb5018 100644 --- a/core/server/src/dispatch/mod.rs +++ b/core/server/src/dispatch/mod.rs
@@ -2877,6 +2877,7 @@ fn roster_node(name: &str) -> ClusterNodeConfig { ClusterNodeConfig { + rack: None, name: name.to_owned(), ip: "127.0.0.1".to_owned(), advertised_address: None,
diff --git a/core/server/src/dispatch/partition.rs b/core/server/src/dispatch/partition.rs index 2b1171e..889b3ef 100644 --- a/core/server/src/dispatch/partition.rs +++ b/core/server/src/dispatch/partition.rs
@@ -253,6 +253,11 @@ return PartitionAdmission::Denied; } }; + // No routing row is coming for a partition this node does not store. + if !crate::namespace::stores_partition(shard, IggyNamespace::from_raw(namespace)) { + send_deny_reply(shard, origin, &header, IggyError::NotPrimary.as_code()).await; + return PartitionAdmission::Denied; + } // Convergence wait: a CreateTopic commit returns to the client before the // per-shard reconcilers seed routing rows and materialise the partition // (next wake/periodic tick). An op arriving inside that window is not lost @@ -923,6 +928,9 @@ partition_id, ) .map_err(ReadPolledMessagesError::Rejected)?; + if !crate::namespace::stores_partition(shard, namespace) { + return Err(ReadPolledMessagesError::Rejected(IggyError::NotPrimary)); + } let read = PartitionRead::PollOnPrimary { consumer, args, @@ -2209,11 +2217,13 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }, CreatedPartitionAssignment { initial_view: 0, partition_id: 1, consensus_group_id: 2, + members: Vec::new(), }, ], } @@ -2538,6 +2548,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], }; md.mux_stm @@ -4081,6 +4092,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], }; md.mux_stm @@ -4467,6 +4479,7 @@ initial_view: 0, partition_id: 0, consensus_group_id: 1, + members: Vec::new(), }], } .to_bytes(),
diff --git a/core/server/src/dispatch/reads.rs b/core/server/src/dispatch/reads.rs index 5835838..56caf1c 100644 --- a/core/server/src/dispatch/reads.rs +++ b/core/server/src/dispatch/reads.rs
@@ -89,7 +89,7 @@ use metadata::impls::metadata::StreamsFrontend; use metadata::permissioner::Permissioner; use server_common::Message; -use shard::{CommitGate, PartitionRead, PartitionReadReply}; +use shard::{CommitGate, PartitionReadReply}; use std::cell::{Cell, RefCell}; use std::future::Future; use std::net::IpAddr; @@ -971,9 +971,8 @@ .ok_or(IggyError::TransientNotAccepted)?; let partitions_count = u32::try_from(partitions_count).map_err(|_| IggyError::TransientNotAccepted)?; - let Some(PartitionReadReply::Primary { view, primary }) = shard - .partition_read(namespace, PartitionRead::Primary) - .await + let Some(PartitionReadReply::Primary { view, primary }) = + crate::namespace::partition_primary(shard, namespace).await else { return Err(IggyError::TransientNotAccepted); };
diff --git a/core/server/src/dispatch/segment_delete.rs b/core/server/src/dispatch/segment_delete.rs index b8a9dea..8fae552 100644 --- a/core/server/src/dispatch/segment_delete.rs +++ b/core/server/src/dispatch/segment_delete.rs
@@ -83,9 +83,8 @@ reserved: [0; 75], }; let result = shard::bus_timeout(&shard.bus, SEGMENT_DELETE_TIMEOUT, async { - let Some(PartitionReadReply::Primary { primary, .. }) = shard - .partition_read(namespace, PartitionRead::Primary) - .await + let Some(PartitionReadReply::Primary { primary, .. }) = + crate::namespace::partition_primary(shard, namespace).await else { return Err(IggyError::TransientNotAccepted); };
diff --git a/core/server/src/dispatch/test_support.rs b/core/server/src/dispatch/test_support.rs index e0d7bc6..47634dc 100644 --- a/core/server/src/dispatch/test_support.rs +++ b/core/server/src/dispatch/test_support.rs
@@ -565,6 +565,7 @@ initial_view: 0, partition_id, consensus_group_id: FIRST_PARTITION_GROUP + u64::from(partition_id), + members: Vec::new(), }) .collect(), } @@ -674,6 +675,7 @@ partition_id: namespace.partition_id().try_into().unwrap(), consensus_group_id: 2, initial_view: 0, + members: Vec::new(), }], } .to_bytes(),
diff --git a/core/server/src/http/error.rs b/core/server/src/http/error.rs index 48e9f87..aa9191a 100644 --- a/core/server/src/http/error.rs +++ b/core/server/src/http/error.rs
@@ -687,6 +687,7 @@ const READ_PATH: &str = "/streams?consistency=linearizable"; fn node(replica_id: u8, ip: &str, http: Option<u16>) -> ClusterNodeConfig { ClusterNodeConfig { + rack: None, name: format!("node-{replica_id}"), ip: ip.to_owned(), advertised_address: None,
diff --git a/core/server/src/http/forward.rs b/core/server/src/http/forward.rs index 21387d9..4645429 100644 --- a/core/server/src/http/forward.rs +++ b/core/server/src/http/forward.rs
@@ -111,7 +111,7 @@ use send_wrapper::SendWrapper; use serde::Deserialize; use server_common::sharding::IggyNamespace; -use shard::{PartitionRead, PartitionReadReply}; +use shard::PartitionReadReply; use tracing::{debug, warn}; use crate::dispatch::reads::hold_for_frontier; @@ -1525,11 +1525,7 @@ namespace: IggyNamespace, ) -> Option<PartitionPrimary> { state.metrics.partition_primary_probes.inc(); - match state - .shard - .partition_read(namespace, PartitionRead::Primary) - .await? - { + match crate::namespace::partition_primary(&state.shard, namespace).await? { PartitionReadReply::Primary { view, primary } => Some(PartitionPrimary { view, replica: primary, @@ -2316,6 +2312,7 @@ roster.enabled = true; roster.nodes.push( ResolvedClusterNode::try_from(ClusterNodeConfig { + rack: None, name: "primary".to_owned(), ip: address.ip().to_string(), advertised_address: None, @@ -2419,7 +2416,7 @@ "the poll waits for its partition" ); let Ok(ShardFrame::Lifecycle(LifecycleFrame::PartitionRead { - read: PartitionRead::UnboundPollOnPrimary { metadata, .. }, + read: shard::PartitionRead::UnboundPollOnPrimary { metadata, .. }, reply, .. })) = shard_inbox.try_recv()
diff --git a/core/server/src/http/reads.rs b/core/server/src/http/reads.rs index 9762959..b139964 100644 --- a/core/server/src/http/reads.rs +++ b/core/server/src/http/reads.rs
@@ -1410,6 +1410,7 @@ initial_view: 0, partition_id: 0, consensus_group_id, + members: Vec::new(), }], }, );
diff --git a/core/server/src/namespace.rs b/core/server/src/namespace.rs index 9b7a574..08b2fb9 100644 --- a/core/server/src/namespace.rs +++ b/core/server/src/namespace.rs
@@ -43,6 +43,7 @@ use metadata::stm::stream::Streams; use server_common::Message; use server_common::sharding::IggyNamespace; +use shard::{PartitionRead, PartitionReadReply}; use std::rc::Rc; pub fn resolve_offset_group_id( @@ -163,6 +164,66 @@ Ok(namespace.inner()) } +/// Whether this node stores `namespace`: true for one every node stores, and +/// for one this node has not seen committed yet. +pub fn stores_partition<B, MJ, S, SB>( + shard: &Rc<ShellShard<B, MJ, S, SB>>, + namespace: IggyNamespace, +) -> bool +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let node = shard.replica_topology().self_replica_id; + shard + .plane + .metadata() + .mux_stm + .streams() + .members_for_namespace(namespace) + .is_none_or(|members| members.is_none_or(|members| members.member(node).is_some())) +} + +/// The primary of `namespace`'s group. A node storing the partition answers +/// from its own replica; any other node answers with the view the group's +/// primary last reported, or the committed initial view before any report. +#[allow(clippy::future_not_send)] +pub async fn partition_primary<B, MJ, S, SB>( + shard: &Rc<ShellShard<B, MJ, S, SB>>, + namespace: IggyNamespace, +) -> Option<PartitionReadReply> +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal<Entry = Message<PrepareHeader>, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let topology = shard.replica_topology(); + let route = shard + .plane + .metadata() + .mux_stm + .streams() + .partition_route_inputs(namespace); + let stored = route.as_ref().is_none_or(|(members, _, _)| { + members.is_none_or(|members| members.member(topology.self_replica_id).is_some()) + }); + if stored { + return shard + .partition_read(namespace, PartitionRead::Primary) + .await; + } + let (members, created_view, reported) = route?; + // A stale view costs one NotPrimary hop; no view leaves the route empty. + let view = reported.unwrap_or(created_view); + let primary = metadata::stm::stream::primary_of(members, view, topology.replica_count)?; + Some(PartitionReadReply::Primary { view, primary }) +} + pub fn resolve_partition_namespace<B, MJ, S, SB>( shard: &Rc<ShellShard<B, MJ, S, SB>>, stream_id: &WireIdentifier, @@ -390,6 +451,35 @@ ); } + #[compio::test] + async fn given_no_report_from_a_partition_this_node_does_not_store_when_routing_should_name_its_initial_primary() + { + const CREATED_VIEW: u32 = 1; + let bus = SpyBus::default(); + let shard = Rc::new(test_shard(&bus, 0, 3, 1)); + let namespace = IggyNamespace::new(0, 0, 0); + shard.plane.metadata().mux_stm.streams().seed_namespace_on( + namespace, + 1, + CREATED_VIEW, + iggy_common::Durability::Persisted, + &[1, 2], + ); + + let route = partition_primary(&shard, namespace).await; + + assert!( + matches!( + route, + Some(PartitionReadReply::Primary { + view: CREATED_VIEW, + primary: 2, + }) + ), + "{route:?}" + ); + } + /// The payload only feeds the error message, so a wire id with no domain /// form reports the default identifier instead of failing the lookup. #[test]
diff --git a/core/server/src/partition_helpers.rs b/core/server/src/partition_helpers.rs index a08665b..13a1e27 100644 --- a/core/server/src/partition_helpers.rs +++ b/core/server/src/partition_helpers.rs
@@ -37,7 +37,8 @@ use compio::fs::create_dir_all; use configs::server::ServerConfig; use consensus::{ - FreshGroupStart, JoinMode, LocalPipeline, VsrConsensus, VsrRestore, VsrState, fresh_group_start, + FreshGroupStart, JoinMode, LocalPipeline, ReplicaSet, VsrConsensus, VsrRestore, VsrState, + fresh_group_start, }; use iggy_common::{IggyByteSize, IggyError, IggyTimestamp, PartitionStats, TopicRuntimeOptions}; use journal::durable_storage::{DiskStorage, DurableStorage}; @@ -63,6 +64,11 @@ const PARTITION_INITIALIZATION_DIR: &str = "partition-initialization"; const RETIREMENT_FENCE_FILE: &str = "retirement.fence"; +/// Creation time, in micros, of the oldest data this node's storage can hold. +const STORAGE_EPOCH_FILE: &str = "storage.epoch"; +/// Clock skew tolerated between this node and the metadata primary that +/// stamped a partition's creation time. +const STORAGE_EPOCH_SKEW_MICROS: u64 = 10_000_000; /// Open the durable superblock for one partition's consensus group and read /// back the last recorded VSR state. @@ -182,10 +188,11 @@ topic_runtime: TopicRuntimeOptions, cluster_id: u128, self_replica_id: u8, - replica_count: u8, + members: ReplicaSet, bus: Rc<IggyMessageBus>, partitions: &IggyPartitions<Rc<IggyMessageBus>, PingPongSuperblock>, ) -> Result<Option<IggyPartition<Rc<IggyMessageBus>>>, ServerError> { + let replica_count = members.len(); let created_revision = partition_metadata.created_revision; match partition_retirement_fence(&config.get_system_path(), namespace).await { Ok(Some(recorded)) if recorded > created_revision => return Ok(None), @@ -208,8 +215,46 @@ Ok(_) => {} } let initialization_dir = partition_initialization_path(&config.get_system_path(), namespace); - match partitions::read_created_revision(&initialization_dir).await { + let initialization = partitions::read_created_revision(&initialization_dir).await; + let lost_with_storage = matches!(initialization, Ok(None)) + && predates_storage( + storage_epoch(&config.get_system_path()).await?, + partition_metadata.created_at.as_micros(), + ); + match initialization { Ok(Some(recorded)) if recorded > created_revision => return Ok(None), + // Boot reaches here before the reconciler for every partition this node + // is a member of. A load would take the empty log for a new member's and + // let it elect, so a wider group rebuilds as the reconciler would. + Ok(None) if lost_with_storage && replica_count > 1 => { + return Box::pin(build_partition_fresh( + config, + partitions.config(), + namespace, + partition_stats, + created_revision, + partition_metadata.created_at.as_micros(), + topic_runtime, + cluster_id, + self_replica_id, + members, + partition_metadata.created_view, + bus, + )) + .await + .map(Some); + } + Ok(None) if lost_with_storage => { + error!( + namespace_raw = namespace.inner(), + created_revision, + "this node alone stores the partition, and its storage began after the \ + partition was created without any record of it: refusing to serve it from \ + offset 0; delete and recreate the topic to accept the loss" + ); + partitions.fence(namespace, created_revision); + return Ok(None); + } Err(source) if source.kind() == std::io::ErrorKind::InvalidData => { error!(%source, namespace_raw = namespace.inner(), "unreadable initialization record; refusing partition"); partitions.fence(namespace, created_revision); @@ -348,7 +393,7 @@ topic_runtime, cluster_id, self_replica_id, - replica_count, + members, Rc::clone(&bus), )) .await @@ -492,10 +537,11 @@ namespace, Arc::clone(&partition_stats), partition_metadata.created_revision, + partition_metadata.created_at.as_micros(), topic_runtime, cluster_id, self_replica_id, - replica_count, + members, partition_metadata.created_view, Rc::clone(&bus), )) @@ -556,9 +602,10 @@ runtime_options: TopicRuntimeOptions, cluster_id: u128, self_replica_id: u8, - replica_count: u8, + members: ReplicaSet, bus: Rc<IggyMessageBus>, ) -> Result<IggyPartition<Rc<IggyMessageBus>>, ServerError> { + let replica_count = members.len(); let stream_id = namespace.stream_id(); let topic_id = namespace.topic_id(); let partition_id = namespace.partition_id(); @@ -612,7 +659,7 @@ let consensus = VsrConsensus::restored( cluster_id, self_replica_id, - replica_count, + members, namespace.inner(), bus, LocalPipeline::with_capacities(prepare_queue_depth, prepare_queue_depth * 2), @@ -930,13 +977,15 @@ namespace: IggyNamespace, stats: Arc<PartitionStats>, created_revision: u64, + created_at_micros: u64, runtime_options: TopicRuntimeOptions, cluster_id: u128, self_replica_id: u8, - replica_count: u8, + members: ReplicaSet, created_view: u32, bus: Rc<IggyMessageBus>, ) -> Result<IggyPartition<Rc<IggyMessageBus>>, ServerError> { + let replica_count = members.len(); if partition_retirement_fence(&config.get_system_path(), namespace) .await? .is_some_and(|recorded| recorded >= created_revision) @@ -951,9 +1000,36 @@ let partition_id = namespace.partition_id(); let first_initialization = partition_needs_initialization(config, namespace, created_revision).await?; + // The initialization record went with the storage, so this build cannot be + // told from a first one. A sole member would restart the log at offset 0. + let lost_with_storage = first_initialization + && predates_storage( + storage_epoch(&config.get_system_path()).await?, + created_at_micros, + ); + if replica_count == 1 && lost_with_storage { + return Err(ServerError::PartitionLostWithStorage { + namespace_raw: namespace.inner(), + created_revision, + }); + } + // A wider group's members joining at the creation view with empty logs + // would form a quorum and restart it the same way if they all lost their + // storage. Joining as a prior participant leaves this member out of every + // election until a member that kept the history transfers it. + if lost_with_storage { + warn!( + stream_id, + topic_id, + partition_id, + created_revision, + "this node's storage began after the partition was created without any record \ + of it: waiting for a member that kept its history" + ); + } let partition_dir = config.get_partition_path(stream_id, topic_id, partition_id); - let restarted = replica_count > 1 && !first_initialization; + let restarted = replica_count > 1 && (!first_initialization || lost_with_storage); if replica_count == 1 && !first_initialization { let directory = Path::new(&partition_dir).join(format!("prepares-{created_revision}")); if !journal::PartitionPrepareJournal::has_published_frontier(&directory) @@ -1036,7 +1112,7 @@ let consensus = VsrConsensus::restored( cluster_id, self_replica_id, - replica_count, + members, namespace.inner(), bus, LocalPipeline::with_capacities(prepare_queue_depth, prepare_queue_depth * 2), @@ -1057,7 +1133,7 @@ let mut partition = IggyPartition::new(stats, consensus); partition .consensus() - .set_recovery_election_allowed(first_initialization); + .set_recovery_election_allowed(first_initialization && !lost_with_storage); partition.set_runtime_options(runtime_options); partition.set_superblock(superblock, recovered_state.as_ref()); // Surface the evicted-ring ceilings from config onto the fresh journal. @@ -1209,6 +1285,38 @@ }) } +/// Record that this node's storage holds nothing older than `now_micros`. +/// Boot calls it when it finds no metadata state in a multi-node cluster. +/// +/// # Errors +/// The record could not be written durably. +pub async fn record_storage_epoch(system_path: &str, now_micros: u64) -> Result<(), ServerError> { + partitions::write_revision_record(system_path, STORAGE_EPOCH_FILE, now_micros) + .await + .map_err(|source| ServerError::PartitionCreatedRevisionIo { + dir: PathBuf::from(system_path), + source, + }) +} + +async fn storage_epoch(system_path: &str) -> Result<Option<u64>, ServerError> { + partitions::read_revision_record(system_path, STORAGE_EPOCH_FILE) + .await + .map_err(|source| ServerError::PartitionCreatedRevisionIo { + dir: PathBuf::from(system_path), + source, + }) +} + +/// Whether a partition created at `created_at_micros` existed before this +/// node's storage began, so a missing initialization record means lost data. +const fn predates_storage(storage_epoch: Option<u64>, created_at_micros: u64) -> bool { + match storage_epoch { + Some(epoch) => created_at_micros.saturating_add(STORAGE_EPOCH_SKEW_MICROS) < epoch, + None => false, + } +} + async fn partition_needs_initialization( config: &ServerConfig, namespace: IggyNamespace, @@ -1385,7 +1493,7 @@ TopicRuntimeOptions::default(), CLUSTER, 0, - replica_count, + ReplicaSet::all(replica_count), Rc::new(IggyMessageBus::new(0)), &partitions, ) @@ -1420,10 +1528,11 @@ namespace, Arc::new(PartitionStats::default()), REVISION, + 0, TopicRuntimeOptions::default(), CLUSTER, 0, - replica_count, + ReplicaSet::all(replica_count), 0, Rc::new(IggyMessageBus::new(0)), ) @@ -1439,7 +1548,7 @@ TopicRuntimeOptions::default(), CLUSTER, 0, - replica_count, + ReplicaSet::all(replica_count), Rc::new(IggyMessageBus::new(0)), &partitions, ) @@ -1457,6 +1566,119 @@ } #[compio::test] + async fn given_storage_younger_than_an_unrecorded_partition_when_loading_should_refuse_a_singleton_and_rebuild_a_wider_member_as_lost() + { + const REVISION: u64 = 7; + const EPOCH: u64 = 1_000 * STORAGE_EPOCH_SKEW_MICROS; + let namespace = IggyNamespace::new(1, 1, 0); + for (replica_count, created_at, served, lost) in [ + (1, EPOCH - 2 * STORAGE_EPOCH_SKEW_MICROS, false, true), + (1, EPOCH - STORAGE_EPOCH_SKEW_MICROS / 2, true, false), + (1, EPOCH + 1, true, false), + (REPLICAS, EPOCH - 2 * STORAGE_EPOCH_SKEW_MICROS, true, true), + (REPLICAS, EPOCH + 1, true, false), + ] { + let root = tempfile::tempdir().unwrap(); + let config = solo_config(&root); + let partitions = solo_partitions(&config); + std::fs::create_dir_all(config.get_system_path()).unwrap(); + record_storage_epoch(&config.get_system_path(), EPOCH) + .await + .unwrap(); + let loaded = load_partition_or_fence( + &config, + namespace, + Arc::new(PartitionStats::default()), + &Partition::new( + 0, + namespace.inner(), + IggyTimestamp::from(created_at), + REVISION, + 0, + ), + TopicRuntimeOptions::default(), + CLUSTER, + 0, + ReplicaSet::all(replica_count), + Rc::new(IggyMessageBus::new(0)), + &partitions, + ) + .await + .unwrap(); + let context = format!("replicas={replica_count} created_at={created_at}"); + assert_eq!(loaded.is_some(), served, "{context}"); + assert_eq!(partitions.is_tombstoned(&namespace), !served, "{context}"); + if let Some(partition) = loaded + && replica_count > 1 + { + assert_eq!(partition.requires_state_transfer(), lost, "{context}"); + assert_eq!( + partition.consensus().recovery_election_allowed(), + !lost, + "{context}: a lost copy elects nothing on its empty log" + ); + } + } + } + + /// The reconciler's path: a partition whose directory went with the + /// storage reaches the fresh build. Boot's loader sends it there too. + #[compio::test] + async fn given_storage_younger_than_an_unrecorded_partition_when_building_fresh_should_refuse_a_singleton_and_keep_a_wider_member_out_of_elections() + { + const REVISION: u64 = 7; + const EPOCH: u64 = 1_000 * STORAGE_EPOCH_SKEW_MICROS; + let namespace = IggyNamespace::new(1, 1, 0); + for (replica_count, created_at, built, lost) in [ + (1, EPOCH - 2 * STORAGE_EPOCH_SKEW_MICROS, false, true), + (1, EPOCH + 1, true, false), + (REPLICAS, EPOCH - 2 * STORAGE_EPOCH_SKEW_MICROS, true, true), + (REPLICAS, EPOCH + 1, true, false), + ] { + let root = tempfile::tempdir().unwrap(); + let config = solo_config(&root); + std::fs::create_dir_all(config.get_system_path()).unwrap(); + record_storage_epoch(&config.get_system_path(), EPOCH) + .await + .unwrap(); + let result = build_partition_fresh( + &config, + &solo_partitions_config(&config), + namespace, + Arc::new(PartitionStats::default()), + REVISION, + created_at, + TopicRuntimeOptions::default(), + CLUSTER, + 0, + ReplicaSet::all(replica_count), + 0, + Rc::new(IggyMessageBus::new(0)), + ) + .await; + let context = format!("replicas={replica_count} created_at={created_at}"); + match result { + Ok(partition) => { + assert!(built, "{context}"); + assert_eq!(partition.requires_state_transfer(), lost, "{context}"); + assert_eq!( + partition.consensus().recovery_election_allowed(), + !lost, + "{context}" + ); + } + Err(error) => { + assert!(!built, "{context}"); + assert!( + matches!(error, ServerError::PartitionLostWithStorage { .. }), + "{context}" + ); + } + } + } + } + + #[compio::test] async fn given_initialized_singleton_when_rebuilding_lost_directory_should_refuse() { let root = tempfile::tempdir().unwrap(); let config = solo_config(&root); @@ -1488,10 +1710,11 @@ namespace, Arc::new(PartitionStats::default()), 0, + 0, TopicRuntimeOptions::default(), CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), CREATED_VIEW, Rc::new(IggyMessageBus::new(0)), ) @@ -1524,7 +1747,7 @@ TopicRuntimeOptions::default(), CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), Rc::new(IggyMessageBus::new(0)), )) .await @@ -1535,10 +1758,11 @@ namespace, Arc::new(PartitionStats::default()), 0, + 0, TopicRuntimeOptions::default(), CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), CREATED_VIEW, Rc::new(IggyMessageBus::new(0)), ) @@ -1591,7 +1815,7 @@ TopicRuntimeOptions::default(), CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), Rc::new(IggyMessageBus::new(0)), &partitions, ) @@ -1615,10 +1839,11 @@ namespace, Arc::new(PartitionStats::default()), 0, + 0, TopicRuntimeOptions::default(), CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), CREATED_VIEW, Rc::new(IggyMessageBus::new(0)), ) @@ -1659,7 +1884,7 @@ runtime, CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), Rc::new(IggyMessageBus::new(0)), )) .await; @@ -1679,7 +1904,7 @@ runtime, CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), Rc::new(IggyMessageBus::new(0)), &partitions, ) @@ -1805,10 +2030,11 @@ namespace, Arc::new(PartitionStats::default()), REVISION, + 0, TopicRuntimeOptions::default(), CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), 0, Rc::new(IggyMessageBus::new(0)), ) @@ -1957,10 +2183,11 @@ namespace, Arc::new(PartitionStats::default()), 0, + 0, runtime, CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), CREATED_VIEW, Rc::new(IggyMessageBus::new(0)), ) @@ -2024,7 +2251,7 @@ runtime, CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), Rc::new(IggyMessageBus::new(0)), )) .await @@ -2102,7 +2329,7 @@ TopicRuntimeOptions::default(), CLUSTER, 0, - 1, + ReplicaSet::all(1), Rc::new(IggyMessageBus::new(0)), &partitions, ) @@ -2153,7 +2380,7 @@ TopicRuntimeOptions::default(), CLUSTER, 0, - 1, + ReplicaSet::all(1), Rc::new(IggyMessageBus::new(0)), &partitions, ) @@ -2195,10 +2422,11 @@ namespace, Arc::new(PartitionStats::default()), 0, + 0, runtime, CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), 0, Rc::new(IggyMessageBus::new(0)), ) @@ -2280,7 +2508,7 @@ runtime, CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), Rc::new(IggyMessageBus::new(0)), )) .await @@ -2320,10 +2548,11 @@ namespace, Arc::new(PartitionStats::default()), 0, + 0, runtime, CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), 0, Rc::new(IggyMessageBus::new(0)), ) @@ -2358,7 +2587,7 @@ runtime, CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), Rc::new(IggyMessageBus::new(0)), &partitions, ) @@ -2444,10 +2673,11 @@ IggyNamespace::new(1, 1, 0), Arc::new(PartitionStats::default()), 0, + 0, TopicRuntimeOptions::default(), CLUSTER, 0, - 1, + ReplicaSet::all(1), 0, Rc::new(IggyMessageBus::new(0)), ) @@ -2488,10 +2718,11 @@ IggyNamespace::new(1, 1, 0), Arc::new(PartitionStats::default()), created_revision, + 0, TopicRuntimeOptions::default(), CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), created_view, Rc::new(IggyMessageBus::new(0)), ) @@ -2537,7 +2768,7 @@ }, CLUSTER, REPLICA, - REPLICAS, + ReplicaSet::all(REPLICAS), Rc::new(IggyMessageBus::new(0)), partitions, ) @@ -2681,7 +2912,7 @@ TopicRuntimeOptions::default(), CLUSTER, 0, - 1, + ReplicaSet::all(1), Rc::new(IggyMessageBus::new(0)), &partitions, ));
diff --git a/core/server/src/partition_reconciler.rs b/core/server/src/partition_reconciler.rs index 892cde1..30adc20 100644 --- a/core/server/src/partition_reconciler.rs +++ b/core/server/src/partition_reconciler.rs
@@ -158,7 +158,7 @@ use ahash::{AHashMap, AHashSet}; use configs::server::ServerConfig; use consensus::Consensus; -use consensus::{MetadataHandle, PartitionsHandle}; +use consensus::{MetadataHandle, PartitionsHandle, ReplicaSet, quorum_view_change}; use futures::FutureExt; use iggy_binary_protocol::primitives::partition_history::PartitionContext; use iggy_binary_protocol::requests::consumer_offsets::DeleteConsumerOffsetRequest; @@ -176,7 +176,7 @@ use metadata::impls::metadata::StreamsFrontend; use metadata::stm::lifecycle::{ CompleteLifecycleRequest, LIFECYCLE_COMPLETIONS_MAX, LifecycleCompletion, LifecycleFence, - RETIRED_PARTITION_OP, + RETIRED_PARTITION_OP, retired_member_op, }; use metadata::stm::stream::{Partition, StatsRegistry}; use partitions::{StreamKeyIdentity, delete_partitions_from_disk, read_created_revision}; @@ -923,7 +923,7 @@ topic_runtime, ctx.cluster_id, ctx.self_replica_id, - ctx.replica_count, + partition_metadata.replica_set(ctx.replica_count), Rc::clone(&ctx.shard.bus), partitions, ) @@ -935,10 +935,11 @@ ns, partition_stats, created_revision, + partition_metadata.created_at.as_micros(), topic_runtime, ctx.cluster_id, ctx.self_replica_id, - ctx.replica_count, + partition_metadata.replica_set(ctx.replica_count), created_view, Rc::clone(&ctx.shard.bus), ) @@ -962,6 +963,17 @@ // knows: nothing is counted, so the pass may fast-skip, and the // commit that moves metadata on bumps the revision and retries it. Ok(None) => {} + // Retrying cannot bring the history back; the tombstone keeps every + // later pass from rebuilding it empty. + Err(err @ crate::server_error::ServerError::PartitionLostWithStorage { .. }) => { + error!( + shard = shard_id, + ns_raw = ns.inner(), + error = %err, + "refusing to rebuild a partition lost with this node's storage" + ); + partitions.fence(ns, created_revision); + } Err(err) => { ctx.record_failure(ns, FailureCause::Add, now); ctx.shard.metrics().record_partition_reconcile_failure(); @@ -1602,6 +1614,15 @@ for (_, stream) in &inner.items { for (topic_id, topic) in &stream.topics { for partition in &topic.partitions { + // A partition this node does not store is no target: it + // gets neither a build nor a routing row here. + if partition + .replica_set(ctx.replica_count) + .member(ctx.self_replica_id) + .is_none() + { + continue; + } entries.push(TargetPartition { ns: IggyNamespace::new(stream.id, topic_id, partition.id), epoch: partition.created_revision, @@ -1837,67 +1858,99 @@ } } -/// On a single-replica node, recovery fences a partition it cannot load, and -/// then no log can install a lifecycle fence. Without this, a delete of the -/// topic, the operator's exit from that fence, would wait forever. The durable -/// retirement fence keeps the incarnation offline, which is what the lifecycle -/// fence would have proven. +/// On a partition this node alone stores, recovery fences an incarnation it +/// cannot load, and then no log can install a lifecycle fence. Without this, a +/// delete of the topic, the operator's exit from that fence, would wait +/// forever. The durable retirement fence keeps the incarnation offline, which +/// is what the lifecycle fence would have proven. +/// +/// A wider partition being deleted gets the same exit once enough of its +/// members lost their copies: each such member retires its copy durably and +/// reports it, and when the members still holding one are too few to elect a +/// primary, a member completes the target. No log could install the fence +/// after that. async fn reconcile_retired_lifecycles(ctx: &ReconcilerCtx) { - if ctx.replica_count != 1 || ctx.lifecycle_reports.batch_in_flight.get() { + if ctx.lifecycle_reports.batch_in_flight.get() { return; } let partitions = ctx.shard.plane.partitions(); let system_path = ctx.config.get_system_path(); - // The fence writes below await, so only the targets that failed recovery - // leave the borrow. + // The fence writes below await, so the targets leave the borrow first. let streams = ctx.shard.plane.metadata().mux_stm.streams(); - let failed = streams.read(|state| { - let mut failed = Vec::new(); + let pending = streams.read(|state| { + let mut pending = Vec::new(); for intent in state.lifecycle_intents.values() { - let mut targets = Vec::new(); - for target in intent + let targets: Vec<_> = intent .partitions .iter() .filter(|target| target.partition_op.is_none()) - { - let namespace = IggyNamespace::new( - intent.stream_id as usize, - target.topic_id as usize, - target.partition_id as usize, - ); - let incarnation = match target.fence { - LifecycleFence::History(transition) => transition.incarnation, - LifecycleFence::Owner(installation) => installation.incarnation, - }; - if partitions.failed_revision(&namespace) == Some(incarnation) { - targets.push((target.topic_id, target.partition_id, namespace, incarnation)); - } - } + .map(|target| { + let (incarnation, deletes) = match target.fence { + LifecycleFence::History(transition) => (transition.incarnation, true), + LifecycleFence::Owner(installation) => (installation.incarnation, false), + }; + RetirementTarget { + topic_id: target.topic_id, + partition_id: target.partition_id, + namespace: IggyNamespace::new( + intent.stream_id as usize, + target.topic_id as usize, + target.partition_id as usize, + ), + incarnation, + deletes, + retired_members: target.retired_members, + } + }) + .collect(); if !targets.is_empty() { - failed.push((intent.context.metadata_op, intent.stream_id, targets)); + pending.push((intent.context.metadata_op, intent.stream_id, targets)); } } - failed + pending }); - for (metadata_op, stream_id, targets) in failed { + for (metadata_op, stream_id, targets) in pending { let mut completions = Vec::new(); - for (topic_id, partition_id, namespace, incarnation) in targets { - // Other tasks run during the awaits, so each target is checked - // again right before its durable write. - if partitions.failed_revision(&namespace) != Some(incarnation) { + for target in targets { + let Some(members) = committed_members(ctx, target.namespace) else { + continue; + }; + if members.member(ctx.self_replica_id).is_none() { continue; } - if let Err(error) = - record_partition_retirement_fence(&system_path, namespace, incarnation).await - { - warn!(%error, namespace_raw = namespace.inner(), - "lifecycle waits for a durable partition fence"); - continue; - } + let partition_op = match retirement_step(ctx, &members, &target) { + Some(RetirementStep::Complete) => RETIRED_PARTITION_OP, + Some(RetirementStep::RetireCopy) => { + // Other tasks run during the awaits, so the copy is checked + // again right before its durable write. + if !copy_lost_here(ctx, &target) { + continue; + } + if let Err(error) = record_partition_retirement_fence( + &system_path, + target.namespace, + target.incarnation, + ) + .await + { + warn!(%error, namespace_raw = target.namespace.inner(), + "lifecycle waits for a durable partition fence"); + continue; + } + if members.len() == 1 { + RETIRED_PARTITION_OP + } else { + // A retired copy must not vote either. + partitions.fence(target.namespace, target.incarnation); + retired_member_op(ctx.self_replica_id) + } + } + None => continue, + }; completions.push(LifecycleCompletion { - topic_id, - partition_id, - partition_op: RETIRED_PARTITION_OP, + topic_id: target.topic_id, + partition_id: target.partition_id, + partition_op, }); // Metadata records only the reported batch, so a fence written // past it is written again on the next pass. @@ -1912,6 +1965,85 @@ } } +/// A pending lifecycle target as [`reconcile_retired_lifecycles`] reads it. +struct RetirementTarget { + topic_id: u32, + partition_id: u32, + namespace: IggyNamespace, + incarnation: u64, + /// The lifecycle removes the partition, so retiring a copy loses nothing. + deletes: bool, + retired_members: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum RetirementStep { + /// Too few members still hold a copy to elect a primary, so no log can + /// fence it. + Complete, + /// This member's copy is lost: retire it durably and report it. + RetireCopy, +} + +/// What this member does for `target`, or `None` to wait for its log. +fn retirement_step( + ctx: &ReconcilerCtx, + members: &ReplicaSet, + target: &RetirementTarget, +) -> Option<RetirementStep> { + if members.len() == 1 { + return copy_lost_here(ctx, target).then_some(RetirementStep::RetireCopy); + } + if !target.deletes { + return None; + } + let retired = members + .nodes() + .iter() + .filter(|&&node| retired_bit(node).is_some_and(|bit| target.retired_members & bit != 0)) + .count(); + // Electing the primary that installs the fence takes a view-change quorum + // of the members that kept a copy. A primary that kept its copy may still + // commit with fewer, but it installs the fence itself, and what it commits + // afterwards belongs to a partition being deleted. + let holding = usize::from(members.len()) - retired; + if holding < quorum_view_change(members.len()) { + return Some(RetirementStep::Complete); + } + let retired_here = + retired_bit(ctx.self_replica_id).is_none_or(|bit| target.retired_members & bit != 0); + (!retired_here && copy_lost_here(ctx, target)).then_some(RetirementStep::RetireCopy) +} + +const fn retired_bit(node: u8) -> Option<u32> { + 1_u32.checked_shl(node as u32) +} + +/// Whether this node's copy of the target incarnation is gone: recovery +/// fenced it, or it waits for a state transfer that would rebuild it. +fn copy_lost_here(ctx: &ReconcilerCtx, target: &RetirementTarget) -> bool { + let partitions = ctx.shard.plane.partitions(); + partitions.failed_revision(&target.namespace) == Some(target.incarnation) + || partitions + .get_by_ns(&target.namespace) + .is_some_and(|partition| { + partition.created_revision() == target.incarnation + && partition.requires_state_transfer() + }) +} + +/// The committed members of `namespace`, every node for a partition created +/// before replication factors. +fn committed_members(ctx: &ReconcilerCtx, namespace: IggyNamespace) -> Option<ReplicaSet> { + ctx.shard + .plane + .metadata() + .mux_stm + .streams() + .members_for_namespace(namespace) + .map(|members| members.unwrap_or_else(|| ReplicaSet::all(ctx.replica_count))) +} + /// The ready completions of the intent at `metadata_op` in batch order, at /// most [`LIFECYCLE_COMPLETIONS_MAX`] of them. The next batch takes the rest. fn lifecycle_batch( @@ -2061,6 +2193,7 @@ use metadata::stm::StateMachine; use metadata::stm::lifecycle::{ CompleteLifecycleRequest, LIFECYCLE_COMPLETIONS_MAX, LifecycleCompletion, + RETIRED_PARTITION_OP, retired_member_op, }; use metadata::stm::stream::Streams; use metadata::stm::user::Users; @@ -2068,14 +2201,14 @@ use server_common::sharding::{IggyNamespace, ShardId}; use server_common::{Message, MessageBag}; use shard::shards_table::{PapayaShardsTable, ShardsTable, calculate_shard_assignment}; - use shard::{IggyShard, PartitionConsensusConfig, ReconcileOp, ShardIdentity}; + use shard::{IggyShard, PartitionConsensusConfig, ReconcileOp, Sender, ShardIdentity}; use std::cell::Cell; use std::mem::size_of; use std::path::Path; use std::rc::Rc; use std::sync::Arc; use std::sync::atomic::Ordering; - use std::time::Instant; + use std::time::{Duration, Instant}; use tempfile::TempDir; type TestMux = MuxStateMachine<iggy_common::variadic!(Users, Streams)>; @@ -2226,6 +2359,7 @@ initial_view: 0, partition_id, consensus_group_id, + members: Vec::new(), } } @@ -3327,10 +3461,11 @@ ns, Arc::clone(&stats), LIVE_EPOCH, + 0, iggy_common::TopicRuntimeOptions::default(), CLUSTER_ID, 0, - 1, + consensus::ReplicaSet::all(1), created_view(&ctx, ns), Rc::clone(&ctx.shard.bus), ) @@ -3358,10 +3493,11 @@ ns, Arc::clone(&stats), LIVE_EPOCH + 1, + 0, iggy_common::TopicRuntimeOptions::default(), CLUSTER_ID, 0, - 1, + consensus::ReplicaSet::all(1), created_view(&ctx, ns), Rc::clone(&ctx.shard.bus), ) @@ -4001,7 +4137,15 @@ seed_topic(&mux, 2, 0, "topic-repair-ceiling", vec![assignment(0, 1)]); let shard = build_test_shard(0, &config, mux); - let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config)); + // The repair peer must be a member of the group it repairs from. + let ctx = Rc::new(ReconcilerCtx::new( + Rc::clone(&shard), + 1, + Rc::new(config), + CLUSTER_ID, + 0, + 2, + )); reconcile_pass(&ctx).await; let ns = IggyNamespace::new(0, 0, 0); @@ -4405,6 +4549,239 @@ } #[compio::test] + async fn given_a_fenced_partition_only_this_node_stores_in_a_larger_cluster_when_deleted_should_complete_the_delete() + { + const NODES: u8 = 3; + let tmp = TempDir::new().unwrap(); + let config = test_config(&tmp); + let mux = TestMux::default(); + let solo = IggyNamespace::new(0, 0, 0); + let shared = IggyNamespace::new(1, 0, 0); + mux.streams() + .seed_namespace_on(solo, 1, 0, iggy_common::Durability::Persisted, &[0]); + mux.streams() + .seed_namespace_on(shared, 2, 0, iggy_common::Durability::Persisted, &[0, 1]); + let (shard, lanes) = build_test_shard_with_inbox(0, &config, mux, 16); + let ctx = Rc::new(ReconcilerCtx::new( + Rc::clone(&shard), + 1, + Rc::new(config), + CLUSTER_ID, + 0, + NODES, + )); + let stm = &shard.plane.metadata().mux_stm; + for namespace in [solo, shared] { + let revision = stm + .streams() + .created_revision_for_namespace(namespace) + .unwrap(); + shard.plane.partitions().fence(namespace, revision); + } + for (op, stream_id) in [(10, 0), (11, 1)] { + stm.update(build_prepare( + op, + Operation::DeleteTopic, + &DeleteTopicRequest { + stream_id: WireIdentifier::numeric(stream_id), + topic_id: WireIdentifier::numeric(0), + }, + )) + .expect("DeleteTopic apply succeeds"); + } + + reconcile_retired_lifecycles(&ctx).await; + + let (_, batches) = drain_lifecycle_reports(&lanes); + let completed: Vec<_> = batches + .iter() + .map(|(request, _)| request.completions().to_vec()) + .collect(); + assert_eq!( + completed, + [vec![LifecycleCompletion { + topic_id: 0, + partition_id: 0, + partition_op: metadata::stm::lifecycle::RETIRED_PARTITION_OP, + }]], + "only the partition no other node stores may complete through its fence" + ); + } + + /// A delete of a partition whose members lost their copies: each such + /// member retires its copy and reports it, and once too few members still + /// hold one to elect a primary, a member completes the target, since no + /// log can install its fence any more. + #[compio::test] + async fn given_a_deleted_partition_whose_members_lost_their_copies_when_reconciling_should_complete_once_too_few_can_elect() + { + const NODES: u8 = 3; + let tmp = TempDir::new().unwrap(); + let config = test_config(&tmp); + let mux = TestMux::default(); + let namespace = IggyNamespace::new(0, 0, 0); + mux.streams().seed_namespace_on( + namespace, + 1, + 0, + iggy_common::Durability::Persisted, + &[0, 1, 2], + ); + let (shard, lanes) = build_test_shard_with_inbox(0, &config, mux, 16); + let ctx = Rc::new(ReconcilerCtx::new( + Rc::clone(&shard), + 1, + Rc::new(config), + CLUSTER_ID, + 0, + NODES, + )); + let stm = &shard.plane.metadata().mux_stm; + let revision = stm + .streams() + .created_revision_for_namespace(namespace) + .unwrap(); + shard.plane.partitions().fence(namespace, revision); + stm.update(build_prepare( + 10, + Operation::DeleteTopic, + &DeleteTopicRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + }, + )) + .expect("DeleteTopic apply succeeds"); + let completion = |partition_op| LifecycleCompletion { + topic_id: 0, + partition_id: 0, + partition_op, + }; + let in_flight = Rc::clone(&ctx.lifecycle_reports.batch_in_flight); + let release = |reply: Sender<bool>| async move { + reply.try_send(true).unwrap(); + compio::time::timeout(Duration::from_secs(5), async { + while in_flight.get() { + compio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .unwrap(); + }; + + reconcile_retired_lifecycles(&ctx).await; + let (_, mut batches) = drain_lifecycle_reports(&lanes); + assert_eq!(batches.len(), 1); + assert_eq!( + batches[0].0.completions(), + [completion(retired_member_op(0))], + "a member that lost its copy retires it, and one of three cannot finish" + ); + assert_eq!( + crate::partition_helpers::partition_retirement_fence( + &ctx.config.get_system_path(), + namespace + ) + .await + .unwrap(), + Some(revision), + "the copy is retired durably before it is reported" + ); + let (request, reply) = batches.pop().unwrap(); + stm.update(build_prepare(11, Operation::CompleteLifecycle, &request)) + .expect("CompleteLifecycle apply succeeds"); + release(reply).await; + + reconcile_retired_lifecycles(&ctx).await; + let (_, batches) = drain_lifecycle_reports(&lanes); + assert!( + batches.is_empty(), + "two members still hold a copy and can elect the primary that fences it" + ); + + let other = + CompleteLifecycleRequest::new(10, 0, vec![completion(retired_member_op(1))]).unwrap(); + stm.update(build_prepare(12, Operation::CompleteLifecycle, &other)) + .expect("another member's retirement applies"); + assert!(stm.streams().lifecycle_pending(10)); + reconcile_retired_lifecycles(&ctx).await; + let (_, mut batches) = drain_lifecycle_reports(&lanes); + assert_eq!(batches.len(), 1); + assert_eq!( + batches[0].0.completions(), + [completion(RETIRED_PARTITION_OP)], + "one member left cannot elect, so the target completes" + ); + let (request, _) = batches.pop().unwrap(); + stm.update(build_prepare(13, Operation::CompleteLifecycle, &request)) + .expect("the final completion applies"); + assert!( + !stm.streams().lifecycle_pending(10), + "the delete finishes and frees the topic" + ); + } + + /// Four members elect with three votes and commit with two, so two lost + /// copies leave survivors that could commit under a primary but cannot + /// elect one. The target completes then, or the delete would wait forever. + #[compio::test] + async fn given_four_members_when_two_lost_their_copies_should_complete_the_delete() { + const NODES: u8 = 4; + let tmp = TempDir::new().unwrap(); + let config = test_config(&tmp); + let mux = TestMux::default(); + let namespace = IggyNamespace::new(0, 0, 0); + mux.streams().seed_namespace_on( + namespace, + 1, + 0, + iggy_common::Durability::Persisted, + &[0, 1, 2, 3], + ); + let (shard, lanes) = build_test_shard_with_inbox(0, &config, mux, 16); + let ctx = Rc::new(ReconcilerCtx::new( + Rc::clone(&shard), + 1, + Rc::new(config), + CLUSTER_ID, + 0, + NODES, + )); + let stm = &shard.plane.metadata().mux_stm; + stm.update(build_prepare( + 10, + Operation::DeleteTopic, + &DeleteTopicRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + }, + )) + .expect("DeleteTopic apply succeeds"); + let completion = |partition_op| LifecycleCompletion { + topic_id: 0, + partition_id: 0, + partition_op, + }; + // This node kept its copy; nodes 1 and 2 came back without theirs. + assert!(!shard.plane.partitions().is_tombstoned(&namespace)); + for (op, node) in [(11, 1), (12, 2)] { + let retired = + CompleteLifecycleRequest::new(10, 0, vec![completion(retired_member_op(node))]) + .unwrap(); + stm.update(build_prepare(op, Operation::CompleteLifecycle, &retired)) + .expect("a member's retirement applies"); + } + + reconcile_retired_lifecycles(&ctx).await; + let (_, batches) = drain_lifecycle_reports(&lanes); + assert_eq!(batches.len(), 1); + assert_eq!( + batches[0].0.completions(), + [completion(RETIRED_PARTITION_OP)], + "two copies left cannot elect a primary of four" + ); + } + + #[compio::test] async fn given_retirement_fence_when_topic_recreated_between_passes_should_rebuild() { let tmp = TempDir::new().unwrap(); let config = test_config(&tmp);
diff --git a/core/server/src/responses.rs b/core/server/src/responses.rs index fe2b1ce..962956f 100644 --- a/core/server/src/responses.rs +++ b/core/server/src/responses.rs
@@ -74,7 +74,8 @@ }; use iggy_common::wire_conversions::{describe_options_response, resource_options_to_wire_split}; use iggy_common::{ - IggyError, IggyTimestamp, OptionsScope, ResourceOptions, UserStatus, public_options, + HeaderValue, IggyError, IggyTimestamp, OptionKey, OptionsScope, ResourceOptions, UserStatus, + public_options, }; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; @@ -82,6 +83,7 @@ use metadata::stm::stream::{Stream, StreamsInner, Topic}; use metadata::stm::user::User; use server_common::Message; +use server_common::sharding::IggyNamespace; use shard::ConnectedClientInfo; use std::cell::RefCell; use std::net::IpAddr; @@ -329,7 +331,14 @@ { match code { DESCRIBE_OPTIONS_CODE => Ok(NonReplicatedResponse::Bytes( - build_describe_options_response(body)?.to_bytes(), + build_describe_options_response( + body, + shard + .plane + .metadata() + .default_replication_factor(shard.replica_topology().replica_count), + )? + .to_bytes(), )), GET_CLUSTER_METADATA_CODE => Ok(NonReplicatedResponse::Bytes( build_cluster_metadata_response(roster, shard, client_ip).to_bytes(), @@ -747,8 +756,12 @@ } } -/// Serve the option catalog for one resource scope. -fn build_describe_options_response(body: &[u8]) -> Result<DescribeOptionsResponse, IggyError> { +/// Serve the option catalog for one resource scope, with the replication +/// factor this node gives a topic created without one. +fn build_describe_options_response( + body: &[u8], + default_replication_factor: u8, +) -> Result<DescribeOptionsResponse, IggyError> { let request = DescribeOptionsRequest::decode_from(body).map_err(|_| IggyError::InvalidCommand)?; let scope = match request.scope { @@ -757,7 +770,13 @@ } _ => return Err(IggyError::InvalidCommand), }; - describe_options_response(scope) + describe_options_response( + scope, + &[( + OptionKey::ReplicationFactor, + HeaderValue::from(u32::from(default_replication_factor)), + )], + ) } /// The explicit and derived blocks a read returns for `stored`. @@ -896,6 +915,7 @@ .topics .get(topic_id) .ok_or(IggyError::InvalidIdentifier)?; + let nodes = shard.replica_topology().replica_count; let partitions = topic .partitions .iter() @@ -906,6 +926,7 @@ topic_id, partition, shard.plane.metadata().applied_frontier().get(), + nodes, ) }) .collect::<Result<Vec<_>, _>>()?; @@ -1011,6 +1032,7 @@ topic_id: usize, partition: &metadata::stm::stream::Partition, metadata_op: u64, + nodes: u8, ) -> Result<PartitionResponse, IggyError> { // Per-partition counters live in the shared stats registry (one `Arc` // across all shards and both left-right buffers). @@ -1042,6 +1064,26 @@ stats.messages_count_inconsistent(), ) }); + let members = partition.replica_set(nodes).nodes().to_vec(); + // The primary's last report, narrowed on the metadata primary to the nodes + // it hears from. No report yet leaves nobody known in sync. + let in_sync_nodes = streams + .stats_registry + .directory() + .get( + IggyNamespace::new(stream_id, topic_id, partition.id), + partition.created_revision, + ) + .map_or(0, |status| status.in_sync_nodes); + let in_sync_members = members + .iter() + .copied() + .filter(|node| { + 1_u32 + .checked_shl(u32::from(*node)) + .is_some_and(|bit| in_sync_nodes & bit != 0) + }) + .collect(); Ok(PartitionResponse { id: usize_to_u32(partition.id)?, created_at: partition.created_at.as_micros(), @@ -1054,6 +1096,8 @@ owner_generation: 0, metadata_op, }, + members, + in_sync_members, }) } @@ -1082,7 +1126,7 @@ #[cfg(test)] mod tests { use super::*; - use iggy_common::{HeaderKey, HeaderValue, OptionKey, OptionValue}; + use iggy_common::{HeaderKey, OptionValue}; use std::str::FromStr; #[test] @@ -1168,7 +1212,8 @@ let partition = Partition::new(0, 1, IggyTimestamp::from(1u64), 0, 0); // Registry miss: the owning shard has not started building. - let predicted = partition_response(&streams, 0, 0, &partition, 0).expect("response builds"); + let predicted = + partition_response(&streams, 0, 0, &partition, 0, 1).expect("response builds"); assert_eq!(predicted.segments_count, 1); assert_eq!(predicted.messages_count, 0); @@ -1179,20 +1224,78 @@ let stats = streams .stats_registry .partition(0, 0, &partition, topic_stats); - let mid_build = partition_response(&streams, 0, 0, &partition, 0).expect("response builds"); + let mid_build = + partition_response(&streams, 0, 0, &partition, 0, 1).expect("response builds"); assert_eq!(mid_build.segments_count, 1); // Materialized: the real counters answer from here on. stats.increment_segments_count(1); stats.increment_messages_count(7); stats.increment_size_bytes(64); - let live = partition_response(&streams, 0, 0, &partition, 0).expect("response builds"); + let live = partition_response(&streams, 0, 0, &partition, 0, 1).expect("response builds"); assert_eq!(live.segments_count, 1); assert_eq!(live.messages_count, 7); assert_eq!(live.size_bytes, 64); } #[test] + fn partition_response_lists_members_and_the_reported_ones_in_sync() { + use iggy_binary_protocol::primitives::partition_status::PartitionStatusEntry; + use metadata::stm::stream::{Partition, StreamsInner}; + + let streams = StreamsInner::new(); + let legacy = Partition::new(0, 1, IggyTimestamp::from(1u64), 0, 0); + let unreported = partition_response(&streams, 0, 0, &legacy, 0, 3).unwrap(); + assert_eq!( + unreported.members, + [0, 1, 2], + "no member list is every node" + ); + assert_eq!(unreported.in_sync_members, Vec::<u8>::new()); + + let report = |in_sync_nodes| PartitionStatusEntry { + namespace: IggyNamespace::new(0, 0, 0).inner(), + created_revision: 0, + view: 0, + commit: 0, + current_offset: 0, + messages_count: 0, + size_bytes: 0, + segments_count: 1, + in_sync_nodes, + }; + streams + .stats_registry + .directory() + .record([report(0b101)], std::time::Instant::now()); + let reported = partition_response(&streams, 0, 0, &legacy, 0, 3).unwrap(); + assert_eq!(reported.in_sync_members, [0, 2]); + + streams.stats_registry.directory().set_ready_nodes(0b011); + let narrowed = partition_response(&streams, 0, 0, &legacy, 0, 3).unwrap(); + assert_eq!( + narrowed.in_sync_members, + [0], + "a member the metadata primary cannot hear is not in sync" + ); + } + + #[test] + fn describe_options_reports_this_nodes_default_replication_factor() { + let body = DescribeOptionsRequest { + scope: OPTIONS_SCOPE_TOPIC, + } + .to_bytes(); + let response = build_describe_options_response(&body, 2).unwrap(); + let entry = response + .entries + .iter() + .find(|entry| entry.key.as_str() == "replication_factor") + .unwrap(); + assert_eq!(entry.default_value.as_ref(), 2_u32.to_le_bytes()); + } + + #[test] fn stats_totals_count_every_committed_partition_before_it_materialises() { use iggy_common::{StreamStats, TopicStats}; use metadata::stm::stream::{Partition, Stream, StreamsInner, Topic};
diff --git a/core/server/src/rewrite.rs b/core/server/src/rewrite.rs index ab5e614..aae2e8a 100644 --- a/core/server/src/rewrite.rs +++ b/core/server/src/rewrite.rs
@@ -72,7 +72,7 @@ use server_common::Message; use std::cell::RefCell; use std::rc::Rc; -use tracing::warn; +use tracing::{debug, warn}; /// The pre-consensus rewrite stages of [`tcp_chain`], in chain order. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -385,14 +385,24 @@ ..OptionColumns::default() }, )?; - if let Some(segment_size) = - TopicCreateOptions::from_resource_options(&admitted.explicit).segment_size - { + let explicit = TopicCreateOptions::from_resource_options(&admitted.explicit); + if let Some(segment_size) = explicit.segment_size { validate_topic_segment_size( segment_size.as_bytes_u64(), iggy_common::MAX_TOPIC_SEGMENT_SIZE, )?; } + let nodes = shard.replica_topology().replica_count; + if let Some(factor) = explicit + .replication_factor + .filter(|factor| *factor > u32::from(nodes)) + { + debug!( + replication_factor = factor, + nodes, "refusing a topic whose replication factor exceeds the node count" + ); + return Err(IggyError::InvalidReplicationFactor); + } let mut stored = derived_defaults(OptionsScope::Topic, &admitted.explicit)?; stored.extend(admitted.explicit); let resolved = TopicResolvedOptions::from(&stored);
diff --git a/core/server/src/server_error.rs b/core/server/src/server_error.rs index cd3473c..3aab0c0 100644 --- a/core/server/src/server_error.rs +++ b/core/server/src/server_error.rs
@@ -231,6 +231,15 @@ namespace_raw: u64, created_revision: u64, }, + #[error( + "partition {namespace_raw} incarnation {created_revision} is stored only on this node, \ + and this node's storage began after it was created without any record of it: \ + refusing to serve it from offset 0; delete and recreate the topic to accept the loss" + )] + PartitionLostWithStorage { + namespace_raw: u64, + created_revision: u64, + }, // Quarantines the one partition rather than treating the group as fresh or // reading through to a superseded view: mirrors the metadata plane's // `RecoveryError::SuperblockUnreadable` policy, minus the boot refusal, @@ -334,6 +343,15 @@ #[error("cluster mode requires --replica-id to identify the current node")] MissingReplicaId, #[error( + "committed metadata places partition {namespace_raw} on node {node}, but the cluster \ + has {nodes} nodes; adding or removing nodes is not supported" + )] + PartitionMemberOutsideCluster { + namespace_raw: u64, + node: u8, + nodes: u8, + }, + #[error( "--replica-id {supplied} was passed with cluster.enabled=false; the WAL would commit \ under replica {default} which permanently fixes this node's identity. Either set \ cluster.enabled=true with a matching nodes[] entry, or drop --replica-id"
diff --git a/core/server_common/src/consensus_message.rs b/core/server_common/src/consensus_message.rs index cf73e99..5c046a2 100644 --- a/core/server_common/src/consensus_message.rs +++ b/core/server_common/src/consensus_message.rs
@@ -848,6 +848,41 @@ } } + /// The replica a partition-group frame names as its sender: a member index + /// of that group. `None` for client requests and metadata-only frames. + #[must_use] + pub fn partition_source_replica(&self) -> Option<u8> { + let generic = match self { + Self::Prepare(message) => message.as_generic(), + Self::PrepareOk(message) => message.as_generic(), + Self::StartViewChange(message) => message.as_generic(), + Self::DoViewChange(message) => message.as_generic(), + Self::StartView(message) => message.as_generic(), + Self::Commit(message) => message.as_generic(), + Self::CommitReply(message) => message.as_generic(), + Self::PrepareProgress(message) => message.as_generic(), + Self::RequestStartView(message) => message.as_generic(), + Self::RequestPrepares(message) => message.as_generic(), + Self::RepairRangeReply(message) => message.as_generic(), + Self::RequestStateTransfer(message) => message.as_generic(), + Self::StateTransferTarget(message) => message.as_generic(), + Self::RequestStateChunk(message) => message.as_generic(), + Self::StateChunk(message) => message.as_generic(), + Self::Request(_) + | Self::RepairPrepare(_) + | Self::ForwardRegister(_) + | Self::ForwardRegisterResult(_) + | Self::ForwardLogout(_) + | Self::ConsumerSessionHeartbeat(_) + | Self::ForwardLogoutResult(_) + | Self::ForwardCompletion(_) + | Self::PrimaryRebalance(_) + | Self::SegmentDelete(_) => return None, + }; + let (_, group) = self.routing(); + (group != METADATA_GROUP).then(|| generic.header().replica) + } + /// Discard the classification and hand back the underlying frame. /// /// Type-erasure only: the backing bytes are untouched, so a later
diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index f2717de..131be88 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs
@@ -38,6 +38,8 @@ ServedState, WatchedState, }; pub use poll::{ConsumerAttachment, PollCompleted}; +/// How often the replication gauges and partition reports refresh. +pub use primary_rebalance::PROBE_INTERVAL as PARTITION_REPORT_INTERVAL; pub use router::CONSENSUS_TICK_INTERVAL; #[cfg(feature = "simulator")] @@ -45,7 +47,7 @@ use consensus::{ AdoptedView, BackupPrefix, ChunkProgress, CommitOutcome, Consensus, ConsensusClock, DVC_HEADERS_MAX, DvcHeaderKind, DvcSuffix, FatalReason, MergedLog, MetadataHandle, MuxPlane, - PartitionsHandle, Pipeline, Plane, PlaneKind, RepairEvidence, ReplicationSend, + PartitionsHandle, Pipeline, Plane, PlaneKind, RepairEvidence, ReplicaSet, ReplicationSend, ReplicationStats, STATE_TRANSFER_MAX_DECODE_RETRIES, STATE_TRANSFER_MAX_STALL_RETRIES, Sequencer, StartViewRefusal, Status, VsrAction, VsrConsensus, build_deny_reply_from_request_header, dvc_blank, dvc_header_kind, encode_prepare_headers, @@ -55,6 +57,9 @@ #[cfg(any(test, feature = "simulator"))] use crossfire::AsyncRxTrait; use futures::FutureExt; +use iggy_binary_protocol::primitives::partition_status::{ + PartitionStatusEntry, decode_status_entries, +}; use iggy_binary_protocol::responses::system::GetMetadataChangesResponse; use iggy_binary_protocol::{ CHECKSUM_UNSEALED, Command, CommitHeader, CommitReplyHeader, ConsensusHeader, ConsumerSession, @@ -145,6 +150,8 @@ created_view: u32, first_initialization: bool, consumer_offsets_max: usize, + /// `None` when every node stores the partition. + members: Option<ReplicaSet>, } #[cfg(feature = "simulator")] @@ -156,6 +163,7 @@ created_view, first_initialization, consumer_offsets_max: partitions::DEFAULT_CONSUMER_OFFSETS_MAX, + members: None, } } @@ -164,6 +172,12 @@ self.consumer_offsets_max = consumer_offsets_max; self } + + #[must_use] + pub const fn with_members(mut self, members: Option<ReplicaSet>) -> Self { + self.members = members; + self + } } /// Replica id + count bundle. @@ -423,6 +437,12 @@ identity: iggy_binary_protocol::requests::system::SessionIdentity, created_revision: u64, }, + /// Arm the incarnation's session gate for `debt_op` and report whether it + /// is paid; see [`partitions::IggyPartition::observe_retirement_debt`]. + RetirementDebt { + created_revision: u64, + debt_op: u64, + }, /// A bound poll. A group read is served only to the partition's installed /// owner. PollOnPrimary { @@ -505,6 +525,7 @@ | Self::Poll { .. } => true, Self::Primary | Self::SessionRetired { .. } + | Self::RetirementDebt { .. } | Self::GroupOffsetState { .. } | Self::ResolveSegmentDeleteOffset { .. } => false, } @@ -524,12 +545,14 @@ /// Reply to a [`PartitionRead`]. #[derive(Debug)] pub enum PartitionReadReply { - /// The partition group's view on this replica and the primary of that view. + /// The partition group's view on this replica and the node id of that + /// view's primary. Primary { view: u32, primary: u8, }, SessionRetired(bool), + RetirementDebt(partitions::RetirementDebtStatus), /// Permanent failure of this incarnation. It is not a retirement proof /// until the caller durably fences the incarnation against future serving. SessionRetirementFailed { @@ -1086,8 +1109,9 @@ /// the per-shard reconciler. No payload: reconciler re-reads target /// state. Drops covered by the periodic safety tick. MetadataCommitTick, - /// Last-known serving views, retried by a bounded sweep on each owning shard. - PartitionPlacementViews(Vec<PartitionPlacementView>), + /// Last-known serving views, retried by a bounded sweep on each owning shard, + /// with the counters of the partitions this node leads. + PartitionPlacementViews(Vec<PartitionPlacementView>, Vec<PartitionStatusEntry>), PrimaryRebalance(Box<primary_rebalance::LocalRebalance>), PartitionPersistenceCompleted(partitions::PersistenceCompletion), /// Wake marker for the reconciler-to-pump funnel. Pump drains the @@ -1949,6 +1973,12 @@ /// Namespaces still owed a placement report in the current interval. partition_placement_owed: Cell<usize>, primary_rebalance: RefCell<primary_rebalance::PrimaryRebalance>, + /// Shard 0: reports of the partitions this node leads, for metadata replies. + partition_status_outbox: primary_rebalance::PartitionStatusOutbox, + /// Shard 0: when stale directory reports were last dropped. + partition_directory_retained_at: Cell<Option<Instant>>, + /// Shard 0: where the next chunk of the directory sweep starts. + partition_directory_retain_cursor: Cell<Option<u64>>, /// Consecutive metadata superblock write failures tolerated before the /// process fail-stops. Defaults to 0 (disabled) so the simulator and tests @@ -2214,6 +2244,9 @@ partition_placement_cursor: Cell::new(None), partition_placement_owed: Cell::new(0), primary_rebalance: RefCell::default(), + partition_status_outbox: primary_rebalance::PartitionStatusOutbox::default(), + partition_directory_retained_at: Cell::new(None), + partition_directory_retain_cursor: Cell::new(None), superblock_wedged_fatal_failures: Cell::new(0), bus_max_message_size: Cell::new(DEFAULT_BUS_MAX_MESSAGE_SIZE), metadata_transfer_attempts: Cell::new(0), @@ -2500,6 +2533,15 @@ (gathered, received == shard_count) } + /// This node's id and the cluster's node count. + #[must_use] + pub const fn replica_topology(&self) -> ReplicaTopology { + ReplicaTopology::new( + self.partition_consensus.self_replica_id, + self.partition_consensus.replica_count, + ) + } + /// Run a partition read on the shard owning `namespace` and await the reply. /// /// Routes a [`LifecycleFrame::PartitionRead`] through the shards table @@ -3110,6 +3152,9 @@ partition_placement_cursor: Cell::new(None), partition_placement_owed: Cell::new(0), primary_rebalance: RefCell::default(), + partition_status_outbox: primary_rebalance::PartitionStatusOutbox::default(), + partition_directory_retained_at: Cell::new(None), + partition_directory_retain_cursor: Cell::new(None), superblock_wedged_fatal_failures: Cell::new(0), bus_max_message_size: Cell::new(DEFAULT_BUS_MAX_MESSAGE_SIZE), metadata_transfer_attempts: Cell::new(0), @@ -4001,9 +4046,18 @@ T: ShardsTable, { self.fence_rebalance_authority(&message); + if !self.partition_frame_source_is_member(&message) { + return; + } match message { MessageBag::PrimaryRebalance(message) => { - self.on_primary_rebalance(*message.header()).await; + let Some(statuses) = + control_suffix_body_verified(&message, message.header().checksum_body) + .and_then(|body| decode_status_entries(body).ok()) + else { + return; + }; + self.on_primary_rebalance(*message.header(), statuses).await; } MessageBag::Request(request) => { // One header read for the pair; `header()` casts on every call. @@ -5243,16 +5297,19 @@ created_view, first_initialization, consumer_offsets_max, + members, } = materialisation; let partitions = self.plane.partitions(); if partitions.contains(&namespace) { return; } - let mut consensus = VsrConsensus::with_clock( + let members = + members.unwrap_or_else(|| ReplicaSet::all(self.partition_consensus.replica_count)); + let mut consensus = VsrConsensus::with_members( self.partition_consensus.cluster_id, self.partition_consensus.self_replica_id, - self.partition_consensus.replica_count, + members, namespace.inner(), self.partition_consensus.bus.clone(), LocalPipeline::new(), @@ -5263,8 +5320,7 @@ // Recovered metadata can precede the group's first materialization; // missing retained history alone cannot distinguish that from data loss. consensus.set_recovery_election_allowed(first_initialization || retained.is_some()); - let restarted = (retained.is_some() || !first_initialization) - && self.partition_consensus.replica_count > 1; + let restarted = (retained.is_some() || !first_initialization) && members.len() > 1; let consensus::FreshGroupStart { join, view_fallback, @@ -5336,7 +5392,7 @@ // Match production's solo reservation handling. A replicated WAL may // retain a suffix the next view discards; anchoring at its dirty offset // would prevent truncation from restoring the earlier append point. - if self.partition_consensus.replica_count == 1 && !partition.durability().is_persisted() { + if members.len() == 1 && !partition.durability().is_persisted() { partition.reanchor_in_memory_to_mint_frontier(partitions.config().segment_size); } partitions.insert(namespace, partition); @@ -5844,7 +5900,7 @@ peer, "adopted a live view while awaiting transfer; requesting partition state transfer" ); - if self.peer_is_known(peer, "StartView") { + if self.member_is_known(partition.consensus().replica_count(), peer, "StartView") { let _ = self .arm_partition_transfer(partition, peer, transfers_inflight) .await; @@ -6138,9 +6194,8 @@ reply.received_op = prefix.received_op; reply.seal(); }); - let _ = self - .bus - .send_to_replica(header.replica, reply.into_generic().into_frozen()) + let _ = consensus + .send_to_member(header.replica, reply.into_generic().into_frozen()) .await; } @@ -6319,7 +6374,10 @@ B: MessageBus, { if !live { - return self.send_repair_prepare(target, entry).await; + let Some(node) = consensus.members().node(target) else { + return false; + }; + return self.send_repair_prepare(node, entry).await; } send_replayed_prepare(consensus, target, op, entry, ReplicationSend::Replay).await } @@ -6366,6 +6424,9 @@ if let Some(ref consensus) = planes.0.consensus && consensus.group() == header.group { + let Some(node) = consensus.members().node(target) else { + return Some(header.from_op.saturating_sub(1)); + }; // A primary-elect can need bodies held only by ViewChange peers. // Other requesters may receive only those peers' committed prefix. if consensus.is_transferring() { @@ -6414,7 +6475,7 @@ self.send_repair_range_reply( cluster, self_id, - target, + node, Command::RangeEvicted, header.nonce, from_op, @@ -6425,7 +6486,7 @@ self.send_repair_range_reply( cluster, self_id, - target, + node, Command::RepairDone, header.nonce, header.from_op.saturating_sub(1), @@ -6442,7 +6503,7 @@ self.send_repair_range_reply( cluster, self_id, - target, + node, Command::RangeEvicted, header.nonce, from_op, @@ -6488,7 +6549,7 @@ self.send_repair_range_reply( cluster, self_id, - target, + node, Command::RepairDone, header.nonce, served_through, @@ -6517,6 +6578,9 @@ { return Some(header.from_op.saturating_sub(1)); } + let Some(node) = partition.consensus().members().node(target) else { + return Some(header.from_op.saturating_sub(1)); + }; let read_key = ServedReadKey::Prepares { group: header.group, target, @@ -6646,7 +6710,7 @@ self.send_repair_range_reply( cluster, self_id, - target, + node, Command::RangeEvicted, header.nonce, retained_from, @@ -6702,7 +6766,7 @@ self.send_repair_range_reply( cluster, self_id, - target, + node, Command::RepairDone, header.nonce, served_through, @@ -7308,7 +7372,11 @@ ); // Same reason as the StartView arm: this id becomes // `session.peer` and later reaches the peer rotation. - if self.peer_is_known(header.replica, "RepairRangeReply") { + if self.member_is_known( + partition.consensus().replica_count(), + header.replica, + "RepairRangeReply", + ) { partition.consensus().begin_state_transfer_await(); let _ = self .arm_partition_transfer( @@ -7441,9 +7509,8 @@ h.size = size_of::<RequestPreparesHeader>() as u32; h.seal(); }); - if self - .bus - .send_to_replica(target, msg.into_generic().into_frozen()) + if consensus + .send_to_member(target, msg.into_generic().into_frozen()) .await .is_err() { @@ -7470,7 +7537,7 @@ /// served-through watermark past a failed send, or the terminating /// `RepairDone` reports ops that were never delivered. #[allow(clippy::future_not_send)] - async fn send_repair_prepare(&self, target: u8, entry: Frozen<MESSAGE_ALIGN>) -> bool + async fn send_repair_prepare(&self, node: u8, entry: Frozen<MESSAGE_ALIGN>) -> bool where B: MessageBus, { @@ -7486,7 +7553,7 @@ return false; }; self.bus - .send_to_replica(target, message.into_frozen()) + .send_to_replica(node, message.into_frozen()) .await .is_ok() } @@ -7497,7 +7564,7 @@ &self, cluster: u128, self_id: u8, - target: u8, + node: u8, command: Command, nonce: u128, op: u64, @@ -7520,7 +7587,7 @@ }); let _ = self .bus - .send_to_replica(target, msg.into_generic().into_frozen()) + .send_to_replica(node, msg.into_generic().into_frozen()) .await; } @@ -8217,9 +8284,8 @@ h.size = size_of::<RequestStateTransferHeader>() as u32; h.seal(); }); - let _ = self - .bus - .send_to_replica(target, msg.into_generic().into_frozen()) + let _ = consensus + .send_to_member(target, msg.into_generic().into_frozen()) .await; } @@ -8236,6 +8302,7 @@ &self, cluster: u128, self_id: u8, + members: ReplicaSet, target: u8, nonce: u128, namespace: u64, @@ -8273,9 +8340,12 @@ } h.seal(); }); + let Some(node) = members.node(target) else { + return; + }; let _ = self .bus - .send_to_replica(target, msg.into_generic().into_frozen()) + .send_to_replica(node, msg.into_generic().into_frozen()) .await; } @@ -8288,6 +8358,7 @@ &self, cluster: u128, self_id: u8, + members: ReplicaSet, target: u8, nonce: u128, namespace: u64, @@ -8312,9 +8383,12 @@ h.size = size_of::<RequestStateChunkHeader>() as u32; h.seal(); }); + let Some(node) = members.node(target) else { + return; + }; let _ = self .bus - .send_to_replica(target, msg.into_generic().into_frozen()) + .send_to_replica(node, msg.into_generic().into_frozen()) .await; } @@ -8377,6 +8451,7 @@ self.send_state_transfer_target( cluster, self_id, + *consensus.members(), header.replica, header.nonce, header.group, @@ -8406,6 +8481,7 @@ self.send_state_transfer_target( cluster, self_id, + *consensus.members(), header.replica, header.nonce, header.group, @@ -8442,6 +8518,7 @@ self.send_state_transfer_target( cluster, self_id, + *consensus.members(), header.replica, header.nonce, header.group, @@ -8685,6 +8762,7 @@ self.send_request_state_chunk( consensus.cluster(), consensus.replica(), + *consensus.members(), peer, nonce, consensus.group(), @@ -8875,6 +8953,7 @@ }; let cluster = consensus.cluster(); let self_id = consensus.replica(); + let members = *consensus.members(); // Never serve a frame the receiving transport will reject: anything past // `max_message_size` tears down the whole replica connection, which reads @@ -8951,10 +9030,12 @@ }; match reply { Some(ChunkReply::Chunk(chunk)) => { - let _ = self - .bus - .send_to_replica(header.replica, chunk.into_generic().into_frozen()) - .await; + if let Some(node) = members.node(header.replica) { + let _ = self + .bus + .send_to_replica(node, chunk.into_generic().into_frozen()) + .await; + } } Some(ChunkReply::Unavailable { transient }) => { tracing::info!( @@ -8966,6 +9047,7 @@ self.send_state_transfer_target( cluster, self_id, + *consensus.members(), header.replica, header.nonce, header.group, @@ -9323,6 +9405,11 @@ let mut repair_ring_entries = 0usize; let mut repair_ring_bytes = 0u64; let mut untrusted_logs = 0usize; + let mut owing_retirement = 0usize; + // The replication gauges are scraped, not driven: once per report + // interval is enough. + let replication_due = + u128::from(self.ticks.get()).is_multiple_of(PARTITION_PLACEMENT_REPORT_INTERVAL_TICKS); for namespace in namespace_scratch.iter() { if let Some(partition) = partitions.get_mut_by_ns(namespace) { partition.expire_transfer_tails( @@ -9331,7 +9418,10 @@ * u64::from(STATE_TRANSFER_OFFER_EXPIRY_MULTIPLE), ); partition.drive_persistence().await; - untrusted_logs += usize::from(partition.consensus().has_untrusted_log()); + let consensus = partition.consensus(); + untrusted_logs += usize::from(consensus.has_untrusted_log()); + owing_retirement += + usize::from(replication_due && partition.owes_session_retirement()); let (entries, bytes) = partition.repair_ring_occupancy(); repair_ring_entries += entries; repair_ring_bytes += bytes; @@ -9359,6 +9449,17 @@ self.metrics .set_repair_ring(repair_ring_entries, repair_ring_bytes); self.metrics.set_partitions_untrusted_log(untrusted_logs); + if replication_due { + // Counted once for the cluster, so a partition whose primary is down + // or stepped down is not left to a node that no longer leads it. + let replication = self.primary_rebalance.borrow().partition_replication(); + self.metrics.set_partition_replication( + replication.under_replicated, + replication.below_write_quorum, + ); + self.metrics + .set_partitions_owing_retirement(owing_retirement); + } // Counted at most ONCE per sweep and only if a re-arm actually fires, // then tracked locally as arms land. Counting per namespace is a full @@ -10105,6 +10206,7 @@ let visited = owed.min(namespaces.len()).min(batch); let partitions = self.plane.partitions(); let mut reports = Vec::with_capacity(visited); + let mut statuses = Vec::new(); for namespace in namespaces[start..] .iter() .chain(&namespaces[..start]) @@ -10126,11 +10228,27 @@ created_revision: partition.created_revision(), view, }); + if consensus.is_primary() { + let stats = &partition.stats; + statuses.push(PartitionStatusEntry { + namespace: namespace.inner(), + created_revision: partition.created_revision(), + view, + commit: consensus.commit_min(), + current_offset: stats.current_offset(), + messages_count: stats.messages_count_inconsistent(), + size_bytes: stats.size_bytes_inconsistent(), + segments_count: stats.segments_count_inconsistent(), + in_sync_nodes: consensus + .in_sync_nodes(consensus::MEMBER_LAG_MAX) + .unwrap_or(0), + }); + } } } if !reports.is_empty() && let Err(error) = sender.try_send(ShardFrame::lifecycle( - LifecycleFrame::PartitionPlacementViews(reports), + LifecycleFrame::PartitionPlacementViews(reports, statuses), )) { self.metrics.record_frame_drop( @@ -10304,12 +10422,6 @@ ) where B: MessageBus, { - // Before anything keyed by the requester: the offer map's documented - // bound is the replica count, and an unvalidated id makes it 256 entries - // per served group, each pinning an offer for its full expiry. - if !self.peer_is_known(header.replica, "RequestStateTransfer") { - return; - } let planes = self.plane.inner(); let config = planes.1.0.config().clone(); let Some(partition) = planes @@ -10326,6 +10438,16 @@ ) { return; } + // Before anything keyed by the requester: an unvalidated id makes the + // offer map 256 entries per served group, each pinning an offer for its + // full expiry. + if !self.member_is_known( + partition.consensus().replica_count(), + header.replica, + "RequestStateTransfer", + ) { + return; + } // Only a request that passed the gates replaces a parked one. A re-send // keeps the wait that its nonce already spent parked, so re-sends // cannot hold the build slot past the bound. @@ -10380,6 +10502,7 @@ self.send_state_transfer_target( cluster, self_id, + *partition.consensus().members(), header.replica, header.nonce, header.group, @@ -10486,6 +10609,7 @@ self.send_state_transfer_target( cluster, self_id, + *partition.consensus().members(), header.replica, header.nonce, header.group, @@ -10511,6 +10635,7 @@ self.send_state_transfer_target( cluster, self_id, + *partition.consensus().members(), header.replica, header.nonce, header.group, @@ -10566,10 +10691,6 @@ }, } - // The requester id keys the offer map, whose bound is the replica count. - if !self.peer_is_known(header.replica, "RequestStateChunk") { - return; - } let read_key = ServedReadKey::Chunk { group: header.group, requester: header.replica, @@ -10588,8 +10709,17 @@ ) { return; } + // The requester id keys the offer map, whose bound is the group size. + if !self.member_is_known( + partition.consensus().replica_count(), + header.replica, + "RequestStateChunk", + ) { + return; + } let cluster = partition.consensus().cluster(); let self_id = partition.consensus().replica(); + let members = *partition.consensus().members(); let serving_view = partition.consensus().view(); let serving_commit_max = partition.consensus().commit_max(); let chunk_len_max = self.state_chunk_len_max(); @@ -10780,10 +10910,12 @@ }; match reply { Some(ChunkReply::Chunk(chunk)) => { - let _ = self - .bus - .send_to_replica(header.replica, chunk.into_generic().into_frozen()) - .await; + if let Some(node) = members.node(header.replica) { + let _ = self + .bus + .send_to_replica(node, chunk.into_generic().into_frozen()) + .await; + } } Some(ChunkReply::Unavailable { transient }) => { tracing::info!( @@ -10796,6 +10928,7 @@ self.send_state_transfer_target( cluster, self_id, + *partition.consensus().members(), header.replica, header.nonce, header.group, @@ -10915,15 +11048,8 @@ .evict_namespace(namespace); } - /// Whether a peer-supplied source replica id names a replica of this - /// cluster. - /// - /// `header.replica` arrives unvalidated on every frame, and the partition - /// transfer paths turn it into ring arithmetic ([`next_transfer_peer`], where - /// id 255 panics in debug and wraps to replica 0 in release -- a silent - /// retarget) and into the served-offer map key, whose documented bound is the - /// replica count rather than 256 entries per served group. One check at the - /// frame's ingress closes both. + /// Whether a peer-supplied source node id names a node of this cluster. + /// Partition frames carry member indices instead; see [`Self::member_is_known`]. fn peer_is_known(&self, replica: u8, frame: &'static str) -> bool { let replica_count = self.partition_consensus.replica_count; if replica < replica_count { @@ -10934,7 +11060,46 @@ frame, replica, replica_count, - "dropping a partition frame whose source replica is outside this cluster" + "dropping a frame whose source replica is outside this cluster" + ); + false + } + + /// Whether a partition-group frame names a sender inside its group, as far + /// as this shard knows the group. Every consensus handler indexes per-member + /// state by that field. + fn partition_frame_source_is_member(&self, message: &MessageBag) -> bool { + let Some(replica) = message.partition_source_replica() else { + return true; + }; + let (_, group) = message.routing(); + let Some(group_size) = self + .plane + .partitions() + .get_by_ns(&IggyNamespace::from_raw(group)) + .map(|partition| partition.consensus().replica_count()) + else { + return true; + }; + self.member_is_known(group_size, replica, "partition consensus frame") + } + + /// Whether a peer-supplied member index names a replica of a partition + /// group of `group_size` members. + /// + /// `header.replica` arrives unvalidated, and the transfer paths turn it into + /// ring arithmetic ([`next_transfer_peer`]) and into the served-offer map + /// key, whose bound is the group size. + fn member_is_known(&self, group_size: u8, replica: u8, frame: &'static str) -> bool { + if replica < group_size { + return true; + } + tracing::warn!( + shard = self.id, + frame, + replica, + group_size, + "dropping a partition frame whose source replica is outside its group" ); false } @@ -11455,11 +11620,6 @@ M: StreamsFrontend, { let header = *msg.header(); - // The peer id reaches `next_transfer_peer`'s ring arithmetic through the - // re-arm below, so it is validated before anything uses it. - if !self.peer_is_known(header.replica, "StateTransferTarget") { - return; - } let planes = self.plane.inner(); let Some(partition) = planes .1 @@ -11475,6 +11635,15 @@ ) { return; } + // The peer id reaches `next_transfer_peer`'s ring arithmetic through the + // re-arm below, so it is validated before anything uses it. + if !self.member_is_known( + partition.consensus().replica_count(), + header.replica, + "StateTransferTarget", + ) { + return; + } if partition.history_is_busy() { partition.defer_view_transition(MessageBag::StateTransferTarget(msg.clone())); return; @@ -12247,12 +12416,17 @@ }); let consensus_ids = { let consensus = partition.consensus(); - (consensus.cluster(), consensus.replica()) + ( + consensus.cluster(), + consensus.replica(), + *consensus.members(), + ) }; if let Some((nonce, peer, artifact, offset, len)) = request { self.send_request_state_chunk( consensus_ids.0, consensus_ids.1, + consensus_ids.2, peer, nonce, namespace, @@ -14441,13 +14615,12 @@ { use std::mem::size_of; - let bus = consensus.message_bus(); let self_id = consensus.replica(); let cluster = consensus.cluster(); let replica_count = consensus.replica_count(); let send = |target: u8, msg: Frozen<MESSAGE_ALIGN>| async move { - if let Err(e) = bus.send_to_replica(target, msg).await { + if let Err(e) = consensus.send_to_member(target, msg).await { tracing::debug!(replica = self_id, target, "bus send failed: {e}"); } }; @@ -14779,10 +14952,15 @@ } /// Wire actions a partition awaiting a state transfer may still send: its -/// probe, and the empty vote that vouches for no history but lets a group -/// whose every log is untrusted start a view. +/// probe, the empty vote that vouches for no history but lets a group whose +/// every log is untrusted start a view, and the reply to a primary's heartbeat +/// in the view it adopted. Without that reply a two-member primary loses its +/// heartbeat quorum and steps down before it can serve the transfer. fn sendable_while_awaiting_transfer(action: &VsrAction) -> bool { - matches!(action, VsrAction::SendRequestStartView { .. }) || action.is_empty_vote() + matches!( + action, + VsrAction::SendRequestStartView { .. } | VsrAction::SendCommitReply { .. } + ) || action.is_empty_vote() } #[allow( @@ -18102,7 +18280,7 @@ } fn receive(inbox: &Receiver<ShardFrame>) -> Vec<PartitionPlacementView> { - let ShardFrame::Lifecycle(LifecycleFrame::PartitionPlacementViews(reports)) = + let ShardFrame::Lifecycle(LifecycleFrame::PartitionPlacementViews(reports, _)) = inbox.try_recv().expect("placement report queued") else { panic!("expected partition placement observations");
diff --git a/core/shard/src/metrics.rs b/core/shard/src/metrics.rs index d536227..2ae8589 100644 --- a/core/shard/src/metrics.rs +++ b/core/shard/src/metrics.rs
@@ -364,6 +364,10 @@ partition_transfer_refusals_total: Counter, partition_applied_divergences_total: Counter, partitions_untrusted_log: Gauge, + partitions_under_replicated: Gauge, + partitions_below_write_quorum: Gauge, + partitions_owing_retirement: Gauge, + retirement_debts_paid_total: Counter, partition_frames_rejected_stale_total: Counter, partition_frames_rejected_ahead_total: Counter, partition_frames_incarnation_mismatch_total: Family<FrameCommandLabel, Counter>, @@ -456,6 +460,10 @@ partition_transfer_refusals_total: Counter::default(), partition_applied_divergences_total: Counter::default(), partitions_untrusted_log: Gauge::default(), + partitions_under_replicated: Gauge::default(), + partitions_below_write_quorum: Gauge::default(), + partitions_owing_retirement: Gauge::default(), + retirement_debts_paid_total: Counter::default(), partition_frames_rejected_stale_total: Counter::default(), partition_frames_rejected_ahead_total: Counter::default(), partition_frames_incarnation_mismatch_total: Family::default(), @@ -845,6 +853,42 @@ self.partitions_untrusted_log.get() } + /// Republished by every partition sweep, over the partitions this shard + /// leads: those with a member out of sync, and those with too few members + /// in sync to commit. The metadata primary's first shard adds the + /// partitions whose primary is down among too many members to commit. + pub fn set_partition_replication(&self, under_replicated: usize, below_write_quorum: usize) { + self.partitions_under_replicated + .set(i64::try_from(under_replicated).unwrap_or(i64::MAX)); + self.partitions_below_write_quorum + .set(i64::try_from(below_write_quorum).unwrap_or(i64::MAX)); + } + + /// Republished with the replication gauges: partitions on this shard that + /// refuse sessions until a retirement debt payment commits. + pub fn set_partitions_owing_retirement(&self, owing: usize) { + self.partitions_owing_retirement + .set(i64::try_from(owing).unwrap_or(i64::MAX)); + } + + pub fn record_retirement_debt_paid(&self) { + self.retirement_debts_paid_total.inc(); + } + + /// Test-only read, mirroring the siblings. + #[cfg(any(test, feature = "simulator"))] + #[must_use] + pub fn partitions_under_replicated_value(&self) -> i64 { + self.partitions_under_replicated.get() + } + + /// Test-only read, mirroring the siblings. + #[cfg(any(test, feature = "simulator"))] + #[must_use] + pub fn partitions_below_write_quorum_value(&self) -> i64 { + self.partitions_below_write_quorum.get() + } + /// Bumped when a parked partition frame is answered instead of served /// because it was addressed to an incarnation this shard no longer holds /// (delete + recreate recycled the namespace's slab keys). Serving it would @@ -1199,6 +1243,30 @@ self.partitions_untrusted_log.clone(), ); registry.register( + "partitions_under_replicated", + "partitions this shard leads with a member it cannot reach or that has trailed \ + the commit point for longer than the lag window", + self.partitions_under_replicated.clone(), + ); + registry.register( + "partitions_below_write_quorum", + "partitions this shard leads with too few members in sync to commit a write; \ + on the metadata primary's first shard also partitions with too few members up \ + whose primary is down", + self.partitions_below_write_quorum.clone(), + ); + registry.register( + "partitions_owing_retirement", + "partitions on this shard refusing sessions until they retire sessions that \ + finalized while the partition lacked a quorum of its members", + self.partitions_owing_retirement.clone(), + ); + registry.register( + "retirement_debts_paid", + "retirement debts this node paid as a partition primary", + self.retirement_debts_paid_total.clone(), + ); + registry.register( "partition_frames_rejected_stale", "parked frames rejected for a dead incarnation; non-zero is a caught anomaly", self.partition_frames_rejected_stale_total.clone(),
diff --git a/core/shard/src/partition_io.rs b/core/shard/src/partition_io.rs index d920751..9ee9a81 100644 --- a/core/shard/src/partition_io.rs +++ b/core/shard/src/partition_io.rs
@@ -1028,6 +1028,7 @@ .send_request_state_chunk( consensus.cluster(), consensus.replica(), + *consensus.members(), PEER, 1, group, @@ -1045,6 +1046,66 @@ } #[compio::test] + async fn given_a_prepare_ok_from_outside_the_group_when_dispatched_should_drop_it() { + const OUTSIDER: u8 = 4; + let bus = Rc::new(IggyMessageBus::new(0)); + let (owner, _sender) = test_owner(&bus, None); + let namespace = IggyNamespace::new(0, 0, 0); + let partitions = owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + let consensus = VsrConsensus::new( + 1, + 0, + 2, + namespace.inner(), + Rc::clone(&bus), + LocalPipeline::new(), + ); + consensus.init(); + partitions.insert( + namespace, + IggyPartition::with_in_memory_storage( + Arc::new(PartitionStats::default()), + consensus, + IggyByteSize::from(SEGMENT_BYTES), + ), + ); + partitions + .on_request_with_reply(send_request(namespace, 1), None) + .await; + let prepare = partitions + .get_by_ns(&namespace) + .unwrap() + .consensus() + .pipeline_head_header() + .expect("the write is pipelined"); + let ack = Message::<PrepareOkHeader>::new(size_of::<PrepareOkHeader>()).transmute_header( + |_, ack: &mut PrepareOkHeader| { + ack.command = Command::PrepareOk; + ack.cluster = 1; + ack.replica = OUTSIDER; + ack.view = prepare.view; + ack.op = prepare.op; + ack.prepare_checksum = prepare.checksum; + ack.operation = prepare.operation; + ack.group = prepare.group; + ack.size = u32::try_from(size_of::<PrepareOkHeader>()).unwrap(); + ack.seal(); + }, + ); + owner.on_message(MessageBag::PrepareOk(ack)).await; + assert_eq!( + partitions + .get_by_ns(&namespace) + .unwrap() + .consensus() + .pipeline_head_header(), + Some(prepare), + "an ack from a replica outside the group must not count" + ); + } + + #[compio::test] async fn given_a_partition_transfer_offer_when_serving_a_chunk_should_stamp_the_hosted_incarnation() { const INCARNATION: u64 = 7; @@ -1064,6 +1125,25 @@ let namespace = IggyNamespace::new(0, 0, 0); let directory = tempfile::tempdir().unwrap(); let partitions = owner.plane.partitions(); + // The requester has to be a member of the group it pulls from. + partitions.remove(&namespace).unwrap(); + let consensus = VsrConsensus::new( + 1, + 0, + 2, + namespace.inner(), + Rc::clone(&bus), + LocalPipeline::new(), + ); + consensus.init(); + partitions.insert( + namespace, + IggyPartition::with_in_memory_storage( + Arc::new(PartitionStats::default()), + consensus, + IggyByteSize::from(SEGMENT_BYTES), + ), + ); let partition = partitions.get_mut_by_ns(&namespace).unwrap(); partition.set_created_revision(INCARNATION); attach_segment_files(partition, directory.path(), false).await; @@ -1077,6 +1157,30 @@ owner .process_loopback(&mut crate::router::LoopbackRound::default()) .await; + let prepare = partitions + .get_by_ns(&namespace) + .unwrap() + .consensus() + .pipeline_head_header() + .expect("the write is pipelined"); + let ack = Message::<PrepareOkHeader>::new(size_of::<PrepareOkHeader>()).transmute_header( + |_, ack: &mut PrepareOkHeader| { + ack.command = Command::PrepareOk; + ack.cluster = 1; + ack.replica = REQUESTER; + ack.view = prepare.view; + ack.op = prepare.op; + ack.prepare_checksum = prepare.checksum; + ack.operation = prepare.operation; + ack.group = prepare.group; + ack.size = u32::try_from(size_of::<PrepareOkHeader>()).unwrap(); + ack.seal(); + }, + ); + owner.on_message(MessageBag::PrepareOk(ack)).await; + owner + .process_loopback(&mut crate::router::LoopbackRound::default()) + .await; assert_eq!(replied.await.unwrap().header().status, 0); let request = Message::<RequestStateTransferHeader>::new(size_of::<RequestStateTransferHeader>()) @@ -1145,25 +1249,14 @@ let namespace = IggyNamespace::new(0, 0, 0); let directory = tempfile::tempdir().unwrap(); let partitions = owner.plane.partitions(); + serve_to_requester(&owner, &bus, namespace); let partition = partitions.get_mut_by_ns(&namespace).unwrap(); - partition.set_created_revision(SERVED_INCARNATION); attach_segment_files(partition, directory.path(), false).await; let journal = &partition.log.journal().inner; journal.set_repair_retention(true); journal.set_ring_caps(1, u64::MAX); for op in 1..=OPS { - let request = send_request(namespace, op).transmute_header( - |old, header: &mut RoutedRequestHeader| { - *header = old; - header.partition_incarnation = SERVED_INCARNATION; - }, - ); - let (reply, replied) = consensus::oneshot_channel(); - partitions.on_request_with_reply(request, Some(reply)).await; - owner - .process_loopback(&mut crate::router::LoopbackRound::default()) - .await; - assert_eq!(replied.await.unwrap().header().status, 0); + commit_acked_write(&owner, namespace, op).await; if op == 1 { owner .on_message(request_state_transfer(namespace, NONCE, SERVED_INCARNATION)) @@ -1243,12 +1336,7 @@ let (mut owner, _sender) = test_owner(&bus, None); owner.partition_consensus.replica_count = TRANSFER_REQUESTER + 1; let namespace = IggyNamespace::new(0, 0, 0); - owner - .plane - .partitions() - .get_mut_by_ns(&namespace) - .unwrap() - .set_created_revision(SERVED_INCARNATION); + serve_to_requester(&owner, &bus, namespace); let directory = tempfile::tempdir().unwrap(); let log_path = directory.path().join("segment.log"); let mut bytes = [PREFIX, SUFFIX].concat(); @@ -1341,12 +1429,7 @@ let (mut owner, _sender) = test_owner(&bus, None); owner.partition_consensus.replica_count = TRANSFER_REQUESTER + 1; let namespace = IggyNamespace::new(0, 0, 0); - owner - .plane - .partitions() - .get_mut_by_ns(&namespace) - .unwrap() - .set_created_revision(SERVED_INCARNATION); + serve_to_requester(&owner, &bus, namespace); let directory = tempfile::tempdir().unwrap(); let offer_of = |name: &str, bytes: &[u8]| { let log_path = directory.path().join(name); @@ -11020,6 +11103,73 @@ /// The replica that asks a [`serving_shard`] for every transfer. const TRANSFER_REQUESTER: u8 = 1; + /// Rebuilds `namespace` at [`SERVED_INCARNATION`] as a two-member group + /// with [`TRANSFER_REQUESTER`] as its backup: a requester has to be a member + /// of the group it pulls from. + fn serve_to_requester(owner: &IoTestShard, bus: &Rc<IggyMessageBus>, namespace: IggyNamespace) { + let partitions = owner.plane.partitions(); + partitions.remove(&namespace).unwrap(); + let consensus = VsrConsensus::new( + 1, + 0, + TRANSFER_REQUESTER + 1, + namespace.inner(), + Rc::clone(bus), + LocalPipeline::new(), + ); + consensus.init(); + let mut partition = IggyPartition::with_in_memory_storage( + Arc::new(PartitionStats::default()), + consensus, + IggyByteSize::from(SEGMENT_BYTES), + ); + partition.set_created_revision(SERVED_INCARNATION); + partitions.insert(namespace, partition); + } + + /// Commits write `op` of a [`serve_to_requester`] group once + /// [`TRANSFER_REQUESTER`] acks it. + #[allow(clippy::future_not_send)] + async fn commit_acked_write(owner: &IoTestShard, namespace: IggyNamespace, op: u64) { + let partitions = owner.plane.partitions(); + let request = send_request(namespace, op).transmute_header( + |old, header: &mut RoutedRequestHeader| { + *header = old; + header.partition_incarnation = SERVED_INCARNATION; + }, + ); + let (reply, replied) = consensus::oneshot_channel(); + partitions.on_request_with_reply(request, Some(reply)).await; + owner + .process_loopback(&mut crate::router::LoopbackRound::default()) + .await; + let prepare = partitions + .get_by_ns(&namespace) + .unwrap() + .consensus() + .pipeline_head_header() + .expect("the write is pipelined"); + let ack = Message::<PrepareOkHeader>::new(size_of::<PrepareOkHeader>()).transmute_header( + |_, ack: &mut PrepareOkHeader| { + ack.command = Command::PrepareOk; + ack.cluster = 1; + ack.replica = TRANSFER_REQUESTER; + ack.view = prepare.view; + ack.op = prepare.op; + ack.prepare_checksum = prepare.checksum; + ack.operation = prepare.operation; + ack.group = prepare.group; + ack.size = u32::try_from(size_of::<PrepareOkHeader>()).unwrap(); + ack.seal(); + }, + ); + owner.on_message(MessageBag::PrepareOk(ack)).await; + owner + .process_loopback(&mut crate::router::LoopbackRound::default()) + .await; + assert_eq!(replied.await.unwrap().header().status, 0); + } + /// A shard whose primaries of `namespaces` each hold one committed op in /// segment files under `directory`. Its I/O owner pins their transfer /// plans, so a transfer request parks until an I/O turn pins its plan. @@ -11036,21 +11186,10 @@ for namespace in namespaces { let directory = directory.join(namespace.inner().to_string()); std::fs::create_dir(&directory).unwrap(); + serve_to_requester(&owner, bus, *namespace); let partition = partitions.get_mut_by_ns(namespace).unwrap(); - partition.set_created_revision(SERVED_INCARNATION); attach_segment_files(partition, &directory, false).await; - let request = send_request(*namespace, 1).transmute_header( - |old, header: &mut RoutedRequestHeader| { - *header = old; - header.partition_incarnation = SERVED_INCARNATION; - }, - ); - let (reply, replied) = consensus::oneshot_channel(); - partitions.on_request_with_reply(request, Some(reply)).await; - owner - .process_loopback(&mut crate::router::LoopbackRound::default()) - .await; - assert_eq!(replied.await.unwrap().header().status, 0); + commit_acked_write(&owner, *namespace, 1).await; } partitions.set_io_notifier( owner.partition_io.notifier(),
diff --git a/core/shard/src/poll.rs b/core/shard/src/poll.rs index 3dd6f94..5ab85b6 100644 --- a/core/shard/src/poll.rs +++ b/core/shard/src/poll.rs
@@ -243,6 +243,19 @@ ) }) .unwrap_or(PartitionReadReply::NotFound), + PartitionRead::RetirementDebt { + created_revision, + debt_op, + } => partitions + .with_partition(&namespace, |partition| { + (partition.created_revision() == created_revision).then(|| { + PartitionReadReply::RetirementDebt( + partition.observe_retirement_debt(debt_op), + ) + }) + }) + .flatten() + .unwrap_or(PartitionReadReply::NotFound), PartitionRead::DeferredPoll(_) => unreachable!("deferred admission handled above"), // Answered above, entry by entry. PartitionRead::ConsumerOffsetsOnPrimary { .. } => { @@ -259,7 +272,7 @@ { PartitionReadReply::Primary { view, - primary: consensus.primary_index(view), + primary: consensus.primary_node(view), } } else { PartitionReadReply::Rejected(IggyError::TransientNotAccepted)
diff --git a/core/shard/src/primary_rebalance.rs b/core/shard/src/primary_rebalance.rs index 3f503be..7e1046d 100644 --- a/core/shard/src/primary_rebalance.rs +++ b/core/shard/src/primary_rebalance.rs
@@ -32,6 +32,9 @@ use consensus::{ Consensus, MetadataHandle, PartitionsHandle, QUORUM_REPLICATION_MAX, VsrConsensus, }; +use iggy_binary_protocol::primitives::partition_status::{ + PartitionStatusEntry, encode_status_entries, +}; use iggy_binary_protocol::{ Command, ConsensusHeader, HEADER_SIZE, PrepareHeader, PrimaryRebalanceHeader, PrimaryRebalanceKind, @@ -40,11 +43,15 @@ use journal::{Journal, JournalHandle}; use message_bus::MessageBus; use metadata::impls::metadata::PartitionPrimaryTransfer; +use metadata::stm::stream::{PartitionReplication, UnreportedPartitions}; use partitions::{IggyPartition, PreparedPrimaryTransfer}; use server_common::Message; use server_common::MessageBag; use server_common::sharding::IggyNamespace; use server_common::sharding::METADATA_GROUP; +use std::cell::{Cell, RefCell}; +use std::collections::BTreeMap; +use std::ops::Bound; use std::time::{Duration, Instant}; pub const PROBE_INTERVAL: Duration = Duration::from_secs(1); @@ -61,6 +68,63 @@ /// Caps the wait after repeated failed moves, so a target that recovered gets /// primaries back within minutes. const TARGET_BACKOFF_MAX: Duration = Duration::from_secs(240); +/// Partition reports one metadata probe or reply carries. +const STATUS_ENTRIES_PER_FRAME: usize = 512; +/// Directory reports one probe batch visits while skipping those of deleted +/// incarnations. +const STATUS_ENTRIES_SCANNED_PER_FRAME: usize = 4 * STATUS_ENTRIES_PER_FRAME; +/// Directory reports one retain pass checks under the directory lock. +const DIRECTORY_RETAIN_CHUNK: usize = 1024; +/// An outbox report not refreshed for this long names a partition this node +/// no longer leads. Each led partition is reported once per probe interval. +const OUTBOX_REPORT_MAX_AGE: Duration = Duration::from_secs(2 * PROBE_INTERVAL.as_secs()); + +/// Shard 0: this node's own partition reports awaiting metadata replies, with +/// when each was last refreshed. +#[derive(Debug, Default)] +pub struct PartitionStatusOutbox { + entries: RefCell<BTreeMap<u64, (PartitionStatusEntry, Instant)>>, + cursor: Cell<Option<u64>>, +} + +impl PartitionStatusOutbox { + pub fn record(&self, statuses: &[PartitionStatusEntry], now: Instant) { + let mut entries = self.entries.borrow_mut(); + for status in statuses { + entries.insert(status.namespace, (*status, now)); + } + } + + /// Drop reports no publish refreshed within [`OUTBOX_REPORT_MAX_AGE`]. + pub fn evict_stale(&self, now: Instant) { + self.entries.borrow_mut().retain(|_, (_, refreshed)| { + now.saturating_duration_since(*refreshed) < OUTBOX_REPORT_MAX_AGE + }); + } + + /// Up to `limit` reports after the cursor, wrapping, advancing it. + pub fn next_batch(&self, limit: usize) -> Vec<PartitionStatusEntry> { + let entries = self.entries.borrow(); + let after = self.cursor.get(); + let start = after.map_or(Bound::Unbounded, Bound::Excluded); + let end = after.map_or(Bound::Excluded(0), Bound::Included); + let batch: Vec<_> = entries + .range((start, Bound::Unbounded)) + .chain(entries.range((Bound::Unbounded, end))) + .take(limit.min(entries.len())) + .map(|(_, (status, _))| *status) + .collect(); + if let Some(last) = batch.last() { + self.cursor.set(Some(last.namespace)); + } + batch + } + + /// The partitions this node leads, as its shards last reported them. + pub fn namespaces(&self) -> Vec<u64> { + self.entries.borrow().keys().copied().collect() + } +} #[cfg(test)] mod tests; @@ -69,6 +133,18 @@ view: Option<u32>, peers: Vec<Peer>, eligible: Vec<bool>, + /// Every peer was heard from in this view, so placement got `eligible` + /// rather than no evidence. + readiness_known: bool, + /// The cluster's partitions short of their factor or of a write quorum, + /// as of the last scan. + replication: PartitionReplication, + /// The nodes the directory last narrowed in-sync reports to. + live_nodes: u32, + unreported: UnreportedPartitions, + /// When this node started coordinating `view`. + view_since: Instant, + next_quorum_scan: Instant, stable_since: Instant, next_move: Instant, cursor: Option<IggyNamespace>, @@ -119,10 +195,26 @@ nonce: Option<u128>, sent_at: Option<Instant>, ready_at: Option<Instant>, + /// When the peer last answered a probe of this view, ready or not. + live_at: Option<Instant>, answered_at: Option<Instant>, + /// The peer answered a probe of this view, or one timed out or never left + /// this node, so `ready_at` reflects the peer rather than the view's start. + heard: bool, /// The peer answered the newest probe that left this node, so it accepts no /// older probe after the next one. replied: bool, + /// The last partition report sent to this peer. + status_cursor: Option<u64>, + /// The last directory change sent to this peer. + changes_sent: u64, +} + +/// Where a peer's reports stood before a probe that may not leave this node. +#[derive(Debug, Clone, Copy, Default)] +struct StatusCursors { + status_cursor: Option<u64>, + changes_sent: u64, } struct PendingMove { @@ -152,33 +244,48 @@ } impl PrimaryRebalance { - fn observe_peer(&mut self, header: &PrimaryRebalanceHeader, now: Instant) { + /// On the metadata primary's first shard, the cluster's partitions short + /// of their factor or of a write quorum. Zero elsewhere, and until every + /// peer was heard from in this view. + pub(crate) const fn partition_replication(&self) -> PartitionReplication { + self.replication + } + + /// Whether `header` answers the peer's outstanding probe. + fn observe_peer(&mut self, header: &PrimaryRebalanceHeader, now: Instant) -> bool { if self.view != Some(header.view) { - return; + return false; } let Some(peer) = self.peers.get_mut(usize::from(header.replica)) else { - return; + return false; }; if peer.nonce != Some(header.nonce) || peer .sent_at .is_none_or(|sent| now.duration_since(sent) > PROBE_TIMEOUT) { - return; + return false; } peer.nonce = None; peer.answered_at = peer.sent_at; peer.ready_at = (header.ready == 1).then_some(now); + peer.live_at = Some(now); + peer.heard = true; peer.replied = true; + true } /// A probe the bus refused never left this node, so nothing can answer it. - /// The peer is probed again at the regular cadence, not after the timeout. - fn forget_unsent(&mut self, replica: u8, nonce: u128) { - if let Some(peer) = self.peers.get_mut(usize::from(replica)) - && peer.nonce == Some(nonce) - { - peer.nonce = None; + /// The peer is probed again at the regular cadence, not after the timeout, + /// and gets the reports that probe carried in the next one. + fn forget_unsent(&mut self, replica: u8, nonce: u128, unsent: StatusCursors) { + if let Some(peer) = self.peers.get_mut(usize::from(replica)) { + if peer.nonce == Some(nonce) { + peer.nonce = None; + peer.heard = true; + } + peer.status_cursor = unsent.status_cursor; + peer.changes_sent = unsent.changes_sent; } } @@ -266,6 +373,12 @@ view: None, peers: Vec::new(), eligible: Vec::new(), + readiness_known: false, + replication: PartitionReplication::default(), + live_nodes: u32::MAX, + unreported: UnreportedPartitions::default(), + view_since: Instant::now(), + next_quorum_scan: Instant::now(), stable_since: Instant::now(), next_move: Instant::now(), cursor: None, @@ -319,9 +432,22 @@ let Some(consensus) = metadata.consensus.as_ref() else { return; }; + if self.id == 0 { + self.retain_partition_directories(self.bus.monotonic_now()); + } if self.id != 0 || !consensus.is_primary() || !consensus_ready(consensus) { - self.primary_rebalance.borrow_mut().reset_authority(); + let mut state = self.primary_rebalance.borrow_mut(); + state.reset_authority(); + state.live_nodes = u32::MAX; + state.replication = PartitionReplication::default(); + drop(state); metadata.observe_replica_readiness(&[]); + if self.id == 0 { + metadata + .mux_stm + .streams() + .set_partition_directory_ready_nodes(u32::MAX); + } return; } let mut outgoing = Vec::new(); @@ -331,18 +457,31 @@ let mut state = self.primary_rebalance.borrow_mut(); let replicas = usize::from(consensus.replica_count()); if state.view != Some(consensus.view()) { + let streams = metadata.mux_stm.streams(); + let changes_sent = streams.partition_status_last_change(); + // The failover that made this node metadata primary can also + // have given it partitions, and its shards may have reported + // them before this view started. No metadata primary relayed + // those, so this node's own reports count as changed after it. + streams.mark_partition_statuses_changed(&self.partition_status_outbox.namespaces()); *state = PrimaryRebalance { view: Some(consensus.view()), - // Placement keeps a peer eligible until it answers not - // ready or its first probe times out. + // A partition's members never move, so placement waits to + // hear from every peer rather than presume the node whose + // failure started this view is ready. + // Reports from before this view went out under the last + // metadata primary or come round in the cycle. The changes + // a failover makes come after it, so they go first. peers: vec![ Peer { - ready_at: Some(now), + changes_sent, ..Peer::default() }; replicas ], eligible: vec![false; replicas], + view_since: now, + next_quorum_scan: now, backoffs: vec![TargetBackoff::default(); replicas], stable_since: now, next_move: now, @@ -372,6 +511,9 @@ let elapsed = peer.sent_at.map_or(PROBE_TIMEOUT + PROBE_INTERVAL, |sent| { now.duration_since(sent) }); + if peer.nonce.is_some() && elapsed > PROBE_TIMEOUT { + peer.heard = true; + } // A backup that answered a probe sent before admission opened // would otherwise wait a full interval for its own credit. The // re-probe is stamped at or after the opening, so it runs once. @@ -390,10 +532,60 @@ peer.sent_at = Some(now); outgoing.push((replica, probe_header(consensus, nonce))); } + let readiness_known = state + .peers + .iter() + .enumerate() + .all(|(index, peer)| index == usize::from(consensus.replica()) || peer.heard); + // Placement readiness drops a node while any of its partitions + // repairs, which says nothing about whether it holds the others. + // Liveness is whether it answers at all. + let live: Vec<bool> = (0..consensus.replica_count()) + .map(|replica| { + replica == consensus.replica() + || state.peers[usize::from(replica)] + .live_at + .is_some_and(|live| now.duration_since(live) <= PROBE_TIMEOUT) + }) + .collect(); + let live_nodes = if readiness_known { + node_mask(&live) + } else { + u32::MAX + }; if changed { state.stable_since = now; state.pending = None; - metadata.observe_replica_readiness(&state.eligible); + } + if changed || readiness_known != state.readiness_known { + state.readiness_known = readiness_known; + metadata.observe_replica_readiness(if readiness_known { + &state.eligible + } else { + &[] + }); + } + if live_nodes != state.live_nodes { + state.live_nodes = live_nodes; + metadata + .mux_stm + .streams() + .set_partition_directory_ready_nodes(live_nodes); + } + if readiness_known && now >= state.next_quorum_scan { + state.next_quorum_scan = now + PROBE_INTERVAL; + let streams = metadata.mux_stm.streams(); + // Reports from before this view came through the last metadata + // primary, so they get as long as this view's own to come round. + let stale_before = now + .checked_sub(report_stale_after(streams.partition_status_count())) + .filter(|cutoff| *cutoff >= state.view_since); + let PrimaryRebalance { + replication, + unreported, + .. + } = &mut *state; + *replication = streams.partition_replication(&live, now, stale_before, unreported); } if let Some(pending) = state.pending.as_ref() { let elapsed = now.duration_since(pending.since); @@ -436,7 +628,9 @@ ) else { return false; }; - if consensus.primary_index(current.view) != target { + if self.partition_primary_node(moved.plan.namespace.inner(), current.view) + != Some(target) + { backoff.record_failure(now); return false; } @@ -484,21 +678,115 @@ } } } - if let Some(probe) = capture { - self.send_rebalance(consensus.primary_index(probe.partition_view), probe) - .await; + if let Some(probe) = capture + && let Some(source) = self.partition_primary_node(probe.namespace, probe.partition_view) + { + self.send_rebalance(source, probe, &[]).await; } for (replica, probe) in outgoing { - if !self.send_rebalance(replica, probe).await { + let (statuses, unsent) = self.next_status_batch(replica); + if !self.send_rebalance(replica, probe, &statuses).await { self.primary_rebalance .borrow_mut() - .forget_unsent(replica, probe.nonce); + .forget_unsent(replica, probe.nonce, unsent); } } } + /// The next run of directory reports for `peer`, advancing its cursors, + /// with the cursors from before it for a send that fails. + fn next_status_batch(&self, peer: u8) -> (Vec<PartitionStatusEntry>, StatusCursors) { + let mut state = self.primary_rebalance.borrow_mut(); + let Some(cursor) = state.peers.get_mut(usize::from(peer)) else { + return (Vec::new(), StatusCursors::default()); + }; + let before = StatusCursors { + status_cursor: cursor.status_cursor, + changes_sent: cursor.changes_sent, + }; + let streams = self.plane.metadata().mux_stm.streams(); + // Changes first: a new primary reaches every node within a probe, not + // once the cycle through every partition comes back to it. + let (mut batch, changes_sent) = + streams.partition_status_changes(cursor.changes_sent, STATUS_ENTRIES_PER_FRAME); + cursor.changes_sent = changes_sent; + let room = STATUS_ENTRIES_PER_FRAME - batch.len(); + if room > 0 { + let (cycle, last) = streams.partition_status_batch( + cursor.status_cursor, + room, + STATUS_ENTRIES_SCANNED_PER_FRAME, + ); + cursor.status_cursor = last; + batch.extend(cycle); + } + (batch, before) + } + + /// Node id of the primary `view` assigns in the committed partition behind + /// `namespace`. + fn partition_primary_node(&self, namespace: u64, view: u32) -> Option<u8> { + let nodes = self.plane.metadata().consensus.as_ref()?.replica_count(); + let members = self + .plane + .metadata() + .mux_stm + .streams() + .members_for_namespace(IggyNamespace::from_raw(namespace))?; + metadata::stm::stream::primary_of(members, view, nodes) + } + + /// Once per probe interval, drop the reports this node no longer leads, + /// and check one chunk of the directory for deleted partitions. The cross + /// shard directory lock is held for one chunk only. + fn retain_partition_directories(&self, now: Instant) { + if self + .partition_directory_retained_at + .get() + .is_some_and(|retained| now.duration_since(retained) < PROBE_INTERVAL) + { + return; + } + self.partition_directory_retained_at.set(Some(now)); + let next = self + .plane + .metadata() + .mux_stm + .streams() + .retain_partition_directory( + self.partition_directory_retain_cursor.get(), + DIRECTORY_RETAIN_CHUNK, + ); + self.partition_directory_retain_cursor.set(next); + self.partition_status_outbox.evict_stale(now); + } + + /// Take in partition reports, this node's own or a peer's. + pub(crate) fn record_partition_statuses(&self, statuses: &[PartitionStatusEntry]) { + let Some(consensus) = self.plane.metadata().consensus.as_ref() else { + return; + }; + if statuses.is_empty() { + return; + } + self.plane + .metadata() + .mux_stm + .streams() + .record_partition_statuses( + statuses, + consensus.node_id(), + consensus.replica_count(), + self.bus.monotonic_now(), + ); + } + #[allow(clippy::future_not_send, clippy::too_many_lines)] - pub(crate) async fn on_primary_rebalance(&self, header: PrimaryRebalanceHeader) { + pub(crate) async fn on_primary_rebalance( + &self, + header: PrimaryRebalanceHeader, + statuses: Vec<PartitionStatusEntry>, + ) { let Some(consensus) = self.plane.metadata().consensus.as_ref() else { return; }; @@ -531,6 +819,7 @@ } if header.kind == PrimaryRebalanceKind::MetadataProbe { self.credit_backup_freshness(consensus, &header, now); + self.record_partition_statuses(&statuses); } let mut reply = header; if header.kind == PrimaryRebalanceKind::MetadataProbe { @@ -552,15 +841,37 @@ .applied_frontier() .has_partition_recovery(), ); - self.send_rebalance(header.replica, reply).await; + let statuses = if header.kind == PrimaryRebalanceKind::MetadataProbe { + self.next_reported_statuses() + } else { + Vec::new() + }; + self.send_rebalance(header.replica, reply, &statuses).await; } PrimaryRebalanceKind::MetadataReply => { if consensus.is_primary() && consensus_ready(consensus) { - let evidence = { + let (answered, evidence) = { let mut state = self.primary_rebalance.borrow_mut(); - state.observe_peer(&header, now); - state.quorum_evidence(consensus.quorum_replication()) + let answered = state.observe_peer(&header, now); + ( + answered, + state.quorum_evidence(consensus.quorum_replication()), + ) }; + // Only a reply to the outstanding probe carries reports + // newer than what the directory already holds. + if answered { + self.record_partition_statuses(&statuses); + let views: Vec<_> = statuses + .iter() + .map(|status| metadata::impls::metadata::PartitionPlacementView { + namespace: IggyNamespace::from_raw(status.namespace), + created_revision: status.created_revision, + view: status.view, + }) + .collect(); + self.plane.metadata().observe_partition_views(&views); + } if let Some(sent) = evidence && self.credit_partition_admission( consensus, @@ -579,7 +890,8 @@ let outgoing = { let mut state = self.primary_rebalance.borrow_mut(); if !state.matches_pending(&header, MovePhase::Capture, now) - || header.replica != consensus.primary_index(header.partition_view) + || Some(header.replica) + != self.partition_primary_node(header.namespace, header.partition_view) { return; } @@ -596,7 +908,7 @@ pending.since = now; (pending.plan.target, pending.probe) }; - self.send_rebalance(outgoing.0, outgoing.1).await; + self.send_rebalance(outgoing.0, outgoing.1, &[]).await; } PrimaryRebalanceKind::PartitionReply => { if !consensus.is_primary() || !consensus_ready(consensus) { @@ -623,9 +935,15 @@ pending.probe.target = pending.plan.target; pending.phase = MovePhase::Transfer; pending.since = now; - (consensus.primary_index(pending.plan.view), pending.probe) + ( + self.partition_primary_node(pending.probe.namespace, pending.plan.view), + pending.probe, + ) }; - if !self.send_rebalance(outgoing.0, outgoing.1).await { + let Some(source) = outgoing.0 else { + return; + }; + if !self.send_rebalance(source, outgoing.1, &[]).await { self.primary_rebalance .borrow_mut() .forget_unsent_grant(outgoing.1.nonce); @@ -656,7 +974,7 @@ #[allow(clippy::future_not_send)] pub(crate) async fn on_local_rebalance(&self, frame: LocalRebalance) { match frame { - LocalRebalance::Wire(header) => self.on_primary_rebalance(header).await, + LocalRebalance::Wire(header) => self.on_primary_rebalance(header, Vec::new()).await, LocalRebalance::Capture(mut header) | LocalRebalance::Probe(mut header) => { let capture = matches!(frame, LocalRebalance::Capture(_)); let namespace = IggyNamespace::from_raw(header.namespace); @@ -677,14 +995,14 @@ })); if let Some(partition) = partition { let coordinator = header.replica; - header.replica = partition.consensus().replica(); + header.replica = partition.consensus().node_id(); header.kind = if capture { header.commit = partition.consensus().commit_max(); PrimaryRebalanceKind::SourceReply } else { PrimaryRebalanceKind::PartitionReply }; - self.send_rebalance(coordinator, header).await; + self.send_rebalance(coordinator, header, &[]).await; } } LocalRebalance::Transfer { namespace, request } => { @@ -837,9 +1155,15 @@ opened } - /// Returns false when the frame did not leave this node. + /// Returns false when the frame did not leave this node. `statuses` rides + /// only on a metadata probe or reply. #[allow(clippy::future_not_send)] - async fn send_rebalance(&self, target: u8, mut header: PrimaryRebalanceHeader) -> bool { + async fn send_rebalance( + &self, + target: u8, + mut header: PrimaryRebalanceHeader, + statuses: &[PartitionStatusEntry], + ) -> bool { if header.kind == PrimaryRebalanceKind::MetadataProbe { let Some(consensus) = self.plane.metadata().consensus.as_ref() else { return false; @@ -874,14 +1198,52 @@ self.send_local_rebalance(0, LocalRebalance::Wire(header)); return true; } + let body = encode_status_entries(statuses); + let size = HEADER_SIZE + body.len(); + header.size = u32::try_from(size).unwrap_or(u32::MAX); + header.checksum_body = if body.is_empty() { + 0 + } else { + u128::from(iggy_common::calculate_checksum(&body)) + }; header.seal(); - let message = Message::<PrimaryRebalanceHeader>::new(HEADER_SIZE) + let mut message = Message::<PrimaryRebalanceHeader>::new(size) .transmute_header(|_, output: &mut PrimaryRebalanceHeader| *output = header); + message.as_mut_slice()[HEADER_SIZE..size].copy_from_slice(&body); self.bus .send_to_replica(target, message.into_generic().into_frozen()) .await .is_ok() } + + /// The next run of this node's own partition reports for a metadata reply. + fn next_reported_statuses(&self) -> Vec<PartitionStatusEntry> { + self.partition_status_outbox + .next_batch(STATUS_ENTRIES_PER_FRAME) + } +} + +/// How long a partition report goes without a newer copy before its primary +/// counts as no longer leading. A node sends the reports of what it leads one +/// frame per probe and leads at most the `reports` held here, so one it keeps +/// sending comes round within that many frames. Twice that, a probe and a +/// probe timeout cover the shards' own pass and a lost reply. +fn report_stale_after(reports: usize) -> Duration { + let passes = u32::try_from(reports.div_ceil(STATUS_ENTRIES_PER_FRAME)).unwrap_or(u32::MAX); + PROBE_INTERVAL + .saturating_mul(passes.saturating_mul(2).saturating_add(1)) + .saturating_add(PROBE_TIMEOUT) +} + +/// `eligible` as a mask of node ids, bit `n` for node `n`. +fn node_mask(eligible: &[bool]) -> u32 { + eligible + .iter() + .enumerate() + .filter(|(_, ready)| **ready) + .filter_map(|(node, _)| u32::try_from(node).ok()) + .filter_map(|node| 1_u32.checked_shl(node)) + .fold(0, |mask, bit| mask | bit) } pub fn consensus_ready<B: MessageBus>(consensus: &VsrConsensus<B>) -> bool {
diff --git a/core/shard/src/primary_rebalance/tests.rs b/core/shard/src/primary_rebalance/tests.rs index 16630b7..4748929 100644 --- a/core/shard/src/primary_rebalance/tests.rs +++ b/core/shard/src/primary_rebalance/tests.rs
@@ -174,6 +174,12 @@ fn arm<SB: SuperblockStore>(shard: &TestShard<SB>, phase: MovePhase) -> PrimaryRebalanceHeader { let namespace = IggyNamespace::new(1, 1, 0); + // The coordinator names the source and target from the committed members. + shard.plane.metadata().mux_stm.streams().seed_namespace( + namespace, + namespace.inner(), + PARTITION_VIEW, + ); let consensus = shard.plane.metadata().consensus.as_ref().unwrap(); let mut probe = probe_header(consensus, NONCE); probe.namespace = namespace.inner(); @@ -254,7 +260,7 @@ } _ => unreachable!(), } - shard.on_primary_rebalance(reply).await; + shard.on_primary_rebalance(reply, Vec::new()).await; assert!( inbox.try_recv().is_err(), "{changed} reply granted a transfer" @@ -278,11 +284,11 @@ source.commit = COMMITTED + 1; let mut wrong_source = source; wrong_source.replica = TARGET; - shard.on_primary_rebalance(wrong_source).await; + shard.on_primary_rebalance(wrong_source, Vec::new()).await; assert!(shard.bus.sent.borrow().is_empty()); assert!(inbox.try_recv().is_err()); - shard.on_primary_rebalance(source).await; + shard.on_primary_rebalance(source, Vec::new()).await; let (target, probe) = shard.bus.sent.borrow()[0]; assert_eq!(target, TARGET); assert_eq!(probe.kind, PrimaryRebalanceKind::PartitionProbe); @@ -290,14 +296,14 @@ assert_eq!(probe.nonce, source.nonce); assert_eq!(probe.replica, 0); assert!(inbox.try_recv().is_err()); - shard.on_primary_rebalance(source).await; + shard.on_primary_rebalance(source, Vec::new()).await; assert_eq!(shard.bus.sent.borrow().len(), 1); let mut reply = probe; reply.kind = PrimaryRebalanceKind::PartitionReply; reply.replica = TARGET; reply.ready = 1; - shard.on_primary_rebalance(reply).await; + shard.on_primary_rebalance(reply, Vec::new()).await; assert!( inbox.try_recv().is_err(), "the coordinator must not grant its own replica of the partition" @@ -312,7 +318,7 @@ assert_eq!(grant.created_revision, CREATED_REVISION); assert_eq!(grant.partition_view, PARTITION_VIEW); assert_eq!(grant.target, TARGET); - shard.on_primary_rebalance(reply).await; + shard.on_primary_rebalance(reply, Vec::new()).await; assert_eq!( shard.bus.sent.borrow().len(), 2, @@ -322,7 +328,7 @@ let (source_node, source_inbox) = owner_at::<PingPongSuperblock>(None, source_replica, REPLICAS, METADATA_VIEW); let received_at = source_node.bus.monotonic_now(); - source_node.on_primary_rebalance(grant).await; + source_node.on_primary_rebalance(grant, Vec::new()).await; let ShardFrame::Lifecycle(LifecycleFrame::PrimaryRebalance(frame)) = source_inbox.try_recv().unwrap() else { @@ -363,7 +369,7 @@ "not ready" => consensus.begin_view_probe(), _ => unreachable!(), } - source_node.on_primary_rebalance(grant).await; + source_node.on_primary_rebalance(grant, Vec::new()).await; assert_eq!(inbox.try_recv().is_ok(), changed == "none", "{changed}"); } } @@ -384,7 +390,7 @@ pending.probe.partition_view = led_view; } reply.partition_view = led_view; - shard.on_primary_rebalance(reply).await; + shard.on_primary_rebalance(reply, Vec::new()).await; assert!( shard.bus.sent.borrow().is_empty(), "a local grant must not leave the node" @@ -416,10 +422,10 @@ let (shard, inbox) = owner(); let mut reply = arm(&shard, MovePhase::Probe); reply.ready = 0; - shard.on_primary_rebalance(reply).await; + shard.on_primary_rebalance(reply, Vec::new()).await; assert!(shard.primary_rebalance.borrow().pending.is_none()); reply.ready = 1; - shard.on_primary_rebalance(reply).await; + shard.on_primary_rebalance(reply, Vec::new()).await; assert!(inbox.try_recv().is_err()); assert!(shard.bus.sent.borrow().is_empty()); } @@ -502,6 +508,63 @@ } } +/// The partition's own members decide who leads a view, not the metadata +/// group's: members [0, 2] of three nodes lead view 7 on node 2, while view 7 +/// modulo three names node 1. A move that held must still reset the target. +#[compio::test] +async fn given_a_move_inside_a_subset_group_when_the_target_holds_should_reset_its_backoff() { + const TARGET_VIEW: u32 = 7; + let (shard, _inbox) = owner(); + let metadata = shard.plane.metadata(); + let namespace = IggyNamespace::new(1, 1, 0); + let streams = metadata.mux_stm.streams(); + streams.seed_namespace_on( + namespace, + namespace.inner(), + PARTITION_VIEW, + iggy_common::Durability::Persisted, + &[0, TARGET], + ); + assert_ne!( + metadata + .consensus + .as_ref() + .unwrap() + .primary_index(TARGET_VIEW), + TARGET + ); + arm(&shard, MovePhase::Transfer); + let created_revision = streams.created_revision_for_namespace(namespace).unwrap(); + let observe = |view| { + metadata.observe_partition_views(&[PartitionPlacementView { + namespace, + created_revision, + view, + }]); + }; + observe(PARTITION_VIEW); + let moved_at = shard.bus.monotonic_now(); + { + let mut state = shard.primary_rebalance.borrow_mut(); + let pending = state.pending.as_mut().unwrap(); + pending.plan.created_revision = created_revision; + pending.since = moved_at; + state.backoffs[usize::from(TARGET)] = TargetBackoff { + until: Some(moved_at), + wait: MOVE_HOLD_INTERVAL, + }; + } + + observe(TARGET_VIEW); + tick_with_ready_peers(&shard, moved_at).await; + tick_with_ready_peers(&shard, moved_at + MOVE_HOLD_INTERVAL).await; + + assert_eq!( + shard.primary_rebalance.borrow().backoffs[usize::from(TARGET)].wait, + Duration::ZERO + ); +} + #[compio::test] async fn given_a_backed_off_target_when_its_probe_move_is_lost_or_held_should_double_or_reset_the_wait() { @@ -594,7 +657,7 @@ .created_revision = views[0].created_revision; shard.bus.disconnected.set(true); - shard.on_primary_rebalance(reply).await; + shard.on_primary_rebalance(reply, Vec::new()).await; shard.bus.disconnected.set(false); assert!(shard.bus.sent.borrow().is_empty()); @@ -638,7 +701,7 @@ pin!(shard.dispatch_message(suspect_primary(METADATA_GROUP, METADATA_VIEW + 1), None)); assert_eq!(futures::poll!(&mut dispatch), Poll::Pending); assert!(superblock.entered.get()); - shard.on_primary_rebalance(reply).await; + shard.on_primary_rebalance(reply, Vec::new()).await; assert!( shard.bus.sent.borrow().is_empty(), "a reply handled during the suspension granted a move" @@ -720,38 +783,72 @@ } #[compio::test] -async fn given_a_new_authority_when_peers_have_not_answered_should_keep_them_eligible_until_their_first_probe_times_out() +async fn given_a_new_authority_when_peers_have_not_answered_should_withhold_readiness_until_each_is_heard() { let (shard, _inbox) = owner_at::<PingPongSuperblock>(None, 0, REPLICAS, 0); let epoch = shard.bus.monotonic_now(); - let eligible = || shard.primary_rebalance.borrow().eligible.clone(); + let readiness = || { + let state = shard.primary_rebalance.borrow(); + (state.readiness_known, state.eligible.clone()) + }; shard.tick_primary_rebalance().await; assert_eq!( - eligible(), - [true; 3], - "placement must not narrow to the coordinator before any peer answers" + readiness(), + (false, vec![true, false, false]), + "no peer counts as ready before it answers" ); let (replica, mut not_ready) = shard.bus.sent.borrow()[0]; not_ready.replica = replica; not_ready.kind = PrimaryRebalanceKind::MetadataReply; - shard.on_primary_rebalance(not_ready).await; + shard.on_primary_rebalance(not_ready, Vec::new()).await; shard.tick_primary_rebalance().await; - assert_eq!(eligible(), [true, false, true]); + assert_eq!( + readiness(), + (false, vec![true, false, false]), + "placement waits while a peer is unheard" + ); shard.bus.now.set(Some(epoch + PROBE_TIMEOUT)); shard.tick_primary_rebalance().await; - assert_eq!(eligible(), [true, false, true]); + assert!(!readiness().0); shard .bus .now .set(Some(epoch + PROBE_TIMEOUT + Duration::from_nanos(1))); shard.tick_primary_rebalance().await; assert_eq!( - eligible(), - [true, false, false], - "an unanswered first probe ends the presumption at its timeout" + readiness(), + (true, vec![true, false, false]), + "an unanswered probe settles its peer as not ready at its timeout" ); + + let (replica, mut ready) = *shard + .bus + .sent + .borrow() + .iter() + .rfind(|(replica, _)| *replica == 2) + .unwrap(); + ready.replica = replica; + ready.kind = PrimaryRebalanceKind::MetadataReply; + ready.ready = 1; + shard.on_primary_rebalance(ready, Vec::new()).await; + shard.tick_primary_rebalance().await; + assert_eq!(readiness(), (true, vec![true, false, true])); +} + +#[compio::test] +async fn given_refused_probes_when_the_coordinator_ticks_again_should_settle_their_peers_as_not_ready() + { + let (shard, _inbox) = owner_at::<PingPongSuperblock>(None, 0, REPLICAS, 0); + shard.bus.disconnected.set(true); + shard.tick_primary_rebalance().await; + assert!(!shard.primary_rebalance.borrow().readiness_known); + shard.tick_primary_rebalance().await; + let state = shard.primary_rebalance.borrow(); + assert!(state.readiness_known); + assert_eq!(state.eligible, [true, false, false]); } #[test] @@ -775,8 +872,12 @@ nonce: Some(NONCE), sent_at: Some(Instant::now()), ready_at: Some(previous), + live_at: Some(previous), answered_at: None, + heard: false, replied: false, + status_cursor: None, + changes_sent: 0, }; let mut invalid = reply; match changed { @@ -920,14 +1021,14 @@ let mut probe = probe_header(consensus, NONCE); probe.replica = 0; probe.quorum_age_ns = 0; - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; probe.nonce += 1; - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; assert_eq!(shard.bus.sent.borrow().last().unwrap().1.ready, 0); assert!(frontier.partition_requests_allowed(shard.bus.monotonic_now())); drop(other_shard); probe.nonce += 1; - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; assert_eq!(shard.bus.sent.borrow().last().unwrap().1.ready, 1); } @@ -979,7 +1080,7 @@ shard.primary_rebalance.borrow_mut().peers[usize::from(peer)].sent_at = Some(epoch + Duration::from_secs(2)); } - shard.on_primary_rebalance(reply).await; + shard.on_primary_rebalance(reply, Vec::new()).await; assert_eq!( frontier.partition_requests_allowed(shard.bus.monotonic_now()), replicas == 3 || peer == 2 @@ -1054,7 +1155,7 @@ primary.tick_primary_rebalance().await; advance(10); - primary.on_primary_rebalance(answer(0)).await; + primary.on_primary_rebalance(answer(0), Vec::new()).await; assert!( primary .plane @@ -1077,13 +1178,13 @@ ); let backup_frontier = backup.plane.metadata().applied_frontier(); - backup.on_primary_rebalance(sent(0).1).await; + backup.on_primary_rebalance(sent(0).1, Vec::new()).await; assert!(!backup_frontier.partition_requests_allowed(backup.bus.monotonic_now())); - backup.on_primary_rebalance(reprobe).await; + backup.on_primary_rebalance(reprobe, Vec::new()).await; assert!(backup_frontier.partition_requests_allowed(backup.bus.monotonic_now())); advance(30); - primary.on_primary_rebalance(answer(1)).await; + primary.on_primary_rebalance(answer(1), Vec::new()).await; advance(40); primary.tick_primary_rebalance().await; assert_eq!(primary.bus.sent.borrow().len(), 4); @@ -1095,8 +1196,8 @@ ); advance(50); - primary.on_primary_rebalance(answer(2)).await; - primary.on_primary_rebalance(answer(3)).await; + primary.on_primary_rebalance(answer(2), Vec::new()).await; + primary.on_primary_rebalance(answer(3), Vec::new()).await; advance(60); primary.tick_primary_rebalance().await; assert_eq!( @@ -1118,8 +1219,8 @@ probe.quorum_age_ns = 0; let mut previous = probe; previous.nonce -= 1; - shard.on_primary_rebalance(previous).await; - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(previous, Vec::new()).await; + shard.on_primary_rebalance(probe, Vec::new()).await; assert!(frontier.partition_requests_allowed(epoch)); shard.bus.now.set(Some(epoch + Duration::from_secs(1))); for altered in ["replay", "reordered", "view", "cluster", "primary"] { @@ -1132,38 +1233,38 @@ "primary" => invalid.replica = 2, _ => unreachable!(), } - shard.on_primary_rebalance(invalid).await; + shard.on_primary_rebalance(invalid, Vec::new()).await; assert!( !frontier.partition_requests_allowed(epoch + Duration::from_secs(8)), "{altered}" ); } shard.primary_rebalance.borrow_mut().reset_authority(); - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; assert!(!frontier.partition_requests_allowed(epoch + Duration::from_secs(8))); probe.nonce += 1; probe.quorum_age_ns = PrimaryRebalanceHeader::NO_QUORUM_EVIDENCE; - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; probe.quorum_age_ns = 0; - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; assert!( !frontier.partition_requests_allowed(epoch + Duration::from_secs(8)), "an absent-evidence probe is consumed too" ); probe.nonce += 1; probe.quorum_age_ns = u64::try_from(Duration::from_secs(8).as_nanos()).unwrap(); - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; assert!(!frontier.partition_requests_allowed(epoch + Duration::from_secs(8))); probe.nonce += 1; probe.quorum_age_ns = u64::try_from(Duration::from_secs(1).as_nanos()).unwrap(); - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; assert!( !frontier.partition_requests_allowed(epoch + Duration::from_secs(8)), "receipt must not hide the advertised evidence age" ); probe.nonce += 1; probe.quorum_age_ns = 0; - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; assert!(frontier.partition_requests_allowed(epoch + Duration::from_secs(8))); assert!(!frontier.partition_requests_allowed(epoch + Duration::from_secs(9))); } @@ -1203,20 +1304,20 @@ probe.quorum_age_ns = 0; probe.commit = COMMITTED - 1; frontier.advance(COMMITTED - 1); - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; assert!(!frontier.partition_requests_allowed(epoch)); probe.nonce += 1; probe.commit = COMMITTED; - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; assert!(!frontier.partition_requests_allowed(epoch)); frontier.advance(COMMITTED); - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; assert!( !frontier.partition_requests_allowed(epoch), "an earlier discarded packet cannot become fresh by replay" ); probe.nonce += 1; - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; assert!(frontier.partition_requests_allowed(epoch)); } @@ -1263,7 +1364,7 @@ let mut probe = probe_header(consensus, NONCE); probe.replica = 0; probe.quorum_age_ns = 0; - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; assert!( !frontier.partition_requests_allowed(epoch), "nothing bounds when the first probe of a view left the primary" @@ -1271,7 +1372,7 @@ let transit = Duration::from_secs(3); shard.bus.now.set(Some(epoch + transit)); probe.nonce += 1; - shard.on_primary_rebalance(probe).await; + shard.on_primary_rebalance(probe, Vec::new()).await; assert!( frontier.partition_requests_allowed( (epoch + window) @@ -1308,7 +1409,7 @@ shard.tick_primary_rebalance().await; advance(10); - shard.on_primary_rebalance(answer(0)).await; + shard.on_primary_rebalance(answer(0), Vec::new()).await; advance(3_100); shard.tick_primary_rebalance().await; assert_eq!(sent(2).0, 1); @@ -1319,9 +1420,284 @@ "the overdue probe can still reach the peer after this one" ); advance(3_200); - shard.on_primary_rebalance(answer(3)).await; + shard.on_primary_rebalance(answer(3), Vec::new()).await; advance(4_200); shard.tick_primary_rebalance().await; assert_eq!(sent(4).0, 2); assert!(carries_evidence(4), "the reply restores the evidence"); } + +#[test] +fn given_outbox_reports_when_a_partition_stops_being_refreshed_should_evict_only_it() { + let status = |namespace| PartitionStatusEntry { + namespace, + created_revision: 1, + view: 0, + commit: 0, + current_offset: 0, + messages_count: 0, + size_bytes: 0, + segments_count: 1, + in_sync_nodes: 0, + }; + let outbox = PartitionStatusOutbox::default(); + let start = Instant::now(); + outbox.record(&[status(1), status(2)], start); + outbox.record(&[status(2)], start + PROBE_INTERVAL); + outbox.evict_stale(start + OUTBOX_REPORT_MAX_AGE); + assert_eq!( + outbox.namespaces(), + [2], + "a report no publish refreshed names a partition this node stopped leading" + ); + assert_eq!(outbox.next_batch(4), [status(2)]); + outbox.record(&[status(3)], start + OUTBOX_REPORT_MAX_AGE); + assert_eq!(outbox.next_batch(1), [status(3)]); + assert_eq!(outbox.next_batch(1), [status(2)], "the cursor wraps"); +} + +/// A peer that does not store a partition routes from what the metadata +/// primary relays, so a new primary must go out in the next probe rather than +/// once the cycle through every partition reaches it again. +#[compio::test] +async fn given_more_reports_than_a_probe_carries_when_a_primary_changes_should_send_it_first() { + const PARTITIONS: usize = STATUS_ENTRIES_PER_FRAME + STATUS_ENTRIES_PER_FRAME / 2; + const PEER: u8 = 1; + const MOVED: usize = PARTITIONS / 2; + let (shard, _inbox) = owner(); + let streams = shard.plane.metadata().mux_stm.streams(); + let namespace = |id: usize| IggyNamespace::new(1, 1, id); + streams.seed_namespace(namespace(PARTITIONS - 1), 1, 0); + let status = |id: usize, view: u32| PartitionStatusEntry { + namespace: namespace(id).inner(), + created_revision: streams + .created_revision_for_namespace(namespace(id)) + .unwrap(), + view, + commit: 0, + current_offset: 0, + messages_count: 0, + size_bytes: 0, + segments_count: 1, + in_sync_nodes: 0, + }; + let reports: Vec<_> = (0..PARTITIONS).map(|id| status(id, 0)).collect(); + streams.record_partition_statuses(&reports, 0, REPLICAS, Instant::now()); + shard.primary_rebalance.borrow_mut().peers = vec![Peer::default(); usize::from(REPLICAS)]; + let sent: usize = (0..2).map(|_| shard.next_status_batch(PEER).0.len()).sum(); + assert!(sent > PARTITIONS, "every first report went out once"); + + streams.record_partition_statuses(&[status(MOVED, 1)], 0, REPLICAS, Instant::now()); + let (batch, _) = shard.next_status_batch(PEER); + assert_eq!( + batch.len(), + STATUS_ENTRIES_PER_FRAME, + "the cycle fills the rest" + ); + assert_eq!( + (batch[0].namespace, batch[0].view), + (namespace(MOVED).inner(), 1) + ); +} + +/// A new metadata view rebuilds the peers. The partition primaries a failover +/// moves report after it, so they must not wait behind every older change. +#[compio::test] +async fn given_a_new_metadata_view_when_a_primary_changes_should_send_it_before_older_changes() { + const PARTITIONS: usize = STATUS_ENTRIES_PER_FRAME * 2; + const PEER: u8 = 1; + const MOVED: usize = PARTITIONS - 1; + let (shard, _inbox) = owner(); + let streams = shard.plane.metadata().mux_stm.streams(); + let namespace = |id: usize| IggyNamespace::new(1, 1, id); + streams.seed_namespace(namespace(PARTITIONS - 1), 1, 0); + let status = |id: usize, view: u32| PartitionStatusEntry { + namespace: namespace(id).inner(), + created_revision: streams + .created_revision_for_namespace(namespace(id)) + .unwrap(), + view, + commit: 0, + current_offset: 0, + messages_count: 0, + size_bytes: 0, + segments_count: 1, + in_sync_nodes: 0, + }; + let reports: Vec<_> = (0..PARTITIONS).map(|id| status(id, 0)).collect(); + streams.record_partition_statuses(&reports, 0, REPLICAS, Instant::now()); + shard.primary_rebalance.borrow_mut().view = None; + // The probes this tick sends fail, so the cursors stay where the view + // started them. + shard.bus.disconnected.set(true); + shard.tick_primary_rebalance().await; + shard.bus.disconnected.set(false); + assert_eq!( + shard.primary_rebalance.borrow().peers.len(), + usize::from(REPLICAS), + "the new view rebuilt the peers" + ); + + streams.record_partition_statuses(&[status(MOVED, 1)], 0, REPLICAS, Instant::now()); + let (batch, _) = shard.next_status_batch(PEER); + assert_eq!( + (batch[0].namespace, batch[0].view), + (namespace(MOVED).inner(), 1) + ); +} + +/// The failover that makes a node metadata primary can also give it +/// partitions, which its shards may report before its metadata view starts. +/// No metadata primary relayed those reports, so they go before older changes. +#[compio::test] +async fn given_own_reports_from_before_the_metadata_view_when_it_starts_should_send_them_first() { + const PARTITIONS: usize = STATUS_ENTRIES_PER_FRAME * 2; + const PEER: u8 = 1; + const TAKEN_OVER: usize = PARTITIONS - 1; + let (shard, _inbox) = owner(); + let streams = shard.plane.metadata().mux_stm.streams(); + let namespace = |id: usize| IggyNamespace::new(1, 1, id); + streams.seed_namespace(namespace(PARTITIONS - 1), 1, 0); + let status = |id: usize, view: u32| PartitionStatusEntry { + namespace: namespace(id).inner(), + created_revision: streams + .created_revision_for_namespace(namespace(id)) + .unwrap(), + view, + commit: 0, + current_offset: 0, + messages_count: 0, + size_bytes: 0, + segments_count: 1, + in_sync_nodes: 0, + }; + let now = shard.bus.monotonic_now(); + let reports: Vec<_> = (0..PARTITIONS).map(|id| status(id, 0)).collect(); + streams.record_partition_statuses(&reports, 0, REPLICAS, now); + // As the router takes in this node's own report while it is a backup. + let taken_over = [status(TAKEN_OVER, 1)]; + shard.partition_status_outbox.record(&taken_over, now); + streams.record_partition_statuses(&taken_over, 0, REPLICAS, now); + shard.primary_rebalance.borrow_mut().view = None; + // The probes this tick sends fail, so the cursors stay where the view + // started them. + shard.bus.disconnected.set(true); + shard.tick_primary_rebalance().await; + shard.bus.disconnected.set(false); + + let (batch, _) = shard.next_status_batch(PEER); + assert_eq!( + (batch[0].namespace, batch[0].view), + (namespace(TAKEN_OVER).inner(), 1) + ); +} + +/// The bus refused the probe, so the peer never got its reports. +#[compio::test] +async fn given_an_unsent_probe_when_probing_again_should_carry_its_reports() { + const PEER: u8 = 1; + let (shard, _inbox) = owner(); + let streams = shard.plane.metadata().mux_stm.streams(); + let namespace = IggyNamespace::new(1, 1, 0); + streams.seed_namespace(namespace, 1, 0); + streams.record_partition_statuses( + &[PartitionStatusEntry { + namespace: namespace.inner(), + created_revision: streams.created_revision_for_namespace(namespace).unwrap(), + view: 2, + commit: 0, + current_offset: 0, + messages_count: 0, + size_bytes: 0, + segments_count: 1, + in_sync_nodes: 0, + }], + 0, + REPLICAS, + Instant::now(), + ); + shard.primary_rebalance.borrow_mut().peers = vec![Peer::default(); usize::from(REPLICAS)]; + let (sent, unsent) = shard.next_status_batch(PEER); + assert_eq!(sent.len(), 2, "the change, then the cycle"); + shard + .primary_rebalance + .borrow_mut() + .forget_unsent(PEER, NONCE, unsent); + let (again, _) = shard.next_status_batch(PEER); + assert_eq!(again, sent); +} + +#[test] +fn given_more_reports_than_a_probe_carries_when_sizing_the_stale_wait_should_allow_two_passes_each() +{ + assert_eq!(report_stale_after(1), Duration::from_secs(6)); + assert_eq!( + report_stale_after(STATUS_ENTRIES_PER_FRAME + 1), + Duration::from_secs(8) + ); +} + +/// A primary reports only while it leads, so a report nobody refreshes names +/// a partition no member leads. One received before the coordinator's view +/// came through the last metadata primary and gets a full wait in this view. +#[compio::test] +async fn given_a_report_no_primary_refreshes_when_the_coordinator_scans_should_count_no_member_in_sync() + { + let (shard, _inbox) = owner(); + let streams = shard.plane.metadata().mux_stm.streams(); + let namespace = IggyNamespace::new(1, 1, 0); + streams.seed_namespace(namespace, 1, 0); + let start = shard.bus.monotonic_now(); + let received = start.checked_sub(Duration::from_secs(60)).unwrap(); + streams.record_partition_statuses( + &[PartitionStatusEntry { + namespace: namespace.inner(), + created_revision: streams.created_revision_for_namespace(namespace).unwrap(), + view: 0, + commit: 0, + current_offset: 0, + messages_count: 0, + size_bytes: 0, + segments_count: 1, + in_sync_nodes: 0b111, + }], + 0, + REPLICAS, + received, + ); + // The probes carry reports, which the recording bus does not take. + shard.bus.disconnected.set(true); + shard.primary_rebalance.borrow_mut().view = None; + shard.tick_primary_rebalance().await; + let scan = async |at: Instant| { + for peer in &mut shard.primary_rebalance.borrow_mut().peers { + peer.heard = true; + peer.live_at = Some(at); + } + shard.bus.now.set(Some(at)); + shard.tick_primary_rebalance().await; + shard.primary_rebalance.borrow().partition_replication() + }; + + assert_eq!( + scan(start + PROBE_INTERVAL).await, + PartitionReplication::default(), + "the view is younger than the wait" + ); + assert_eq!( + streams.partition_status(namespace).unwrap().in_sync_nodes, + 0b111 + ); + let aged = start + report_stale_after(1) + PROBE_INTERVAL; + assert_eq!( + scan(aged).await, + PartitionReplication { + under_replicated: 1, + below_write_quorum: 1, + } + ); + assert_eq!( + streams.partition_status(namespace).unwrap().in_sync_nodes, + 0 + ); +}
diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs index 95d3e47..f1cdfd2 100644 --- a/core/shard/src/router.rs +++ b/core/shard/src/router.rs
@@ -1096,9 +1096,12 @@ ); } } - LifecycleFrame::PartitionPlacementViews(reports) => { + LifecycleFrame::PartitionPlacementViews(reports, statuses) => { if self.id == 0 { self.plane.metadata().observe_partition_views(&reports); + self.partition_status_outbox + .record(&statuses, self.bus.monotonic_now()); + self.record_partition_statuses(&statuses); } else { self.metrics.record_frame_drop( frame_drop_variant::PARTITION_PLACEMENT_VIEWS,
diff --git a/core/simulator/src/client.rs b/core/simulator/src/client.rs index a0ce46d..88236e2 100644 --- a/core/simulator/src/client.rs +++ b/core/simulator/src/client.rs
@@ -449,6 +449,21 @@ name: &str, partitions_count: u32, ) -> Message<RoutedRequestHeader> { + self.create_topic_with_factor(stream, name, partitions_count, None) + } + + /// [`Self::create_topic`] with an explicit replication factor; `None` + /// leaves it to the server default. + /// + /// # Panics + /// As [`Self::create_topic`]. + pub fn create_topic_with_factor( + &self, + stream: &str, + name: &str, + partitions_count: u32, + replication_factor: Option<u32>, + ) -> Message<RoutedRequestHeader> { let wire = CreateTopicRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), partitions_count, @@ -456,6 +471,7 @@ options: iggy_common::TopicCreateOptions { durability: iggy_common::Durability::Persisted, consumer_offset_durability: iggy_common::Durability::Persisted, + replication_factor, ..Default::default() } .to_wire()
diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs index 75542be..0729baa 100644 --- a/core/simulator/src/lib.rs +++ b/core/simulator/src/lib.rs
@@ -1658,11 +1658,11 @@ // makes the carried-forward superblock load-bearing: the group recovers its // recorded `(view, log_view)` instead of re-entering view 0. The metadata half // of the seed already ran inside `new_shard`, ahead of the replay. + // A group the replay has not committed yet comes up once it has, the way + // the production reconciler builds it after boot. for namespace in materialized_namespaces { let replica = &self.replicas[idx]; - if !materialise_partition(replica, namespace, self.consumer_offsets_max) - && replica.seeded_namespaces.borrow().contains_key(&namespace) - { + if !materialise_partition(replica, namespace, self.consumer_offsets_max) { replica.deferred_partitions.borrow_mut().insert(namespace); } } @@ -2090,6 +2090,19 @@ let created_view = streams .created_view_for_namespace(namespace) .expect("a committed partition records its creation view"); + let members = streams + .members_for_namespace(namespace) + .expect("a committed partition records its members"); + let metadata_consensus = replica.shards[0] + .plane + .metadata() + .consensus + .as_ref() + .expect("shard 0 runs metadata consensus"); + // Production builds a partition only on the nodes that store it. + if members.is_some_and(|members| members.member(metadata_consensus.replica()).is_none()) { + return false; + } // One store per incarnation, minted on first materialisation and reused after, so // the recorded view survives a replica restart. A deleted incarnation's store // must not reach its successor, and metadata replayed behind the store waits for @@ -2142,7 +2155,8 @@ retained, persistence, PartitionMaterialisation::new(epoch, created_view, first_initialization) - .with_consumer_offsets_max(consumer_offsets_max), + .with_consumer_offsets_max(consumer_offsets_max) + .with_members(members), ); } for shard in &replica.shards { @@ -2172,6 +2186,16 @@ } Command::DoViewChange => { let header = message.try_as_typed::<DoViewChangeHeader>().ok()?.header(); + // An empty vote leaves its sender in its own view by design: it + // holds no log and the sender acks nothing until a transfer. + let empty_vote = header.untrusted_log() + && header.log_view == 0 + && header.op == 0 + && header.commit == 0 + && header.size as usize == size_of::<DoViewChangeHeader>(); + if empty_vote { + return None; + } (header.group, header.partition_incarnation) } Command::StartView => { @@ -5388,7 +5412,25 @@ request: Message<RoutedRequestHeader>, ) { const PROGRESS_STEPS: usize = 4_000; - let reply = submit_and_wait_for_reply(sim, client.client_id(), 0, request); + const TRANSIENT_RETRIES: usize = 64; + const RETRY_STEPS: usize = 100; + // Placement refuses a create until the coordinator heard from every + // replica of its view, and a client retries that refusal. + let mut accepted = None; + for _ in 0..TRANSIENT_RETRIES { + let reply = submit_and_wait_for_reply(sim, client.client_id(), 0, request.clone()); + if metadata::stm::result::result_code(reply.body()) + != Some(IggyError::TransientNotAccepted.as_code()) + { + accepted = Some(reply); + break; + } + for _ in 0..RETRY_STEPS { + sim.step(); + } + } + let reply = accepted + .unwrap_or_else(|| submit_and_wait_for_reply(sim, client.client_id(), 0, request)); let code = metadata::stm::result::result_code(reply.body()); assert!( code == Some(0) || code == Some(IggyError::TransientNotCommitted.as_code()), @@ -12074,10 +12116,12 @@ /// The metadata primary admits a create that both backups journal, then /// crashes before any ack reaches it, so its client has no reply. The - /// survivors elect a primary that inherits the create uncommitted. A create - /// that primary admits meanwhile must count the inherited partitions once, - /// and the inherited create must commit once, with the assignments the - /// crashed primary persisted. + /// survivors elect a primary that inherits the create uncommitted. Until it + /// commits, that primary has not applied its view and has no readiness for + /// it, so it refuses another create rather than presume the crashed node + /// ready. The inherited create must commit once, with the assignments the + /// crashed primary persisted, and the other create must lead only on the + /// survivors. #[test] #[allow(clippy::too_many_lines)] fn given_an_inherited_create_when_the_new_primary_admits_another_should_count_and_commit_it_once() @@ -12092,10 +12136,8 @@ const ADMITTED_PARTITIONS: u32 = 3; // Nothing is placed before the inherited create, so it takes replica 0. const INHERITED_VIEWS: [u32; 1] = [0]; - // Counting the inherited partition once fills replicas 1 and 2 first. - // Dropping it starts again at replica 0 ([0, 1, 2]), and counting it - // twice repeats replica 1 ([1, 2, 1]). - const ADMITTED_VIEWS: [u32; 3] = [1, 2, 0]; + // Replica 0 crashed, so only the survivors lead, alternating. + const ADMITTED_VIEWS: [u32; 3] = [1, 2, 1]; const PREPARE_STEPS: usize = 200; const ELECTION_STEPS: usize = 5_000; const ADMISSION_STEPS: usize = 2_000; @@ -12196,7 +12238,31 @@ let admitted_create = admitter.create_topic(STREAM, ADMITTED_TOPIC, ADMITTED_PARTITIONS); let admitted_request = admitted_create.header().request; - let (admitted_op, admitted_assignments) = (0..ADMISSION_STEPS) + sim.submit_request(ADMITTER, leader, admitted_create.deep_copy().into_generic()); + let refused = (0..ADMISSION_STEPS) + .find_map(|_| { + sim.step() + .into_iter() + .find(|reply| reply.header().client == ADMITTER) + }) + .expect("the new primary must answer the second create"); + assert!( + setup_reply_is_transient(&refused), + "the new primary must refuse a create while it has no readiness for its view" + ); + assert!( + journaled_creates(&sim, leader, ADMITTER, admitted_request).is_empty(), + "the refused create must not be prepared" + ); + assert!( + metadata_progress(&sim, leader).1 < inherited_op, + "the inherited create must still be pending when the second one is refused" + ); + + for (from, to) in [(1, 2), (2, 1)] { + set_prepare_ok_delivery(&mut sim, from, to, true); + } + let reply = (0..COMMIT_STEPS) .find_map(|step| { if step % RESUBMIT_STEPS == 0 { sim.submit_request( @@ -12205,33 +12271,6 @@ admitted_create.deep_copy().into_generic(), ); } - for reply in sim.step() { - assert!( - reply.header().client == ADMITTER && setup_reply_is_transient(&reply), - "only a transient refusal of the second create may be answered \ - before the survivors ack" - ); - } - journaled_creates(&sim, leader, ADMITTER, admitted_request) - .into_iter() - .next() - }) - .expect("the new primary must admit a create while it holds the inherited one"); - assert!( - metadata_progress(&sim, leader).1 < inherited_op, - "the inherited create must still be pending when the second one is admitted" - ); - assert_eq!( - views(&admitted_assignments), - ADMITTED_VIEWS, - "the second create must count the inherited partition exactly once" - ); - - for (from, to) in [(1, 2), (2, 1)] { - set_prepare_ok_delivery(&mut sim, from, to, true); - } - let reply = (0..COMMIT_STEPS) - .find_map(|_| { sim.step().into_iter().find(|reply| { reply.header().client == ADMITTER && reply.header().request == admitted_request @@ -12239,12 +12278,20 @@ }) }) .expect("the second create must commit once the survivors ack"); + assert_eq!(iggy_binary_protocol::result_code(reply.body()), Some(0)); + let admitted_op = reply.header().op; + assert!( + admitted_op > inherited_op, + "the second create must be admitted after the inherited one commits" + ); + let [(journaled_op, admitted_assignments)] = + <[_; 1]>::try_from(journaled_creates(&sim, leader, ADMITTER, admitted_request)) + .expect("the leader must journal the second create once"); + assert_eq!(journaled_op, admitted_op); assert_eq!( - ( - reply.header().op, - iggy_binary_protocol::result_code(reply.body()) - ), - (admitted_op, Some(0)) + views(&admitted_assignments), + ADMITTED_VIEWS, + "the crashed primary must lead none of the second create's partitions" ); // Commits are ordered, so the inherited create committed before the @@ -15795,6 +15842,15 @@ replicas: u8, seed: u64, ) -> (Simulator, SimClient, IggyNamespace) { + placed_partition_cluster(replicas, replicas, seed) + } + + /// A cluster of `replicas` nodes with one partition stored on `factor` of them. + fn placed_partition_cluster( + replicas: u8, + factor: u8, + seed: u64, + ) -> (Simulator, SimClient, IggyNamespace) { server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { enabled: false, size: iggy_common::IggyByteSize::from(0u64), @@ -15814,11 +15870,21 @@ let client = SimClient::new(1); sim.shell_login(&client); commit_write(&mut sim, 1, 0, client.create_stream("freshness")); + if factor < replicas { + // Loads the lowest nodes first, so the measured partition lands on + // members whose indices are not their node ids. + commit_write( + &mut sim, + 1, + 0, + client.create_topic_with_factor("freshness", "filler", 1, Some(u32::from(factor))), + ); + } let created = commit_write( &mut sim, 1, 0, - client.create_topic("freshness", "partition", 1), + client.create_topic_with_factor("freshness", "partition", 1, Some(u32::from(factor))), ); let namespace = sim.replicas[0].shards[0] .plane @@ -15835,12 +15901,32 @@ step_until_applied(&mut sim, replica, created); } sim.init_partition(namespace); + if factor < replicas { + // The gauges count a partition no member runs as unable to commit, + // so the filler runs too. + sim.init_partition( + sim.replicas[0].shards[0] + .plane + .metadata() + .mux_stm + .streams() + .namespace_from_partition( + &iggy_binary_protocol::WireIdentifier::named("freshness").unwrap(), + &iggy_binary_protocol::WireIdentifier::named("filler").unwrap(), + 0, + ) + .unwrap(), + ); + } + let members = partition_members(&sim, namespace); + assert_eq!(members.len(), factor); assert!( (0..replicas).all(|replica| { sim.partition_consensus_state(usize::from(replica), namespace) .is_some() + == members.member(replica).is_some() }), - "freshness fixture must materialise its committed partition" + "fixture must materialise its partition on exactly its members" ); sim.discover_partition_contexts(&client); for replica in 1..replicas { @@ -15861,6 +15947,475 @@ panic!("healthy cluster never credited every node"); } + fn partition_members(sim: &Simulator, namespace: IggyNamespace) -> consensus::ReplicaSet { + sim.replicas[0].shards[0] + .plane + .metadata() + .mux_stm + .streams() + .members_for_namespace(namespace) + .flatten() + .expect("a placed partition lists its members") + } + + #[test] + #[allow(clippy::too_many_lines)] + fn given_a_three_member_partition_on_five_nodes_when_a_member_fails_should_commit_and_catch_up() + { + const REPLICAS: u8 = 5; + let (mut sim, client, namespace) = placed_partition_cluster(REPLICAS, 3, 0xF2E5_0030); + let members = partition_members(&sim, namespace); + assert_eq!(members.nodes(), &[1, 3, 4]); + let primary = (0..REPLICAS) + .find(|replica| { + sim.partition_consensus_state(usize::from(*replica), namespace) + .is_some_and(|state| state.is_primary) + }) + .expect("one member leads the partition"); + assert!(members.member(primary).is_some()); + let write = |sim: &mut Simulator, payload: &'static [u8]| { + super::tests::submit_and_wait_for_reply( + sim, + 1, + primary, + client.send_messages(namespace, &[bytes::Bytes::from_static(payload)]), + ) + .header() + .status + }; + assert_eq!(write(&mut sim, b"all members"), 0); + + let filler = sim.replicas[0].shards[0] + .plane + .metadata() + .mux_stm + .streams() + .namespace_from_partition( + &iggy_binary_protocol::WireIdentifier::named("freshness").unwrap(), + &iggy_binary_protocol::WireIdentifier::named("filler").unwrap(), + 0, + ) + .expect("the filler topic places the lowest nodes first"); + let filler_members = partition_members(&sim, filler); + // The gauges count every partition, so the crash spares the filler's. + let backup = *members + .nodes() + .iter() + .find(|node| **node != primary && filler_members.member(**node).is_none()) + .expect("a backup the filler does not store"); + let under_replicated = |sim: &Simulator| { + sim.replicas + .iter() + .enumerate() + .filter(|(replica, _)| !sim.is_crashed(u8::try_from(*replica).unwrap())) + .flat_map(|(_, replica)| replica.shards.iter()) + .fold((0, 0), |(under, below), shard| { + let metrics = shard.metrics(); + ( + under + metrics.partitions_under_replicated_value(), + below + metrics.partitions_below_write_quorum_value(), + ) + }) + }; + assert_eq!(under_replicated(&sim), (0, 0)); + + sim.replica_crash(backup); + assert_eq!( + write(&mut sim, b"two of three"), + 0, + "a quorum of the members still commits" + ); + advance_freshness_time( + &mut sim, + consensus::MEMBER_LAG_MAX + shard::PARTITION_REPORT_INTERVAL, + ); + assert_eq!( + under_replicated(&sim), + (1, 0), + "one member down leaves two copies, still a write quorum" + ); + let committed = sim + .partition_consensus_state(usize::from(primary), namespace) + .expect("the primary hosts the partition") + .commit_min; + + sim.replica_restart(backup); + assert!( + (0..SETUP_TOTAL_STEPS).any(|_| { + sim.step(); + sim.partition_consensus_state(usize::from(backup), namespace) + .is_some_and(|state| state.commit_min >= committed) + }), + "the restarted member catches up" + ); + assert_eq!( + write(&mut sim, b"all members again"), + 0, + "the returned member acks again" + ); + assert!( + (0..SETUP_TOTAL_STEPS).any(|_| { + sim.step(); + under_replicated(&sim) == (0, 0) + }), + "the partition is fully replicated again" + ); + let outsiders: Vec<u8> = (0..REPLICAS) + .filter(|replica| members.member(*replica).is_none()) + .collect(); + for &replica in &outsiders { + assert!( + sim.partition_consensus_state(usize::from(replica), namespace) + .is_none(), + "node {replica} stores no copy" + ); + let refused = super::tests::submit_and_wait_for_reply( + &mut sim, + 1, + replica, + client.send_messages(namespace, &[bytes::Bytes::from_static(b"elsewhere")]), + ); + assert_eq!( + refused.header().status, + IggyError::NotPrimary.as_code(), + "node {replica} redirects instead of waiting for a partition it never stores" + ); + } + + advance_freshness_time(&mut sim, std::time::Duration::from_secs(5)); + let view = sim + .partition_consensus_state(usize::from(primary), namespace) + .expect("the primary hosts the partition") + .view; + for &replica in &outsiders { + let streams = sim.replicas[usize::from(replica)].shards[0] + .plane + .metadata() + .mux_stm + .streams(); + let status = streams + .partition_status(namespace) + .unwrap_or_else(|| panic!("node {replica} heard from the primary")); + assert_eq!(status.view, view, "node {replica}"); + assert_eq!(status.messages_count, 3, "node {replica}"); + let counted = streams.read(|inner| { + inner + .stats_registry + .partition_get( + namespace.stream_id(), + namespace.topic_id(), + namespace.partition_id(), + ) + .map(|stats| stats.messages_count_inconsistent()) + }); + assert_eq!( + counted, + Some(3), + "node {replica} counts the remote partition" + ); + } + } + + /// The normal primary of `namespace` and its view, ignoring `skip`. + fn partition_leader( + sim: &Simulator, + namespace: IggyNamespace, + skip: Option<u8>, + ) -> Option<(u8, u32)> { + (0..u8::try_from(sim.replicas.len()).unwrap()) + .filter(|replica| Some(*replica) != skip) + .find_map(|replica| { + sim.partition_consensus_state(usize::from(replica), namespace) + .filter(|state| state.is_primary && state.status == consensus::Status::Normal) + .map(|state| (replica, state.view)) + }) + } + + /// A write's final status, retrying transient refusals as clients do. + fn write_through( + sim: &mut Simulator, + client: &SimClient, + namespace: IggyNamespace, + target: u8, + payload: &'static [u8], + ) -> u32 { + const TRANSIENT_RETRIES: usize = 64; + const RETRY_STEPS: usize = 10; + let mut status = IggyError::TransientNotAccepted.as_code(); + for _ in 0..TRANSIENT_RETRIES { + status = super::tests::submit_and_wait_for_reply( + sim, + 1, + target, + client.send_messages(namespace, &[bytes::Bytes::from_static(payload)]), + ) + .header() + .status; + if status != IggyError::TransientNotAccepted.as_code() { + break; + } + for _ in 0..RETRY_STEPS { + sim.step(); + } + } + status + } + + #[test] + fn given_a_three_member_partition_on_five_nodes_when_its_primary_fails_should_elect_among_members() + { + const REPLICAS: u8 = 5; + let (mut sim, client, namespace) = placed_partition_cluster(REPLICAS, 3, 0xF2E5_0040); + let members = partition_members(&sim, namespace); + let (primary, view) = + partition_leader(&sim, namespace, None).expect("one member leads the partition"); + assert!(members.member(primary).is_some()); + assert_eq!( + write_through(&mut sim, &client, namespace, primary, b"first primary"), + 0 + ); + + sim.replica_crash(primary); + let (elected, elected_view) = (0..SETUP_TOTAL_STEPS) + .find_map(|_| { + sim.step(); + partition_leader(&sim, namespace, Some(primary)) + .filter(|(_, elected_view)| *elected_view > view) + }) + .expect("the surviving members elect a new primary"); + assert!( + members.member(elected).is_some(), + "node {elected} leads a partition it does not store" + ); + assert_eq!( + write_through(&mut sim, &client, namespace, elected, b"second primary"), + 0, + "two of three members still commit" + ); + let committed = sim + .partition_consensus_state(usize::from(elected), namespace) + .expect("the elected primary hosts the partition") + .commit_min; + + advance_freshness_time(&mut sim, std::time::Duration::from_secs(5)); + for replica in (0..REPLICAS).filter(|replica| members.member(*replica).is_none()) { + assert!( + sim.partition_consensus_state(usize::from(replica), namespace) + .is_none(), + "node {replica} took part in a view change of a partition it does not store" + ); + let status = sim.replicas[usize::from(replica)].shards[0] + .plane + .metadata() + .mux_stm + .streams() + .partition_status(namespace) + .unwrap_or_else(|| panic!("node {replica} heard from the elected primary")); + assert!( + status.view >= elected_view, + "node {replica} still routes to view {}", + status.view + ); + } + + sim.replica_restart(primary); + assert!( + (0..SETUP_TOTAL_STEPS).any(|_| { + sim.step(); + sim.partition_consensus_state(usize::from(primary), namespace) + .is_some_and(|state| { + state.status == consensus::Status::Normal + && state.view >= elected_view + && state.commit_min >= committed + }) + }), + "the old primary rejoins the new view and catches up" + ); + let (leader, _) = partition_leader(&sim, namespace, None).expect("a member leads"); + assert_eq!( + write_through(&mut sim, &client, namespace, leader, b"all members again"), + 0 + ); + } + + #[test] + fn given_a_two_member_partition_when_its_primary_fails_should_wait_for_it_without_electing() { + const REPLICAS: u8 = 5; + const OUTAGE_STEPS: u32 = 2_000; + let (mut sim, client, namespace) = placed_partition_cluster(REPLICAS, 2, 0xF2E5_0050); + let members = partition_members(&sim, namespace); + let (primary, view) = + partition_leader(&sim, namespace, None).expect("one member leads the partition"); + assert!(members.member(primary).is_some()); + assert_eq!( + write_through(&mut sim, &client, namespace, primary, b"both members"), + 0 + ); + + // Two members need both for a view change, so the survivor never leads. + sim.replica_crash(primary); + for _ in 0..OUTAGE_STEPS { + sim.step(); + assert_eq!(partition_leader(&sim, namespace, Some(primary)), None); + } + assert_eq!( + cluster_below_write_quorum(&sim, Some(primary)), + 1, + "the metadata primary counts the pair that no primary is up to count" + ); + + sim.replica_restart(primary); + let (leader, _) = (0..SETUP_TOTAL_STEPS) + .find_map(|_| { + sim.step(); + partition_leader(&sim, namespace, None) + .filter(|(_, leader_view)| *leader_view >= view) + }) + .expect("the pair resumes once both members are back"); + assert!(members.member(leader).is_some()); + assert_eq!( + write_through(&mut sim, &client, namespace, leader, b"pair again"), + 0 + ); + assert!( + (0..SETUP_TOTAL_STEPS).any(|_| { + sim.step(); + cluster_below_write_quorum(&sim, None) == 0 + }), + "no node counts the pair once both members are back" + ); + } + + /// A two-member partition with its backup down cannot commit. Its primary + /// hears too few members, steps down and keeps trying to elect, so the + /// count must not depend on which node leads it at the moment. + #[test] + fn given_a_two_member_partition_when_its_backup_is_down_should_count_it_below_write_quorum_once() + { + const REPLICAS: u8 = 5; + const SAMPLES: usize = 200; + let (mut sim, client, namespace) = placed_partition_cluster(REPLICAS, 2, 0xF2E5_0070); + let members = partition_members(&sim, namespace); + let (primary, _) = + partition_leader(&sim, namespace, None).expect("one member leads the partition"); + assert_eq!( + write_through(&mut sim, &client, namespace, primary, b"both members"), + 0 + ); + let backup = *members + .nodes() + .iter() + .find(|node| **node != primary) + .expect("two members"); + + sim.replica_crash(backup); + advance_freshness_time( + &mut sim, + consensus::MEMBER_LAG_MAX + shard::PARTITION_REPORT_INTERVAL, + ); + for _ in 0..SAMPLES { + for _ in 0..10 { + sim.step(); + } + assert_eq!(cluster_below_write_quorum(&sim, Some(backup)), 1); + } + } + + /// `partitions_below_write_quorum` summed over every shard of every node + /// but `down`, as an operator sums it over the cluster. + fn cluster_below_write_quorum(sim: &Simulator, down: Option<u8>) -> i64 { + sim.replicas + .iter() + .enumerate() + .filter(|(replica, _)| down.is_none_or(|down| *replica != usize::from(down))) + .flat_map(|(_, replica)| replica.shards.iter()) + .map(|shard| shard.metrics().partitions_below_write_quorum_value()) + .sum() + } + + /// Either member of a two-member partition can lose its storage, as one + /// `--fresh` replica at a time does. Both durabilities are persisted, so + /// only that member's empty vote lets the survivor, which holds every + /// committed op, lead a view to transfer the history back from. In-memory + /// storage does not model the transfer, so this stops at the view. + #[test] + fn given_a_two_member_partition_when_either_member_returns_without_its_storage_should_lead_on_the_survivor() + { + const REPLICAS: u8 = 5; + for wipe_primary in [true, false] { + let (mut sim, client, namespace) = + placed_partition_cluster(REPLICAS, 2, 0xF2E5_0060 + u64::from(wipe_primary)); + let members = partition_members(&sim, namespace); + let (primary, _) = + partition_leader(&sim, namespace, None).expect("one member leads the partition"); + let survivor = *members + .nodes() + .iter() + .find(|node| **node != primary) + .expect("two members"); + let (wiped, survivor) = if wipe_primary { + (primary, survivor) + } else { + (survivor, primary) + }; + assert_eq!( + write_through(&mut sim, &client, namespace, primary, b"before the wipe"), + 0 + ); + let committed = sim + .partition_consensus_state(usize::from(primary), namespace) + .expect("the primary hosts the partition") + .commit_min; + + sim.replica_crash(wiped); + sim.replica_restart_losing_partitions(wiped, &[namespace]); + let rebuilt = |sim: &Simulator| { + sim.replicas[usize::from(wiped)] + .partition_shard(namespace) + .plane + .partitions() + .contains(&namespace) + }; + assert!( + (0..SETUP_TOTAL_STEPS).any(|_| { + sim.step(); + rebuilt(&sim) + }), + "the restarted member rebuilds the partition" + ); + // As the server marks a group whose storage went with the disk. + sim.replicas[usize::from(wiped)] + .partition_shard(namespace) + .plane + .partitions() + .get_mut_by_ns(&namespace) + .expect("the rebuilt partition is live") + .force_requires_state_transfer_for_test(); + // Nothing commits until the transfer lands, since both members ack. + assert!( + (0..SETUP_TOTAL_STEPS).any(|_| { + sim.step(); + sim.partition_consensus_state(usize::from(survivor), namespace) + .is_some_and(|state| { + state.status == Status::Normal + && state.is_primary + && (!wipe_primary || state.view > 0) + }) + }), + "wipe_primary={wipe_primary}: node {survivor} never led: {:?}", + sim.partition_consensus_state(usize::from(survivor), namespace) + ); + let kept = + super::tests::retained_prepare(&sim, usize::from(survivor), namespace, committed); + assert!( + kept.body() + .windows(b"before the wipe".len()) + .any(|bytes| bytes == b"before the wipe"), + "wipe_primary={wipe_primary}: the view dropped the committed op" + ); + } + } + fn partition_fresh(sim: &Simulator, replica: u8) -> bool { let shard = &sim.replicas[usize::from(replica)].shards[0]; shard @@ -16235,13 +16790,22 @@ request: Message<RoutedRequestHeader>, ) -> u64 { let request_id = request.header().request; - sim.submit_request(client_id, target, request.into_generic()); + let request = request.into_generic(); + sim.submit_request(client_id, target, request.deep_copy()); for _ in 0..SETUP_TOTAL_STEPS { if let Some(reply) = sim .step() .into_iter() .find(|reply| reply.header().request == request_id) { + // Placement refuses a create until the primary has heard from + // every node in its view, and a client retries that refusal. + if iggy_binary_protocol::result_code(reply.body()) + == Some(iggy_common::IggyError::TransientNotAccepted.as_code()) + { + sim.submit_request(client_id, target, request.deep_copy()); + continue; + } assert_eq!( reply.header().status, 0,
diff --git a/core/simulator/src/workload/ops/create_partitions.rs b/core/simulator/src/workload/ops/create_partitions.rs index 5b8b601..ea5a67e 100644 --- a/core/simulator/src/workload/ops/create_partitions.rs +++ b/core/simulator/src/workload/ops/create_partitions.rs
@@ -87,7 +87,9 @@ Outcome::PartitionIdSpaceExhausted => { unreachable!("create_partitions does not target PartitionIdSpaceExhausted") } - Outcome::PartitionResizeDisabled => None, + // The sim client never sets a replication factor, so admission and + // apply agree on the member count. + Outcome::PartitionResizeDisabled | Outcome::InvalidReplicationFactor => None, } }
diff --git a/core/simulator/src/workload/ops/create_topic.rs b/core/simulator/src/workload/ops/create_topic.rs index b097a53..f869511 100644 --- a/core/simulator/src/workload/ops/create_topic.rs +++ b/core/simulator/src/workload/ops/create_topic.rs
@@ -85,6 +85,7 @@ // stream, and the primary mints dense 0-based partition ids, so neither // partition guard is reachable from the wire. Outcome::InvalidOptionValue + | Outcome::InvalidReplicationFactor | Outcome::TooManyTopics | Outcome::InvalidPartitionsCount | Outcome::PartitionIdSpaceExhausted => None,
diff --git a/examples/node/package-lock.json b/examples/node/package-lock.json index 7aa2e08..0e0c239 100644 --- a/examples/node/package-lock.json +++ b/examples/node/package-lock.json
@@ -23,7 +23,7 @@ }, "../../foreign/node": { "name": "apache-iggy", - "version": "0.10.1-edge.6", + "version": "0.10.1-edge.8", "license": "Apache-2.0", "dependencies": { "@node-rs/xxhash": "1.7.8",
diff --git a/examples/python/uv.lock b/examples/python/uv.lock index ec939e5..b259564 100644 --- a/examples/python/uv.lock +++ b/examples/python/uv.lock
@@ -8,7 +8,7 @@ [[package]] name = "apache-iggy" -version = "0.9.1.dev6" +version = "0.9.1.dev8" source = { directory = "../../foreign/python" } [package.metadata]
diff --git a/foreign/cpp/Cargo.toml b/foreign/cpp/Cargo.toml index 7065ebc..bdcf5d2 100644 --- a/foreign/cpp/Cargo.toml +++ b/foreign/cpp/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy-cpp" -version = "0.1.8" +version = "0.1.10" edition = "2024" license = "Apache-2.0"
diff --git a/foreign/cpp/MODULE.bazel b/foreign/cpp/MODULE.bazel index 49ed253..9ea7c4d 100644 --- a/foreign/cpp/MODULE.bazel +++ b/foreign/cpp/MODULE.bazel
@@ -17,7 +17,7 @@ module( name = "iggy_cpp", - version = "0.1.8", + version = "0.1.10", ) bazel_dep(name = "rules_cc", version = "0.2.26")
diff --git a/foreign/cpp/include/iggy.hpp b/foreign/cpp/include/iggy.hpp index 108cef0..d4e3ce6 100644 --- a/foreign/cpp/include/iggy.hpp +++ b/foreign/cpp/include/iggy.hpp
@@ -1557,6 +1557,26 @@ */ [[nodiscard]] PartitionContext Context() const noexcept { return context_; } + /** + * @brief Returns the nodes that store the partition. + * + * While one member is down, a write still commits if a quorum of the + * others is in sync. + * @return Node IDs in member order, owned by this value. The reference + * remains valid while this Partition remains alive. + */ + [[nodiscard]] const std::vector<std::uint8_t> &Members() const noexcept { return members_; } + + /** + * @brief Returns the members the partition's primary last reported in + * sync and that are up. + * @return Node IDs among Members(), owned by this value. Empty until the + * primary first reports, and while no member that is up can lead + * the partition. The reference remains valid while this Partition + * remains alive. + */ + [[nodiscard]] const std::vector<std::uint8_t> &InSyncMembers() const noexcept { return in_sync_members_; } + private: Partition(std::uint32_t id, std::uint64_t created_at, @@ -1564,14 +1584,18 @@ std::uint64_t current_offset, std::uint64_t size_bytes, std::uint64_t messages_count, - PartitionContext context) + PartitionContext context, + std::vector<std::uint8_t> members, + std::vector<std::uint8_t> in_sync_members) : id_(id), created_at_(created_at), segments_count_(segments_count), current_offset_(current_offset), size_bytes_(size_bytes), messages_count_(messages_count), - context_(context) {} + context_(context), + members_(std::move(members)), + in_sync_members_(std::move(in_sync_members)) {} static Partition FromFfi(ffi::Partition partition); @@ -1584,6 +1608,8 @@ std::uint64_t size_bytes_; std::uint64_t messages_count_; PartitionContext context_; + std::vector<std::uint8_t> members_; + std::vector<std::uint8_t> in_sync_members_; }; /** @@ -2994,6 +3020,7 @@ MessagesRequiredToSave, SizeOfMessagesRequiredToSave, PreallocateSegments, + ReplicationFactor, }; constexpr std::string_view to_string(const TopicOptionKey key) { @@ -3020,6 +3047,8 @@ return "size_of_messages_required_to_save"; case TopicOptionKey::PreallocateSegments: return "preallocate_segments"; + case TopicOptionKey::ReplicationFactor: + return "replication_factor"; } throw std::invalid_argument("Unknown topic option key"); } @@ -3276,6 +3305,30 @@ } /** + * @brief Returns the number of nodes that store each partition. + * @return Configured replication factor, or `std::nullopt` to apply the + * server's `[cluster] default_replication_factor`: every node + * unless an operator set a number. + */ + [[nodiscard]] std::optional<std::uint32_t> ReplicationFactor() const noexcept { return replication_factor_; } + + /** + * @brief Sets the number of nodes that store each partition. + * + * A write commits on a quorum of these nodes, so 2 keeps two copies but + * stops accepting writes while either node is down. + * @param replication_factor Number of nodes, from 1 to 32 inclusive and at + * most the cluster's node count. + * @return Reference to this options object. + * @note The server rejects an out-of-range value with + * InvalidReplicationFactor (2018). + */ + TopicCreateOptions &SetReplicationFactor(std::uint32_t replication_factor) noexcept { + replication_factor_ = replication_factor; + return *this; + } + + /** * @brief Returns additional topic settings as key-value pairs. * * Use this for supported settings that do not have a dedicated setter. @@ -3326,6 +3379,7 @@ std::optional<std::uint32_t> messages_required_to_save_; std::optional<std::uint64_t> size_of_messages_required_to_save_; std::optional<bool> preallocate_segments_; + std::optional<std::uint32_t> replication_factor_; std::map<std::string, std::string> raw_; friend class IggyBlockingClient;
diff --git a/foreign/cpp/src/client.cpp b/foreign/cpp/src/client.cpp index 329ad1a..70cae8e 100644 --- a/foreign/cpp/src/client.cpp +++ b/foreign/cpp/src/client.cpp
@@ -317,6 +317,13 @@ ffi_options.has_preallocate_segments = false; ffi_options.preallocate_segments = false; } + if (auto value = options.ReplicationFactor()) { + ffi_options.has_replication_factor = true; + ffi_options.replication_factor = *value; + } else { + ffi_options.has_replication_factor = false; + ffi_options.replication_factor = 0; + } return TopicDetails::FromFfi(Handle()->create_topic(stream.ToFfi(), name, std::move(ffi_options), error_info)); }); }
diff --git a/foreign/cpp/src/client.rs b/foreign/cpp/src/client.rs index 7ff2d3d..5df680f 100644 --- a/foreign/cpp/src/client.rs +++ b/foreign/cpp/src/client.rs
@@ -1822,6 +1822,7 @@ | IggyError::StreamEncryptionUnavailable | IggyError::UnsupportedCriticalOption(_) | IggyError::CannotModifyRootUser + | IggyError::InvalidReplicationFactor ) } @@ -1934,6 +1935,15 @@ assert_eq!(root.code, IggyError::CannotModifyRootUser.as_code()); assert_eq!(root.kind, "cannot_modify_root_user"); + let mut factor = ffi::ErrorInfo::default(); + factor.record(&IggyError::HttpResponseError( + 400, + r#"{"id":2018,"code":"invalid_replication_factor","reason":"Invalid replication factor","field":null}"# + .to_owned(), + )); + assert_eq!(factor.code, IggyError::InvalidReplicationFactor.as_code()); + assert_eq!(factor.kind, "invalid_replication_factor"); + let mut other = ffi::ErrorInfo::default(); other.record(&IggyError::HttpResponseError( 400,
diff --git a/foreign/cpp/src/lib.rs b/foreign/cpp/src/lib.rs index d14c3cf..357d908 100644 --- a/foreign/cpp/src/lib.rs +++ b/foreign/cpp/src/lib.rs
@@ -104,6 +104,11 @@ /// The owner generation is zero: topic details belong to no consumer /// group. context: PartitionContext, + /// Node ids that store the partition, in member order. + members: Vec<u8>, + /// Members the partition's primary last reported in sync and that are + /// up. Empty until the primary first reports. + in_sync_members: Vec<u8>, } struct TopicDetails { @@ -690,6 +695,8 @@ size_of_messages_required_to_save: u64, has_preallocate_segments: bool, preallocate_segments: bool, + has_replication_factor: bool, + replication_factor: u32, has_partition_resize_policy: bool, partition_resize_policy: String, raw_options: Vec<HeaderEntry>,
diff --git a/foreign/cpp/src/type_conversion.rs b/foreign/cpp/src/type_conversion.rs index ae1dbdf..6ac2476 100644 --- a/foreign/cpp/src/type_conversion.rs +++ b/foreign/cpp/src/type_conversion.rs
@@ -511,6 +511,8 @@ size_bytes: partition.size.as_bytes_u64(), messages_count: partition.messages_count, context: partition.context.into(), + members: partition.members, + in_sync_members: partition.in_sync_members, } } } @@ -743,6 +745,11 @@ } else { None }, + replication_factor: if options.has_replication_factor { + Some(options.replication_factor) + } else { + None + }, raw, }) } @@ -1916,6 +1923,20 @@ ) } + fn topic_partition(members: Vec<u8>, in_sync_members: Vec<u8>) -> RustPartition { + RustPartition { + id: 1, + created_at: IggyTimestamp::from(11), + segments_count: 1, + current_offset: 2, + size: RustIggyByteSize::from(13), + messages_count: 3, + context: CONTEXT, + members, + in_sync_members, + } + } + #[test] fn given_optional_bridge_fields_when_converting_should_preserve_presence() { let unnamed = RustStreamUpdateOptions::try_from(ffi::StreamUpdateOptions { @@ -2122,20 +2143,22 @@ #[test] fn should_keep_the_partition_context_of_a_topic_partition() { - let partition = RustPartition { - id: 1, - created_at: IggyTimestamp::from(11), - segments_count: 1, - current_offset: 2, - size: RustIggyByteSize::from(13), - messages_count: 3, - context: CONTEXT, - }; + let partition = topic_partition(Vec::new(), Vec::new()); assert_eq!(fields(&ffi::Partition::from(partition).context), (3, 5, 7)); } #[test] + fn given_topic_partition_with_members_when_bridged_should_keep_members_and_in_sync_members() { + let partition = topic_partition(vec![1, 3, 4], vec![1, 4]); + + let bridged = ffi::Partition::from(partition); + + assert_eq!(bridged.members, [1, 3, 4]); + assert_eq!(bridged.in_sync_members, [1, 4]); + } + + #[test] fn should_keep_the_partition_context_of_polled_messages() { let polled = RustPolledMessages::new(CONTEXT, PARTITION, 8, 1, vec![message_at(7)]);
diff --git a/foreign/cpp/src/type_conversions.cpp b/foreign/cpp/src/type_conversions.cpp index 9fb7450..5279003 100644 --- a/foreign/cpp/src/type_conversions.cpp +++ b/foreign/cpp/src/type_conversions.cpp
@@ -274,7 +274,9 @@ Partition Partition::FromFfi(ffi::Partition partition) { return Partition(partition.id, partition.created_at, partition.segments_count, partition.current_offset, - partition.size_bytes, partition.messages_count, PartitionContext::FromFfi(partition.context)); + partition.size_bytes, partition.messages_count, PartitionContext::FromFfi(partition.context), + std::vector<std::uint8_t>(partition.members.begin(), partition.members.end()), + std::vector<std::uint8_t>(partition.in_sync_members.begin(), partition.in_sync_members.end())); } TopicDetails TopicDetails::FromFfi(ffi::TopicDetails topic) {
diff --git a/foreign/cpp/tests/e2e/resource_options.cpp b/foreign/cpp/tests/e2e/resource_options.cpp index d671539..50f6a64 100644 --- a/foreign/cpp/tests/e2e/resource_options.cpp +++ b/foreign/cpp/tests/e2e/resource_options.cpp
@@ -71,6 +71,7 @@ iggy::TopicOptionKey::MessagesRequiredToSave, iggy::TopicOptionKey::SizeOfMessagesRequiredToSave, iggy::TopicOptionKey::PreallocateSegments, + iggy::TopicOptionKey::ReplicationFactor, }; constexpr iggy::UserOptionKey kUserOptionKeys[] = { @@ -711,9 +712,18 @@ EXPECT_EQ(spec.Key(), entry.at("key").get<std::string>()); EXPECT_EQ(static_cast<std::uint8_t>(spec.Kind()), kind_codes.at(entry.at("kind").get<std::string>())); EXPECT_EQ(FlagsOf(spec), entry.at("flags").get<std::uint32_t>()); - EXPECT_EQ(DefaultText(spec), entry.at("default").is_null() - ? std::nullopt - : std::optional<std::string>(entry.at("default").get<std::string>())); + const auto default_text = DefaultText(spec); + if (spec.Key() == iggy::to_string(iggy::TopicOptionKey::ReplicationFactor)) { + // The server's [cluster] default_replication_factor decides this default, so the + // static catalog in the fixture has none. + ASSERT_TRUE(default_text.has_value()); + EXPECT_GE(std::stoul(*default_text), 1u); + EXPECT_LE(std::stoul(*default_text), 32u); + } else { + EXPECT_EQ(default_text, entry.at("default").is_null() + ? std::nullopt + : std::optional<std::string>(entry.at("default").get<std::string>())); + } EXPECT_FALSE(spec.Description().empty()); } }
diff --git a/foreign/cpp/tests/e2e/topic.cpp b/foreign/cpp/tests/e2e/topic.cpp index 9202863..7991719 100644 --- a/foreign/cpp/tests/e2e/topic.cpp +++ b/foreign/cpp/tests/e2e/topic.cpp
@@ -294,7 +294,8 @@ .SetDurability(iggy::Durability::Persisted) .SetMessagesRequiredToSave(messages_required_to_save) .SetSizeOfMessagesRequiredToSave(size_of_messages_required_to_save) - .SetPreallocateSegments(false); + .SetPreallocateSegments(false) + .SetReplicationFactor(1); ASSERT_NO_THROW(client.CreateTopic(iggy::Identifier::String(stream_name), topic_name, options)); @@ -308,6 +309,7 @@ EXPECT_EQ(explicit_options.count("messages_required_to_save"), 1u); EXPECT_EQ(explicit_options.count("size_of_messages_required_to_save"), 1u); EXPECT_EQ(explicit_options.count("preallocate_segments"), 1u); + EXPECT_EQ(explicit_options.count("replication_factor"), 1u); auto check_kind = [&](const std::string &key, iggy::HeaderKind expected) { auto it = explicit_options.find(key); @@ -320,6 +322,7 @@ check_kind("messages_required_to_save", iggy::HeaderKind::Uint32); check_kind("size_of_messages_required_to_save", iggy::HeaderKind::Uint64); check_kind("preallocate_segments", iggy::HeaderKind::Bool); + check_kind("replication_factor", iggy::HeaderKind::Uint32); EXPECT_EQ(explicit_options.at("segment_size").Value(), (std::vector<std::uint8_t>{0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00})); @@ -328,8 +331,9 @@ EXPECT_EQ(explicit_options.at("size_of_messages_required_to_save").Value(), (std::vector<std::uint8_t>{0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00})); EXPECT_EQ(explicit_options.at("preallocate_segments").Value(), (std::vector<std::uint8_t>{0x00})); + EXPECT_EQ(explicit_options.at("replication_factor").Value(), (std::vector<std::uint8_t>{0x01, 0x00, 0x00, 0x00})); - EXPECT_EQ(explicit_options.size(), 6u); + EXPECT_EQ(explicit_options.size(), 7u); EXPECT_EQ(topic_details.Options().Derived().count("segment_size"), 0u) << "segment_size was set explicitly, so it cannot be derived"; }
diff --git a/foreign/cpp/tests/fixtures/resource_options.json b/foreign/cpp/tests/fixtures/resource_options.json index 202888f..3911499 100644 --- a/foreign/cpp/tests/fixtures/resource_options.json +++ b/foreign/cpp/tests/fixtures/resource_options.json
@@ -16,18 +16,18 @@ }, "constants": { "protocol_version": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "protocol_version_min": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "command_codes": { "create_stream": { @@ -343,6 +343,21 @@ "default": "false" }, { + "key": "replication_factor", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": null + }, + { "key": "encryption_key", "scopes": [ "stream" @@ -439,7 +454,7 @@ { "scope": "topic", "request_hex": "01", - "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "response_hex": "0c000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465127265706c69636174696f6e5f666163746f720b00000000007b0100004e6f64657320746861742073746f7265206561636820706172746974696f6e2c20312e2e3d333220616e64206174206d6f73742074686520636c75737465722773206e6f646520636f756e742e204f6d69747465642c20746865207365727665722773205b636c75737465725d2064656661756c745f7265706c69636174696f6e5f666163746f72206170706c6965733a206576657279206e6f646520756e6c65737320616e206f70657261746f72207365742061206e756d6265722e204120777269746520636f6d6d697473206f6e20612071756f72756d206f66207468657365206e6f6465732c20736f2032206b656570732074776f20636f70696573206275742073746f70732077726974696e67207768696c652065697468657220697320646f776e2c20616e642031206b65657073206f6e6520636f70792074686174207265706c696361746564206475726162696c6974792061636b6e6f776c6564676573206265666f72652069742072656163686573206469736b", "entries": [ { "key": "name", @@ -506,6 +521,12 @@ "kind": "bool", "flags": 0, "default": "false" + }, + { + "key": "replication_factor", + "kind": "uint32", + "flags": 0, + "default": null } ] },
diff --git a/foreign/cpp/tests/unit/resource_options.cpp b/foreign/cpp/tests/unit/resource_options.cpp index 73f1e0f..4a3e05e 100644 --- a/foreign/cpp/tests/unit/resource_options.cpp +++ b/foreign/cpp/tests/unit/resource_options.cpp
@@ -183,6 +183,8 @@ }}, {"preallocate_segments", [] { return TopicCreateOptions().SetPreallocateSegments(true).PreallocateSegments().has_value(); }}, + {"replication_factor", + [] { return TopicCreateOptions().SetReplicationFactor(1).ReplicationFactor().has_value(); }}, }, { {"name", [] { return TopicUpdateOptions().SetName("renamed").Name().has_value(); }},
diff --git a/foreign/cpp/tests/unit/unit_tests.cpp b/foreign/cpp/tests/unit/unit_tests.cpp index 4bb7315..1761eb2 100644 --- a/foreign/cpp/tests/unit/unit_tests.cpp +++ b/foreign/cpp/tests/unit/unit_tests.cpp
@@ -132,6 +132,7 @@ EXPECT_FALSE(options.MessagesRequiredToSave().has_value()); EXPECT_FALSE(options.SizeOfMessagesRequiredToSave().has_value()); EXPECT_FALSE(options.PreallocateSegments().has_value()); + EXPECT_FALSE(options.ReplicationFactor().has_value()); EXPECT_TRUE(options.RawEntries().empty()); } @@ -194,6 +195,13 @@ EXPECT_EQ(*disabled.PreallocateSegments(), false); } +TEST(TopicCreateOptionsTest, ReplicationFactorStoresValue) { + iggy::TopicCreateOptions options; + options.SetReplicationFactor(3); + ASSERT_TRUE(options.ReplicationFactor().has_value()); + EXPECT_EQ(*options.ReplicationFactor(), 3u); +} + TEST(TopicCreateOptionsTest, MaximumValuesPreserved) { iggy::TopicCreateOptions options; options.SetSegmentSize(std::numeric_limits<std::uint64_t>::max());
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs index f5b60e1..11e3c8b 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs
@@ -114,6 +114,12 @@ protected virtual ResourceMapping[] ResourceMappings => []; /// <summary> + /// The cluster size, which is what a server on the default settings reports as the default replication + /// factor. + /// </summary> + public int ClusterNodeCount => NodeCount; + + /// <summary> /// Cluster size, mirroring the Rust integration harness knob of the same name: the /// IGGY_TEST_CLUSTER_NODES environment variable decides (default 1, a standalone node with /// clustering disabled; 2 or more, a replicated cluster). A subclass can pin its own size
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/OptionsTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/OptionsTests.cs index 82f3ef3..c4d19c8 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/OptionsTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/OptionsTests.cs
@@ -38,6 +38,7 @@ private const int ReadOnlyOption = 4105; private const int UnsupportedOptionKey = 4041; private const int InvalidTopicName = 2014; + private const int InvalidReplicationFactor = 2018; private static readonly string[] TopicCatalogKeys = [ @@ -51,7 +52,8 @@ "partition_resize_policy", "messages_required_to_save", "size_of_messages_required_to_save", - "preallocate_segments" + "preallocate_segments", + "replication_factor" ]; private static readonly string[] NeverReturnedKeys = ["encryption_key", "encryption_key_wrapped", "critical_keys"]; @@ -112,6 +114,9 @@ specs["preallocate_segments"].Kind.ShouldBe(HeaderKind.Bool); specs["preallocate_segments"].DefaultValue.ShouldBe([0]); + specs["replication_factor"].Kind.ShouldBe(HeaderKind.Uint32); + BitConverter.ToUInt32(specs["replication_factor"].DefaultValue).ShouldBe((uint)Fixture.ClusterNodeCount); + specs["message_expiry"].Updatable.ShouldBeTrue(); specs["message_expiry"].Resettable.ShouldBeTrue(); specs["compression_algorithm"].Updatable.ShouldBeFalse(); @@ -177,7 +182,8 @@ PartitionResizePolicy = PartitionResizePolicy.Fixed, MessagesRequiredToSave = 1, SizeOfMessagesRequiredToSave = 1, - PreallocateSegments = true + PreallocateSegments = true, + ReplicationFactor = 1 }.ToDictionary().Keys; typedKeys.ShouldAllBe(key => catalogKeys.Contains(key)); @@ -231,6 +237,27 @@ [Test] [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))] + public async Task given_replication_factor_when_creating_topic_should_echo_it_and_refuse_zero(Protocol protocol) + { + var client = await Fixture.CreateAuthenticatedClient(protocol); + var streamName = $"replicated-{Guid.NewGuid():N}"; + await client.CreateStreamAsync(streamName); + var streamId = Identifier.String(streamName); + + await client.CreateTopicAsync(streamId, "replicated", 1, new TopicOptions { ReplicationFactor = 1 }); + + var fetched = await client.GetTopicByIdAsync(streamId, Identifier.String("replicated")); + fetched.ShouldNotBeNull(); + var factor = fetched.Options!.Single(kv => kv.Key.AsString() == "replication_factor").Value; + factor.Kind.ShouldBe(HeaderKind.Uint32); + factor.ToUInt32().ShouldBe(1u); + var refused = await Should.ThrowAsync<IggyInvalidStatusCodeException>(client.CreateTopicAsync(streamId, + "unreplicated", 1, new TopicOptions { ReplicationFactor = 0 })); + refused.StatusCode.ShouldBe(InvalidReplicationFactor); + } + + [Test] + [MethodDataSource<IggyServerFixture>(nameof(IggyServerFixture.ProtocolData))] public async Task given_stream_key_when_creating_should_read_matching_projection(Protocol protocol) { var client = await Fixture.CreateAuthenticatedClient(protocol);
diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs index 8285368..00f2859 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs
@@ -92,6 +92,7 @@ response.Name.ShouldBe("Get Topic"); response.CompressionAlgorithm.ShouldBe(CompressionAlgorithm.Gzip); response.Partitions!.Count().ShouldBe(2); + response.Partitions!.ShouldAllBe(partition => partition.Members.Count > 0); response.MessageExpiry.ShouldBe(TimeSpan.FromMinutes(10)); response.Size.ShouldBe(0u); response.PartitionsCount.ShouldBe(2u);
diff --git a/foreign/csharp/Iggy_SDK/Contracts/OptionKeys.cs b/foreign/csharp/Iggy_SDK/Contracts/OptionKeys.cs index 1bbe698..e344095 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/OptionKeys.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/OptionKeys.cs
@@ -67,7 +67,8 @@ new("partition_resize_policy", HeaderKind.String, OptionFlags.None), new("messages_required_to_save", HeaderKind.Uint32, OptionFlags.None), new("size_of_messages_required_to_save", HeaderKind.Uint64, OptionFlags.None), - new("preallocate_segments", HeaderKind.Bool, OptionFlags.None) + new("preallocate_segments", HeaderKind.Bool, OptionFlags.None), + new("replication_factor", HeaderKind.Uint32, OptionFlags.None) ]; private static readonly OptionDescriptor[] UserKeys = @@ -115,6 +116,7 @@ TopicOptionKey.MessagesRequiredToSave => "messages_required_to_save", TopicOptionKey.SizeOfMessagesRequiredToSave => "size_of_messages_required_to_save", TopicOptionKey.PreallocateSegments => "preallocate_segments", + TopicOptionKey.ReplicationFactor => "replication_factor", _ => throw new ArgumentOutOfRangeException(nameof(key), key, "Unknown topic option key") }); }
diff --git a/foreign/csharp/Iggy_SDK/Contracts/PartitionResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/PartitionResponse.cs index 1095a2b..cd20ffc 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/PartitionResponse.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/PartitionResponse.cs
@@ -59,4 +59,15 @@ /// </summary> [JsonConverter(typeof(SizeConverter))] public required ulong Size { get; init; } + + /// <summary> + /// Node ids that store the partition, in member order. + /// </summary> + public IReadOnlyList<byte> Members { get; init; } = []; + + /// <summary> + /// Node ids among <see cref="Members" /> that the partition's primary last reported in sync and that are up. + /// Empty until the primary first reports. + /// </summary> + public IReadOnlyList<byte> InSyncMembers { get; init; } = []; }
diff --git a/foreign/csharp/Iggy_SDK/Contracts/TopicOptionKey.cs b/foreign/csharp/Iggy_SDK/Contracts/TopicOptionKey.cs index d227af6..085ba6c 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/TopicOptionKey.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/TopicOptionKey.cs
@@ -54,5 +54,8 @@ SizeOfMessagesRequiredToSave, /// <summary><c>preallocate_segments</c>, create-only.</summary> - PreallocateSegments + PreallocateSegments, + + /// <summary><c>replication_factor</c>, create-only.</summary> + ReplicationFactor }
diff --git a/foreign/csharp/Iggy_SDK/Contracts/TopicOptions.cs b/foreign/csharp/Iggy_SDK/Contracts/TopicOptions.cs index 48e8ed5..cc9400a 100644 --- a/foreign/csharp/Iggy_SDK/Contracts/TopicOptions.cs +++ b/foreign/csharp/Iggy_SDK/Contracts/TopicOptions.cs
@@ -88,6 +88,18 @@ public bool? PreallocateSegments { get; init; } /// <summary> + /// Number of nodes that store each partition, 1 to 32 and at most the cluster's node count. Null lets the + /// server's <c>[cluster] default_replication_factor</c> apply: every node, unless an operator set a number, + /// which is lowered to the node count when the cluster is smaller. Fixed at creation. + /// </summary> + /// <remarks> + /// Two replicas keep two copies but stop accepting writes while either node is down, like Kafka with + /// <c>min.insync.replicas=2</c>. The server refuses an out-of-range value with error code 2018, + /// <c>invalid_replication_factor</c>. + /// </remarks> + public uint? ReplicationFactor { get; init; } + + /// <summary> /// Option keys with no property of their own, keyed by option name, for keys newer than this build. A /// property above wins on collision, and the server refuses a key outside its catalog by name. A raw /// durability is kept unless the matching property asks for persisted durability. @@ -161,6 +173,11 @@ options[Name(TopicOptionKey.PreallocateSegments)] = HeaderValue.FromBool(preallocateSegments); } + if (ReplicationFactor is { } replicationFactor) + { + options[Name(TopicOptionKey.ReplicationFactor)] = HeaderValue.FromUInt32(replicationFactor); + } + return options; }
diff --git a/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj b/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj index fe043fc..28dad14 100644 --- a/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj +++ b/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj
@@ -26,7 +26,7 @@ <TargetFrameworks>net8.0;net10.0</TargetFrameworks> <AssemblyName>Apache.Iggy</AssemblyName> <RootNamespace>Apache.Iggy</RootNamespace> - <Version>0.9.1-edge.6</Version> + <Version>0.9.1-edge.8</Version> <GenerateDocumentationFile>true</GenerateDocumentationFile> </PropertyGroup>
diff --git a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs index c66c916..3e792dd 100644 --- a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs
@@ -40,6 +40,8 @@ private const int CACHE_METRICS_ENTRY_SIZE = 32; private const int OPEN_FILES_FIELDS_SIZE = 8 + 8; private const int MIN_TOPIC_SIZE = 33 + 4 + 4; + private const int PARTITION_COUNTERS_SIZE = 40 + PartitionContext.ENCODED_SIZE; + private const int MIN_PARTITION_SIZE = PARTITION_COUNTERS_SIZE + 1 + 1; private const int MIN_OPTION_SPEC_SIZE = 1 + 1 + 1 + 4 + 4; private const int CLUSTER_NODE_TAIL_SIZE = 4 * 2 + 1 + 1; private const int MIN_CLUSTER_NODE_SIZE = 1 + 4 + 4 + CLUSTER_NODE_TAIL_SIZE; @@ -1001,10 +1003,11 @@ internal static TopicResponse MapTopic(ReadOnlySpan<byte> payload) { var (topic, position) = MapToTopic(payload, 0); - List<PartitionResponse> partitions = new(); - var length = payload.Length; + var partitionsCount = ValidatedCollectionSize(topic.PartitionsCount, payload.Length - position, + MIN_PARTITION_SIZE, "Partitions count"); + List<PartitionResponse> partitions = new(partitionsCount); - while (position < length) + for (var i = 0; i < partitionsCount; i++) { var (partition, readBytes) = MapToPartition(payload, position); partitions.Add(partition); @@ -1057,25 +1060,56 @@ private static (PartitionResponse partition, int readBytes) MapToPartition(ReadOnlySpan<byte> payload, int position) { + if (payload.Length - position < MIN_PARTITION_SIZE) + { + throw new MalformedResponseException($"Partition at byte {position} is truncated."); + } + var id = BinaryPrimitives.ReadUInt32LittleEndian(payload[position..(position + 4)]); var createdAt = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 4)..(position + 12)]); var segmentsCount = BinaryPrimitives.ReadUInt32LittleEndian(payload[(position + 12)..(position + 16)]); var currentOffset = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 16)..(position + 24)]); var sizeBytes = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 24)..(position + 32)]); var messagesCount = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 32)..(position + 40)]); - var readBytes = 40 + PartitionContext.ENCODED_SIZE; + var context = MapPartitionContext(payload[(position + 40)..]); + var end = position + PARTITION_COUNTERS_SIZE; + var members = ReadNodeIds(payload, ref end, "Partition members"); + var inSyncMembers = ReadNodeIds(payload, ref end, "Partition in-sync members"); return ( new PartitionResponse { - Context = MapPartitionContext(payload[(position + 40)..]), + Context = context, Id = id, SegmentsCount = segmentsCount, CurrentOffset = currentOffset, Size = sizeBytes, CreatedAt = DateTimeOffsetUtils.FromUnixTimeMicroSeconds(createdAt).LocalDateTime, - MessagesCount = messagesCount - }, readBytes); + MessagesCount = messagesCount, + Members = members, + InSyncMembers = inSyncMembers + }, end - position); + } + + private static byte[] ReadNodeIds(ReadOnlySpan<byte> payload, ref int position, string field) + { + if (position >= payload.Length) + { + throw new MalformedResponseException($"{field} count at byte {position} is missing."); + } + + var count = payload[position]; + position += 1; + var remaining = payload.Length - position; + if (count > remaining) + { + throw new MalformedResponseException( + $"{field} count {count} exceeds remaining payload of {remaining} bytes."); + } + + var nodeIds = payload.Slice(position, count).ToArray(); + position += count; + return nodeIds; } internal static List<ConsumerGroupResponse> MapConsumerGroups(ReadOnlySpan<byte> payload)
diff --git a/foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs b/foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs index 448f4a2..587fafc 100644 --- a/foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs +++ b/foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs
@@ -35,7 +35,7 @@ internal const int PROTOCOL_VERSION_MAJOR = 0; internal const int PROTOCOL_VERSION_MINOR = 12; - internal const int PROTOCOL_VERSION_PATCH = 1; + internal const int PROTOCOL_VERSION_PATCH = 2; /// <summary>Packed protocol version: <c>major << 20 | minor << 10 | patch</c>, 10 bits each.</summary> internal const uint PROTOCOL_VERSION =
diff --git a/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpPartitionContextTests.cs b/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpPartitionContextTests.cs index bd6c960..79a1be0 100644 --- a/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpPartitionContextTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/ClientTests/HttpPartitionContextTests.cs
@@ -139,6 +139,23 @@ Assert.Empty(handler.PollQuery); } + /// <summary>A server older than the node lists omits them, which reads as empty.</summary> + [Fact] + public async Task given_topic_details_with_node_lists_when_reading_the_topic_should_map_them() + { + var handler = new TopicHandler(Original); + var client = Client(handler); + + var topic = await client.GetTopicByIdAsync(Identifier.Numeric(1), Identifier.Numeric(2), + TestContext.Current.CancellationToken); + + var partitions = topic!.Partitions!.ToList(); + Assert.Empty(partitions[0].Members); + Assert.Empty(partitions[0].InSyncMembers); + Assert.Equal(new byte[] { 0, 1, 2 }, partitions[1].Members); + Assert.Equal(new byte[] { 0 }, partitions[1].InSyncMembers); + } + private static HttpMessageStream Client(TopicHandler handler) { return new HttpMessageStream(new HttpClient(handler) { BaseAddress = new Uri("http://localhost") }); @@ -242,7 +259,9 @@ "current_offset": 0, "size": "0 B", "messages_count": 0, - "context": {{ContextJson(context)}} + "context": {{ContextJson(context)}}, + "members": [0, 1, 2], + "in_sync_members": [0] } ] }
diff --git a/foreign/csharp/Iggy_SDK_Tests/Fixtures/resource_options.json b/foreign/csharp/Iggy_SDK_Tests/Fixtures/resource_options.json index 202888f..3911499 100644 --- a/foreign/csharp/Iggy_SDK_Tests/Fixtures/resource_options.json +++ b/foreign/csharp/Iggy_SDK_Tests/Fixtures/resource_options.json
@@ -16,18 +16,18 @@ }, "constants": { "protocol_version": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "protocol_version_min": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "command_codes": { "create_stream": { @@ -343,6 +343,21 @@ "default": "false" }, { + "key": "replication_factor", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": null + }, + { "key": "encryption_key", "scopes": [ "stream" @@ -439,7 +454,7 @@ { "scope": "topic", "request_hex": "01", - "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "response_hex": "0c000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465127265706c69636174696f6e5f666163746f720b00000000007b0100004e6f64657320746861742073746f7265206561636820706172746974696f6e2c20312e2e3d333220616e64206174206d6f73742074686520636c75737465722773206e6f646520636f756e742e204f6d69747465642c20746865207365727665722773205b636c75737465725d2064656661756c745f7265706c69636174696f6e5f666163746f72206170706c6965733a206576657279206e6f646520756e6c65737320616e206f70657261746f72207365742061206e756d6265722e204120777269746520636f6d6d697473206f6e20612071756f72756d206f66207468657365206e6f6465732c20736f2032206b656570732074776f20636f70696573206275742073746f70732077726974696e67207768696c652065697468657220697320646f776e2c20616e642031206b65657073206f6e6520636f70792074686174207265706c696361746564206475726162696c6974792061636b6e6f776c6564676573206265666f72652069742072656163686573206469736b", "entries": [ { "key": "name", @@ -506,6 +521,12 @@ "kind": "bool", "flags": 0, "default": "false" + }, + { + "key": "replication_factor", + "kind": "uint32", + "flags": 0, + "default": null } ] },
diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs index d4020ca..736eb19 100644 --- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs
@@ -41,6 +41,12 @@ public sealed class BinaryMapper { + /// <summary> + /// The partition record the Rust encoder pins in get_topic.rs, so every SDK decodes the same bytes. + /// </summary> + private const string PinnedPartitionHex = + "01000000004cf1238e0100000200000063000000000000000004000000000000640000000000000017000000000000000000000000000000650000000000000003010304020104"; + [Theory] [InlineData(null)] [InlineData("null")] @@ -279,17 +285,22 @@ sizeBytes, messagesCount, createdAt2, BinaryFactory.CreateOptionEntry(2, "message_expiry", 12, BitConverter.GetBytes((ulong)messageExpiry)), BinaryFactory.CreateOptionEntry(2, "max_topic_size", 12, BitConverter.GetBytes(maxTopicSize))); - - var combinedPayload = new byte[topicPayload.Length]; - topicPayload.CopyTo(combinedPayload.AsSpan()); + byte[] payload = + [ + ..topicPayload, + ..Enumerable.Range(0, (int)partitionsCount).SelectMany(partitionId => + BinaryFactory.CreatePartitionPayload((uint)partitionId, createdAt2, 1, 0, 0, 0, + new PartitionContext(1, 0, 2), [0], [0])) + ]; // Act - var response = Mappers.BinaryMapper.MapTopic(combinedPayload); + var response = Mappers.BinaryMapper.MapTopic(payload); // Assert Assert.NotNull(response); Assert.Equal(messagesCount, response.MessagesCount); Assert.Equal(partitionsCount, response.PartitionsCount); + Assert.Equal((int)partitionsCount, response.Partitions!.Count()); Assert.Equal(sizeBytes, response.Size); Assert.Equal(topicId, response.Id); Assert.Equal(topicName, response.Name); @@ -301,14 +312,13 @@ [Fact] public void MapTopic_ReadsAPartitionRecordWithItsContext() { - var (topicId, partitionsCount, topicName, _, sizeBytes, messagesCount, createdAt, _) = + var (topicId, _, topicName, _, sizeBytes, messagesCount, createdAt, _) = TopicFactory.CreateTopicResponseFields(); var context = new PartitionContext(17, 9, 52); byte[] payload = [ - ..BinaryFactory.CreateTopicPayload(topicId, partitionsCount, topicName, sizeBytes, messagesCount, - createdAt), - ..BinaryFactory.CreatePartitionPayload(3, createdAt, 7, 101, 2048, 102, context) + ..BinaryFactory.CreateTopicPayload(topicId, 1, topicName, sizeBytes, messagesCount, createdAt), + ..BinaryFactory.CreatePartitionPayload(3, createdAt, 7, 101, 2048, 102, context, [0, 1, 2], [0]) ]; var response = Mappers.BinaryMapper.MapTopic(payload); @@ -324,18 +334,120 @@ } [Fact] + public void MapTopic_DecodesThePinnedPartitionVector() + { + var partitionBytes = Convert.FromHexString(PinnedPartitionHex); + byte[] payload = [.. PinnedTopicHeader(1), .. partitionBytes]; + + var response = Mappers.BinaryMapper.MapTopic(payload); + + Assert.Equal(71, partitionBytes.Length); + Assert.Equal(partitionBytes, PinnedPartition(1, [1, 3, 4], [1, 4])); + var partition = Assert.Single(response.Partitions!); + Assert.Equal(1u, partition.Id); + Assert.Equal(DateTimeOffset.FromUnixTimeMilliseconds(1_710_000_000), partition.CreatedAt); + Assert.Equal(2u, partition.SegmentsCount); + Assert.Equal(99ul, partition.CurrentOffset); + Assert.Equal(1024ul, partition.Size); + Assert.Equal(100ul, partition.MessagesCount); + Assert.Equal(new PartitionContext(23, 0, 101), partition.Context); + Assert.Equal(new byte[] { 1, 3, 4 }, partition.Members); + Assert.Equal(new byte[] { 1, 4 }, partition.InSyncMembers); + } + + [Fact] + public void MapTopic_DecodesAPartitionWithEmptyNodeLists() + { + var pinned = Convert.FromHexString(PinnedPartitionHex); + byte[] partitionBytes = [.. pinned[..64], 0, 0]; + byte[] payload = [.. PinnedTopicHeader(1), .. partitionBytes]; + + var response = Mappers.BinaryMapper.MapTopic(payload); + + Assert.Equal(66, partitionBytes.Length); + var partition = Assert.Single(response.Partitions!); + Assert.Equal(1u, partition.Id); + Assert.Equal(new PartitionContext(23, 0, 101), partition.Context); + Assert.Empty(partition.Members); + Assert.Empty(partition.InSyncMembers); + } + + [Fact] + public void MapTopic_AdvancesByTheLengthEachPartitionTakes() + { + byte[] payload = + [ + ..PinnedTopicHeader(3), + ..PinnedPartition(1, [1, 3, 4], [1, 4]), + ..PinnedPartition(2, [], []), + ..PinnedPartition(3, [0, 1, 2], [0]) + ]; + + var response = Mappers.BinaryMapper.MapTopic(payload); + + var partitions = response.Partitions!.ToList(); + Assert.Equal([1u, 2u, 3u], partitions.Select(partition => partition.Id)); + Assert.All(partitions, partition => Assert.Equal(100ul, partition.MessagesCount)); + Assert.All(partitions, partition => Assert.Equal(new PartitionContext(23, 0, 101), partition.Context)); + Assert.Equal(new byte[] { 1, 3, 4 }, partitions[0].Members); + Assert.Equal(new byte[] { 1, 4 }, partitions[0].InSyncMembers); + Assert.Empty(partitions[1].Members); + Assert.Empty(partitions[1].InSyncMembers); + Assert.Equal(new byte[] { 0, 1, 2 }, partitions[2].Members); + Assert.Equal(new byte[] { 0 }, partitions[2].InSyncMembers); + } + + [Fact] + public void MapTopic_WithAPartitionCutShort_Throws() + { + var header = PinnedTopicHeader(1); + byte[] payload = [.. header, .. Convert.FromHexString(PinnedPartitionHex)]; + + for (var length = header.Length; length < payload.Length; length++) + { + var truncated = payload[..length]; + Assert.Throws<MalformedResponseException>(() => Mappers.BinaryMapper.MapTopic(truncated)); + } + } + + [Fact] + public void MapTopic_WithFewerPartitionsThanItsCount_Throws() + { + byte[] payload = + [ + ..PinnedTopicHeader(2), + ..PinnedPartition(1, [1, 3, 4], [1, 4]), + ..PinnedPartition(2, [1, 3, 4], [1, 4])[..70] + ]; + + Assert.Throws<MalformedResponseException>(() => Mappers.BinaryMapper.MapTopic(payload)); + } + + private static byte[] PinnedTopicHeader(uint partitionsCount) + { + return BinaryFactory.CreateTopicPayload(7, partitionsCount, "replicated", 2048, 300, 1_710_000_000_000); + } + + /// <summary>The counters of <see cref="PinnedPartitionHex" /> under its own id and node lists.</summary> + private static byte[] PinnedPartition(uint id, byte[] members, byte[] inSyncMembers) + { + return BinaryFactory.CreatePartitionPayload(id, 1_710_000_000_000, 2, 99, 1024, 100, + new PartitionContext(23, 0, 101), members, inSyncMembers); + } + + [Fact] public void MapTopic_WithAnOptionOfAnUnknownKind_KeepsTheOtherEntries() { // Arrange: an option kind a newer server may introduce, between two this build knows. const byte stringKind = 2; const byte unknownKind = 200; - var (topicId, partitionsCount, topicName, _, sizeBytes, messagesCount, createdAt, _) = + var (topicId, _, topicName, _, sizeBytes, messagesCount, createdAt, _) = TopicFactory.CreateTopicResponseFields(); var options = new List<byte>(); options.AddRange(BinaryFactory.CreateOptionEntry(stringKind, "segment_size", stringKind, "1GB"u8.ToArray())); options.AddRange(BinaryFactory.CreateOptionEntry(stringKind, "future_option", unknownKind, [0xAA, 0xBB])); options.AddRange(BinaryFactory.CreateOptionEntry(stringKind, "preallocate_segments", 3, [1])); - var topicPayload = BinaryFactory.CreateTopicPayload(topicId, partitionsCount, topicName, + var topicPayload = BinaryFactory.CreateTopicPayload(topicId, 0, topicName, sizeBytes, messagesCount, createdAt, options.ToArray()); // Act
diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/OptionsBlockGoldenVectorTests.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/OptionsBlockGoldenVectorTests.cs index 5d9bec2..8a94d09 100644 --- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/OptionsBlockGoldenVectorTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/OptionsBlockGoldenVectorTests.cs
@@ -100,7 +100,7 @@ BinaryPrimitives.WriteUInt32LittleEndian(payload, 1); BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(4), 1750000000000000); - BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(12), 1); + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(12), 0); BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(16), 0); BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(24), 0); payload[32] = (byte)name.Length;
diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/ResourceOptionsTests.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/ResourceOptionsTests.cs index fec5f12..f722573 100644 --- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/ResourceOptionsTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/ResourceOptionsTests.cs
@@ -509,7 +509,8 @@ PartitionResizePolicy = PartitionResizePolicy.Fixed, MessagesRequiredToSave = 64, SizeOfMessagesRequiredToSave = 4096, - PreallocateSegments = true + PreallocateSegments = true, + ReplicationFactor = 3 }.ToDictionary().Keys.ToHashSet()); Assert.Equal(updateKeys, new TopicUpdate {
diff --git a/foreign/csharp/Iggy_SDK_Tests/UtilityTests/TopicOptionsTests.cs b/foreign/csharp/Iggy_SDK_Tests/UtilityTests/TopicOptionsTests.cs index 73ae30a..a3cbc2d 100644 --- a/foreign/csharp/Iggy_SDK_Tests/UtilityTests/TopicOptionsTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/UtilityTests/TopicOptionsTests.cs
@@ -31,10 +31,11 @@ Durability = Apache.Iggy.Enums.Durability.Persisted, MessagesRequiredToSave = 1024, SizeOfMessagesRequiredToSave = 1048576, - PreallocateSegments = false + PreallocateSegments = false, + ReplicationFactor = 3 }.ToDictionary(); - Assert.Equal(6, options.Count); + Assert.Equal(7, options.Count); Assert.Equal(HeaderKind.Uint64, options["segment_size"].Kind); Assert.Equal(134217728UL, options["segment_size"].ToUInt64()); @@ -50,6 +51,9 @@ Assert.Equal(HeaderKind.Bool, options["preallocate_segments"].Kind); Assert.False(options["preallocate_segments"].ToBool()); + + Assert.Equal(HeaderKind.Uint32, options["replication_factor"].Kind); + Assert.Equal(3U, options["replication_factor"].ToUInt32()); } [Fact]
diff --git a/foreign/csharp/Iggy_SDK_Tests/Utils/BinaryFactory.cs b/foreign/csharp/Iggy_SDK_Tests/Utils/BinaryFactory.cs index 794008e..83611f5 100644 --- a/foreign/csharp/Iggy_SDK_Tests/Utils/BinaryFactory.cs +++ b/foreign/csharp/Iggy_SDK_Tests/Utils/BinaryFactory.cs
@@ -133,11 +133,15 @@ return payload; } - /// <summary>One partition record of a topic details reply: 40 bytes of counters, then the 24-byte context.</summary> + /// <summary> + /// One partition record of a topic details reply: 40 bytes of counters, the 24-byte context, then the + /// members and the in-sync members, each a u8 count followed by that many u8 node ids. + /// </summary> internal static byte[] CreatePartitionPayload(uint id, ulong createdAt, uint segmentsCount, ulong currentOffset, - ulong sizeBytes, ulong messagesCount, PartitionContext context) + ulong sizeBytes, ulong messagesCount, PartitionContext context, byte[] members, byte[] inSyncMembers) { - var payload = new byte[40 + PartitionContext.ENCODED_SIZE]; + var countersSize = 40 + PartitionContext.ENCODED_SIZE; + var payload = new byte[countersSize + 1 + members.Length + 1 + inSyncMembers.Length]; BinaryPrimitives.WriteUInt32LittleEndian(payload, id); BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(4), createdAt); BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(12), segmentsCount); @@ -147,6 +151,11 @@ BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(40), context.Incarnation); BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(48), context.OwnerGeneration); BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(56), context.MetadataOp); + payload[countersSize] = (byte)members.Length; + members.CopyTo(payload.AsSpan(countersSize + 1)); + var inSyncPosition = countersSize + 1 + members.Length; + payload[inSyncPosition] = (byte)inSyncMembers.Length; + inSyncMembers.CopyTo(payload.AsSpan(inSyncPosition + 1)); return payload; }
diff --git a/foreign/csharp/README.md b/foreign/csharp/README.md index b5df3d2..207bfdd 100644 --- a/foreign/csharp/README.md +++ b/foreign/csharp/README.md
@@ -28,7 +28,7 @@ from the same checkout. The SDK targets .NET 8 and .NET 10. Repository examples require .NET 10. `0.9.0` includes the independent message and consumer-offset durability options. -This checkout uses binary protocol 0.12.1. Servers and applicable SDKs must deploy together. Mixed versions and +This checkout uses binary protocol 0.12.2. Servers and applicable SDKs must deploy together. Mixed versions and rolling upgrades are unsupported. This checkout requires fresh storage and refuses older data directories without modification. `--fresh` deletes existing data in the configured directory. See the [server recovery guide](../../core/server/README.md#upgrade-recovery).
diff --git a/foreign/go/binary_serialization/binary_response_deserializer.go b/foreign/go/binary_serialization/binary_response_deserializer.go index 080bf74..463a16c 100644 --- a/foreign/go/binary_serialization/binary_response_deserializer.go +++ b/foreign/go/binary_serialization/binary_response_deserializer.go
@@ -18,6 +18,7 @@ package binaryserialization import ( + "bytes" "encoding/binary" "errors" "fmt" @@ -495,10 +496,11 @@ return &iggcon.TopicDetails{}, err } - partitions := make([]iggcon.PartitionContract, 0) - length := len(payload) - - for position < length { + // The header's count delimits the entries, so a body short of it is + // refused rather than read as fewer partitions. + partitions := make([]iggcon.PartitionContract, 0, + boundedCapacity(topic.PartitionsCount, len(payload)-position, partitionMinimumSize)) + for i := uint32(0); i < topic.PartitionsCount; i++ { partition, readBytes, err := DeserializePartition(payload, position) if err != nil { return nil, err @@ -615,13 +617,20 @@ return binary.LittleEndian.Uint64(value.Value) } +// partitionCountersSize covers id + created_at + segments_count + +// current_offset + size_bytes + messages_count + context. +const partitionCountersSize = 4 + 8 + 4 + 8 + 8 + 8 + iggcon.PartitionContextSize // 64 bytes + +// partitionMinimumSize is a partition whose members and in-sync node lists +// are both empty: the counters and one u8 count per list. +const partitionMinimumSize = partitionCountersSize + 1 + 1 // 66 bytes + func DeserializePartition(payload []byte, position int) (iggcon.PartitionContract, int, error) { - const partitionSize = 40 + iggcon.PartitionContextSize - if position < 0 || position > len(payload) || len(payload)-position < partitionSize { + if position < 0 || position > len(payload) || len(payload)-position < partitionMinimumSize { return iggcon.PartitionContract{}, 0, fmt.Errorf("truncated partition response") } var partitionContext iggcon.PartitionContext - if err := partitionContext.UnmarshalBinary(payload[position+40 : position+partitionSize]); err != nil { + if err := partitionContext.UnmarshalBinary(payload[position+40 : position+partitionCountersSize]); err != nil { return iggcon.PartitionContract{}, 0, err } id := binary.LittleEndian.Uint32(payload[position : position+4]) @@ -630,7 +639,14 @@ currentOffset := binary.LittleEndian.Uint64(payload[position+16 : position+24]) sizeBytes := binary.LittleEndian.Uint64(payload[position+24 : position+32]) messagesCount := binary.LittleEndian.Uint64(payload[position+32 : position+40]) - readBytes := partitionSize + members, membersEnd, err := deserializeNodeIds(payload, position+partitionCountersSize, "partition members") + if err != nil { + return iggcon.PartitionContract{}, 0, err + } + inSyncMembers, end, err := deserializeNodeIds(payload, membersEnd, "partition in-sync members") + if err != nil { + return iggcon.PartitionContract{}, 0, err + } partition := iggcon.PartitionContract{ Context: partitionContext, @@ -640,9 +656,26 @@ CurrentOffset: currentOffset, SizeBytes: sizeBytes, MessagesCount: messagesCount, + Members: members, + InSyncMembers: inSyncMembers, } - return partition, readBytes, nil + return partition, end - position, nil +} + +// deserializeNodeIds reads a u8 count followed by that many u8 node ids and +// returns the position past the list. +func deserializeNodeIds(payload []byte, position int, field string) ([]uint8, int, error) { + if position >= len(payload) { + return nil, 0, fmt.Errorf("truncated %s count at offset %d", field, position) + } + count := int(payload[position]) + start := position + 1 + if len(payload)-start < count { + return nil, 0, fmt.Errorf("truncated %s at offset %d: need %d bytes, got %d", + field, start, count, len(payload)-start) + } + return bytes.Clone(payload[start : start+count]), start + count, nil } func DeserializeConsumerGroups(payload []byte) []iggcon.ConsumerGroup {
diff --git a/foreign/go/binary_serialization/vsr_response_deserializer_test.go b/foreign/go/binary_serialization/vsr_response_deserializer_test.go index 63eca92..80849f4 100644 --- a/foreign/go/binary_serialization/vsr_response_deserializer_test.go +++ b/foreign/go/binary_serialization/vsr_response_deserializer_test.go
@@ -571,3 +571,128 @@ _, _, err = DeserializeToTopic(derivedLies, 0) assert.Error(t, err, "a derived options length of 0xFFFFFFFF must not panic or pass") } + +// pinnedPartition is the partition the Rust encoder pins in +// core/binary_protocol/src/responses/topics/get_topic.rs: id 1, members +// [1 3 4] and in-sync members [1 4]. +const pinnedPartition = "01000000004cf1238e0100000200000063000000000000000004000000000000640000000000000017000000000000000000000000000000650000000000000003010304020104" + +// encodePartition writes the pinned partition's counters under the given id +// and node lists. +func encodePartition(id uint32, members, inSyncMembers []uint8) []byte { + entry := binary.LittleEndian.AppendUint32(nil, id) + entry = binary.LittleEndian.AppendUint64(entry, 1710000000000) + entry = binary.LittleEndian.AppendUint32(entry, 2) + entry = binary.LittleEndian.AppendUint64(entry, 99) + entry = binary.LittleEndian.AppendUint64(entry, 1024) + entry = binary.LittleEndian.AppendUint64(entry, 100) + entry = binary.LittleEndian.AppendUint64(entry, 23) + entry = binary.LittleEndian.AppendUint64(entry, 0) + entry = binary.LittleEndian.AppendUint64(entry, 101) + entry = append(entry, uint8(len(members))) + entry = append(entry, members...) + entry = append(entry, uint8(len(inSyncMembers))) + return append(entry, inSyncMembers...) +} + +// encodeTopicHeader writes a topic named "t" with empty options blocks. +func encodeTopicHeader(partitionsCount uint32) []byte { + header := binary.LittleEndian.AppendUint32(nil, 9) + header = binary.LittleEndian.AppendUint64(header, 1710000000000) + header = binary.LittleEndian.AppendUint32(header, partitionsCount) + header = binary.LittleEndian.AppendUint64(header, 0) + header = binary.LittleEndian.AppendUint64(header, 0) + header = append(header, 1, 't') + header = binary.LittleEndian.AppendUint32(header, 0) + return binary.LittleEndian.AppendUint32(header, 0) +} + +func TestDeserializePartition_DecodesThePinnedVector(t *testing.T) { + payload, err := hex.DecodeString(pinnedPartition) + require.NoError(t, err) + require.Len(t, payload, 71) + + partition, readBytes, err := DeserializePartition(payload, 0) + require.NoError(t, err) + + assert.Equal(t, iggcon.PartitionContract{ + Context: iggcon.PartitionContext{Incarnation: 23, OwnerGeneration: 0, MetadataOp: 101}, + Id: 1, + MessagesCount: 100, + CreatedAt: 1710000000000, + SegmentsCount: 2, + CurrentOffset: 99, + SizeBytes: 1024, + Members: []uint8{1, 3, 4}, + InSyncMembers: []uint8{1, 4}, + }, partition) + assert.Equal(t, len(payload), readBytes) + assert.Equal(t, payload, encodePartition(1, []uint8{1, 3, 4}, []uint8{1, 4}), + "the test encoder must write the pinned bytes") +} + +func TestDeserializePartition_ReadsEmptyNodeListsAsTwoCountBytes(t *testing.T) { + payload := encodePartition(1, nil, nil) + require.Len(t, payload, 66) + + partition, readBytes, err := DeserializePartition(payload, 0) + require.NoError(t, err) + assert.Equal(t, 66, readBytes) + assert.Empty(t, partition.Members) + assert.Empty(t, partition.InSyncMembers) + assert.Equal(t, uint64(99), partition.CurrentOffset) +} + +func TestDeserializePartition_RejectsEveryTruncation(t *testing.T) { + payload, err := hex.DecodeString(pinnedPartition) + require.NoError(t, err) + + for length := 0; length < len(payload); length++ { + _, _, err := DeserializePartition(payload[:length], 0) + assert.Error(t, err, "a partition truncated to %d bytes must not decode", length) + } +} + +func TestDeserializeTopic_AdvancesByEachPartitionsEncodedLength(t *testing.T) { + payload := encodeTopicHeader(3) + payload = append(payload, encodePartition(1, []uint8{1, 3, 4}, []uint8{1, 4})...) + payload = append(payload, encodePartition(2, nil, nil)...) + payload = append(payload, encodePartition(3, []uint8{0, 1, 2}, []uint8{0})...) + + details, err := DeserializeTopic(payload) + require.NoError(t, err) + + require.Len(t, details.Partitions, 3) + assert.Equal(t, []uint32{1, 2, 3}, + []uint32{details.Partitions[0].Id, details.Partitions[1].Id, details.Partitions[2].Id}) + assert.Equal(t, []uint8{1, 3, 4}, details.Partitions[0].Members) + assert.Equal(t, []uint8{1, 4}, details.Partitions[0].InSyncMembers) + assert.Empty(t, details.Partitions[1].Members) + assert.Empty(t, details.Partitions[1].InSyncMembers) + assert.Equal(t, []uint8{0, 1, 2}, details.Partitions[2].Members) + assert.Equal(t, []uint8{0}, details.Partitions[2].InSyncMembers) + for _, partition := range details.Partitions { + assert.Equal(t, uint64(1710000000000), partition.CreatedAt, "partition %d", partition.Id) + assert.Equal(t, uint64(101), partition.Context.MetadataOp, "partition %d", partition.Id) + } +} + +func TestDeserializeTopic_RejectsABodyShortOfThePartitionsCount(t *testing.T) { + payload := encodeTopicHeader(2) + payload = append(payload, encodePartition(1, []uint8{0}, []uint8{0})...) + + _, err := DeserializeTopic(payload) + assert.Error(t, err, "a declared partition the body does not carry must surface") +} + +func TestDeserializeTopic_RejectsEveryTruncationOfThePartitions(t *testing.T) { + payload := encodeTopicHeader(2) + headerLength := len(payload) + payload = append(payload, encodePartition(1, []uint8{1, 3, 4}, []uint8{1, 4})...) + payload = append(payload, encodePartition(2, []uint8{2}, nil)...) + + for length := headerLength; length < len(payload); length++ { + _, err := DeserializeTopic(payload[:length]) + assert.Error(t, err, "a topic truncated to %d bytes must not decode", length) + } +}
diff --git a/foreign/go/client/tcp/tcp_core_test.go b/foreign/go/client/tcp/tcp_core_test.go index 8d1ebc2..9fa120b 100644 --- a/foreign/go/client/tcp/tcp_core_test.go +++ b/foreign/go/client/tcp/tcp_core_test.go
@@ -874,3 +874,14 @@ copy(body[nameLenOffset+1:], name) return body } + +// partitionDetailsBody is one GetTopic/CreateTopic partition entry: zeroed +// counters and context, then the two u8-counted node lists. +func partitionDetailsBody(id uint32, members, inSyncMembers []uint8) []byte { + body := binary.LittleEndian.AppendUint32(nil, id) + body = append(body, make([]byte, 8+4+8+8+8+iggcon.PartitionContextSize)...) + body = append(body, uint8(len(members))) + body = append(body, members...) + body = append(body, uint8(len(inSyncMembers))) + return append(body, inSyncMembers...) +}
diff --git a/foreign/go/client/tcp/tcp_topic_management_test.go b/foreign/go/client/tcp/tcp_topic_management_test.go index 1ec24b1..43faab5 100644 --- a/foreign/go/client/tcp/tcp_topic_management_test.go +++ b/foreign/go/client/tcp/tcp_topic_management_test.go
@@ -32,9 +32,12 @@ func TestCreateTopic_DecodesTheCreatedTopic(t *testing.T) { client, serverConn := newPipeClient(t) + body := append(resultSection(), topicDetailsBody(t, 3)...) + body = append(body, partitionDetailsBody(1, []uint8{0, 1, 2}, []uint8{0})...) + body = append(body, partitionDetailsBody(2, nil, nil)...) + body = append(body, partitionDetailsBody(3, []uint8{2, 0, 1}, []uint8{2, 0, 1})...) serve(serverConn, func(_ int, _ request) []byte { - return replyFrame(vsr.OperationCreateTopic, - append(resultSection(), topicDetailsBody(t, 3)...)) + return replyFrame(vsr.OperationCreateTopic, body) }) partitions := uint32(3) @@ -43,6 +46,14 @@ require.NoError(t, err) assert.Equal(t, uint32(3), topic.PartitionsCount) assert.Equal(t, "orders", topic.Name) + require.Len(t, topic.Partitions, 3) + assert.Equal(t, []uint8{0, 1, 2}, topic.Partitions[0].Members) + assert.Equal(t, []uint8{0}, topic.Partitions[0].InSyncMembers) + assert.Empty(t, topic.Partitions[1].Members) + assert.Empty(t, topic.Partitions[1].InSyncMembers) + assert.Equal(t, uint32(3), topic.Partitions[2].Id) + assert.Equal(t, []uint8{2, 0, 1}, topic.Partitions[2].Members) + assert.Equal(t, []uint8{2, 0, 1}, topic.Partitions[2].InSyncMembers) } func TestCreateTopic_RejectsInvalidArgumentsWithoutWriting(t *testing.T) {
diff --git a/foreign/go/contracts/partitions.go b/foreign/go/contracts/partitions.go index a12efca..90f51c8 100644 --- a/foreign/go/contracts/partitions.go +++ b/foreign/go/contracts/partitions.go
@@ -76,6 +76,11 @@ SegmentsCount uint32 `json:"segmentsCount"` CurrentOffset uint64 `json:"currentOffset"` SizeBytes uint64 `json:"sizeBytes"` + // Members are the node ids that store the partition, in member order. + Members []uint8 `json:"members"` + // InSyncMembers are the members the partition's primary last reported in + // sync and that are up. Empty until the primary first reports. + InSyncMembers []uint8 `json:"inSyncMembers"` } type PartitioningKind int
diff --git a/foreign/go/contracts/resource_options_test.go b/foreign/go/contracts/resource_options_test.go index cb49dce..b1a1096 100644 --- a/foreign/go/contracts/resource_options_test.go +++ b/foreign/go/contracts/resource_options_test.go
@@ -486,6 +486,7 @@ string(iggcon.TopicOptionMessagesRequiredToSave), string(iggcon.TopicOptionSizeOfMessagesRequiredToSave), string(iggcon.TopicOptionPreallocateSegments), + string(iggcon.TopicOptionReplicationFactor), }, "user": { string(iggcon.UserOptionUsername), @@ -561,6 +562,7 @@ "messages_required_to_save": {create: "MessagesRequiredToSave"}, "size_of_messages_required_to_save": {create: "SizeOfMessagesRequiredToSave"}, "preallocate_segments": {create: "PreallocateSegments"}, + "replication_factor": {create: "ReplicationFactor"}, }, "user": { "username": {update: "Username"},
diff --git a/foreign/go/contracts/testdata/resource_options.json b/foreign/go/contracts/testdata/resource_options.json index 202888f..3911499 100644 --- a/foreign/go/contracts/testdata/resource_options.json +++ b/foreign/go/contracts/testdata/resource_options.json
@@ -16,18 +16,18 @@ }, "constants": { "protocol_version": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "protocol_version_min": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "command_codes": { "create_stream": { @@ -343,6 +343,21 @@ "default": "false" }, { + "key": "replication_factor", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": null + }, + { "key": "encryption_key", "scopes": [ "stream" @@ -439,7 +454,7 @@ { "scope": "topic", "request_hex": "01", - "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "response_hex": "0c000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465127265706c69636174696f6e5f666163746f720b00000000007b0100004e6f64657320746861742073746f7265206561636820706172746974696f6e2c20312e2e3d333220616e64206174206d6f73742074686520636c75737465722773206e6f646520636f756e742e204f6d69747465642c20746865207365727665722773205b636c75737465725d2064656661756c745f7265706c69636174696f6e5f666163746f72206170706c6965733a206576657279206e6f646520756e6c65737320616e206f70657261746f72207365742061206e756d6265722e204120777269746520636f6d6d697473206f6e20612071756f72756d206f66207468657365206e6f6465732c20736f2032206b656570732074776f20636f70696573206275742073746f70732077726974696e67207768696c652065697468657220697320646f776e2c20616e642031206b65657073206f6e6520636f70792074686174207265706c696361746564206475726162696c6974792061636b6e6f776c6564676573206265666f72652069742072656163686573206469736b", "entries": [ { "key": "name", @@ -506,6 +521,12 @@ "kind": "bool", "flags": 0, "default": "false" + }, + { + "key": "replication_factor", + "kind": "uint32", + "flags": 0, + "default": null } ] },
diff --git a/foreign/go/contracts/topic_options.go b/foreign/go/contracts/topic_options.go index f1d417f..68a4ef2 100644 --- a/foreign/go/contracts/topic_options.go +++ b/foreign/go/contracts/topic_options.go
@@ -39,6 +39,7 @@ TopicOptionMessagesRequiredToSave TopicOptionKey = "messages_required_to_save" TopicOptionSizeOfMessagesRequiredToSave TopicOptionKey = "size_of_messages_required_to_save" TopicOptionPreallocateSegments TopicOptionKey = "preallocate_segments" + TopicOptionReplicationFactor TopicOptionKey = "replication_factor" ) // TopicOptions are the options CreateTopic may carry. A nil *TopicOptions @@ -72,6 +73,8 @@ SizeOfMessagesRequiredToSave *uint64 // PreallocateSegments, see PreallocateSegmentsOption. PreallocateSegments *bool + // ReplicationFactor, see ReplicationFactorOption. + ReplicationFactor *uint32 // Raw carries keys newer than this build. Raw []HeaderEntry } @@ -156,6 +159,17 @@ return boolOption(TopicOptionPreallocateSegments, enabled) } +// ReplicationFactorOption sets how many nodes store each partition of the +// topic: 1 to 32 and at most the cluster's node count. Omitted, the server's +// [cluster] default_replication_factor applies: every node unless an operator +// set a number. A write commits on a quorum of those nodes, so 2 keeps two +// copies but stops taking writes while either node is down, like Kafka with +// min.insync.replicas=2. A value out of range is refused with +// InvalidReplicationFactor. +func ReplicationFactorOption(nodes uint32) HeaderEntry { + return uint32Option(TopicOptionReplicationFactor, nodes) +} + func stringOption(key TopicOptionKey, value string) HeaderEntry { return HeaderEntry{ Key: HeaderKey{Kind: String, Value: []byte(key)},
diff --git a/foreign/go/contracts/topic_options_test.go b/foreign/go/contracts/topic_options_test.go index c812184..de7e8c0 100644 --- a/foreign/go/contracts/topic_options_test.go +++ b/foreign/go/contracts/topic_options_test.go
@@ -83,6 +83,13 @@ wantKind: Bool, wantValue: []byte{0}, }, + { + name: "replication factor", + entry: ReplicationFactorOption(3), + wantKey: "replication_factor", + wantKind: Uint32, + wantValue: []byte{3, 0, 0, 0}, + }, } for _, test := range tests { @@ -114,6 +121,7 @@ MessagesRequiredToSaveOption(7), SizeOfMessagesRequiredToSaveOption(4096), PreallocateSegmentsOption(false), + ReplicationFactorOption(3), } decoded, err := DeserializeHeaders(GetHeadersBytes(entries))
diff --git a/foreign/go/contracts/version.go b/foreign/go/contracts/version.go index 2345522..0f5e3d1 100644 --- a/foreign/go/contracts/version.go +++ b/foreign/go/contracts/version.go
@@ -17,4 +17,4 @@ package iggcon -const Version = "0.9.1-edge.6" +const Version = "0.9.1-edge.8"
diff --git a/foreign/go/internal/command/topic.go b/foreign/go/internal/command/topic.go index 7eaf7a4..2db2418 100644 --- a/foreign/go/internal/command/topic.go +++ b/foreign/go/internal/command/topic.go
@@ -135,6 +135,9 @@ if options.PreallocateSegments != nil { typed = append(typed, iggcon.PreallocateSegmentsOption(*options.PreallocateSegments)) } + if options.ReplicationFactor != nil { + typed = append(typed, iggcon.ReplicationFactorOption(*options.ReplicationFactor)) + } return typed, nil }
diff --git a/foreign/go/internal/command/topic_test.go b/foreign/go/internal/command/topic_test.go index a4707bf..53eeca9 100644 --- a/foreign/go/internal/command/topic_test.go +++ b/foreign/go/internal/command/topic_test.go
@@ -229,6 +229,7 @@ iggcon.MessagesRequiredToSaveOption(7), iggcon.SizeOfMessagesRequiredToSaveOption(4096), iggcon.PreallocateSegmentsOption(false), + iggcon.ReplicationFactorOption(3), }}, } @@ -261,6 +262,7 @@ {"messages_required_to_save", iggcon.Uint32, []byte{7, 0, 0, 0}}, {"size_of_messages_required_to_save", iggcon.Uint64, []byte{0, 16, 0, 0, 0, 0, 0, 0}}, {"preallocate_segments", iggcon.Bool, []byte{0}}, + {"replication_factor", iggcon.Uint32, []byte{3, 0, 0, 0}}, } if len(byKey) != len(want) { t.Fatalf("options block carries %d keys, want %d: %v", len(byKey), len(want), byKey) @@ -280,6 +282,47 @@ } } +func TestCreateTopic_ReplicationFactorFieldRidesTheBlockAsUint32(t *testing.T) { + streamId, err := iggcon.NewIdentifier(uint32(1)) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + replicationFactor := uint32(2) + command := CreateTopic{ + StreamId: streamId, + Name: "topic", + Options: iggcon.TopicOptions{ReplicationFactor: &replicationFactor}, + } + + serialized, err := command.MarshalBinary() + if err != nil { + t.Fatalf("failed to serialize CreateTopic: %v", err) + } + + const streamIdSize = 2 + 4 + optionsOffset := streamIdSize + 4 + 1 + len(command.Name) + options, err := iggcon.DeserializeHeaders(serialized[optionsOffset:]) + if err != nil { + t.Fatalf("options block is not valid TLV: %v", err) + } + var found bool + for _, entry := range options { + if string(entry.Key.Value) != "replication_factor" { + continue + } + found = true + if entry.Value.Kind != iggcon.Uint32 { + t.Errorf("replication_factor kind = %d, want Uint32", entry.Value.Kind) + } + if !bytes.Equal(entry.Value.Value, []byte{2, 0, 0, 0}) { + t.Errorf("replication_factor value = %v, want [2 0 0 0]", entry.Value.Value) + } + } + if !found { + t.Errorf("replication_factor is missing from the options block: %v", options) + } +} + func TestCreateTopic_NonStringOptionKeyIsRejected(t *testing.T) { streamId, err := iggcon.NewIdentifier(uint32(1)) if err != nil {
diff --git a/foreign/go/internal/command/update_resource.go b/foreign/go/internal/command/update_resource.go index 670e649..6669c3a 100644 --- a/foreign/go/internal/command/update_resource.go +++ b/foreign/go/internal/command/update_resource.go
@@ -69,6 +69,7 @@ {key: string(iggcon.TopicOptionMessagesRequiredToSave), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeTopic}}, {key: string(iggcon.TopicOptionSizeOfMessagesRequiredToSave), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeTopic}}, {key: string(iggcon.TopicOptionPreallocateSegments), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeTopic}}, + {key: string(iggcon.TopicOptionReplicationFactor), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeTopic}}, {key: string(iggcon.StreamOptionEncryptionKey), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeStream}}, {key: "encryption_key_wrapped", scopes: []iggcon.OptionsScope{iggcon.OptionsScopeStream}, readOnly: true}, {key: string(iggcon.StreamOptionEncryptionKeySource), scopes: []iggcon.OptionsScope{iggcon.OptionsScopeStream}, readOnly: true},
diff --git a/foreign/go/internal/vsr/register.go b/foreign/go/internal/vsr/register.go index bdc6dd5..0ddf6d3 100644 --- a/foreign/go/internal/vsr/register.go +++ b/foreign/go/internal/vsr/register.go
@@ -25,10 +25,10 @@ ) // ProtocolVersion is the packed semver of the wire contract this codec -// implements, 0.12.1 per core/binary_protocol/src/version.rs. Update it with +// implements, 0.12.2 per core/binary_protocol/src/version.rs. Update it with // the Rust constant during stable release preparation. Edge builds sharing // this number are not guaranteed to have compatible layouts. -var ProtocolVersion = packProtocolVersion(0, 12, 1) +var ProtocolVersion = packProtocolVersion(0, 12, 2) // packProtocolVersion packs a semver into the ten-bits-per-field layout the // register handshake carries.
diff --git a/foreign/go/internal/vsr/register_test.go b/foreign/go/internal/vsr/register_test.go index d94234d..a6cc2bc 100644 --- a/foreign/go/internal/vsr/register_test.go +++ b/foreign/go/internal/vsr/register_test.go
@@ -36,10 +36,10 @@ } func TestProtocolVersion_PacksTheWireContractSemver(t *testing.T) { - assert.Equal(t, uint32(12289), ProtocolVersion) + assert.Equal(t, uint32(12290), ProtocolVersion) assert.Equal(t, uint32(0), ProtocolVersion>>20) assert.Equal(t, uint32(12), (ProtocolVersion>>10)&0x3FF) - assert.Equal(t, uint32(1), ProtocolVersion&0x3FF) + assert.Equal(t, uint32(2), ProtocolVersion&0x3FF) } func TestSerializeLoginRegister_EncodesTheDeclaredLayout(t *testing.T) {
diff --git a/foreign/go/tests/e2e_test.go b/foreign/go/tests/e2e_test.go index 91991b6..ac96303 100644 --- a/foreign/go/tests/e2e_test.go +++ b/foreign/go/tests/e2e_test.go
@@ -528,13 +528,14 @@ // Values inside the bounds the catalog reports: a 512-byte multiple segment // size at the floor, a non-zero message threshold, preallocation off so - // nothing is reserved on disk. + // nothing is reserved on disk, and one replica, which fits any cluster. typed := []iggcon.HeaderEntry{ iggcon.SegmentSizeOption(1024 * 1024), iggcon.DurabilityOption(iggcon.DurabilityPersisted), iggcon.MessagesRequiredToSaveOption(7), iggcon.SizeOfMessagesRequiredToSaveOption(4096), iggcon.PreallocateSegmentsOption(false), + iggcon.ReplicationFactorOption(1), } specs, err := connected.DescribeOptions(ctx, iggcon.OptionsScopeTopic) @@ -583,6 +584,12 @@ }) require.Error(t, err) + // A partition needs at least one replica. + _, err = connected.CreateTopic(ctx, streamId, name+"-zero-replicas", &iggcon.TopicOptions{ + Raw: []iggcon.HeaderEntry{iggcon.ReplicationFactorOption(0)}, + }) + require.ErrorIs(t, err, ierror.ErrInvalidReplicationFactor) + // A segment size off the 512-byte grid is refused, but zero is not: it // leaves the key derived. _, err = connected.CreateTopic(ctx, streamId, name+"-odd-segment", &iggcon.TopicOptions{
diff --git a/foreign/java/gradle.properties b/foreign/java/gradle.properties index 4bd57b1..246e4ad 100644 --- a/foreign/java/gradle.properties +++ b/foreign/java/gradle.properties
@@ -15,5 +15,5 @@ # specific language governing permissions and limitations # under the License. -version=0.9.6-SNAPSHOT +version=0.9.8-SNAPSHOT group=org.apache.iggy
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java index e0a973b..8d76705 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java
@@ -45,7 +45,7 @@ * protocol version through its current version, including patch components. * Versions outside this range, including newer patches, are rejected. */ - static final int PROTOCOL_VERSION = 12 << 10 | 1; // 0.12.1 + static final int PROTOCOL_VERSION = 12 << 10 | 2; // 0.12.2 static final String SDK_NAME = "java-sdk";
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/partition/Partition.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/partition/Partition.java index 5eedb96..12ae580 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/partition/Partition.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/partition/Partition.java
@@ -20,7 +20,15 @@ package org.apache.iggy.partition; import java.math.BigInteger; +import java.util.List; +/** + * A partition as the server reports it. + * + * @param members node ids that store the partition, in member order + * @param inSyncMembers node ids among {@code members} that the partition's primary last reported in sync and that + * are up, empty until the primary first reports + */ public record Partition( Long id, BigInteger createdAt, @@ -28,4 +36,11 @@ BigInteger currentOffset, String size, BigInteger messagesCount, - PartitionContext context) {} + PartitionContext context, + List<Integer> members, + List<Integer> inSyncMembers) { + public Partition { + members = members == null ? List.of() : List.copyOf(members); + inSyncMembers = inSyncMembers == null ? List.of() : List.copyOf(inSyncMembers); + } +}
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java index 95b77f7..f4138fe 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java
@@ -105,6 +105,9 @@ // 33-byte fixed part + a one-character name (the server rejects an empty // one) + two u32 options-block length prefixes. private static final int MIN_TOPIC_BYTES = 42; + // Counters, context and the u8 counts of two empty node lists. + private static final int MIN_PARTITION_BYTES = + 2 * Integer.BYTES + 4 * Long.BYTES + PartitionContext.ENCODED_SIZE + 2; private static final int OPEN_FILES_FIELDS_BYTES = 2 * Long.BYTES; private BytesDeserializer() {} @@ -142,8 +145,11 @@ public static TopicDetails readTopicDetails(ByteBuf response) { var topic = readTopic(response); - List<Partition> partitions = new ArrayList<>(); - while (response.isReadable()) { + // Count-driven: a partition entry ends in variable-length node lists. + int partitionsCount = validatedCollectionSize( + topic.partitionsCount(), response.readableBytes(), MIN_PARTITION_BYTES, "Topic partitions count"); + List<Partition> partitions = new ArrayList<>(partitionsCount); + for (int i = 0; i < partitionsCount; i++) { partitions.add(readPartition(response)); } @@ -156,12 +162,19 @@ } public static Partition readPartition(ByteBuf response) { + if (response.readableBytes() < MIN_PARTITION_BYTES) { + throw new IggyMalformedResponseException("Partition needs at least " + MIN_PARTITION_BYTES + + " bytes, remaining payload has " + response.readableBytes()); + } var partitionId = response.readUnsignedIntLE(); var createdAt = readU64AsBigInteger(response); var segmentsCount = response.readUnsignedIntLE(); var currentOffset = readU64AsBigInteger(response); var size = readU64AsBigInteger(response); var messagesCount = readU64AsBigInteger(response); + var context = readPartitionContext(response); + var members = readNodeIds(response, "partition members"); + var inSyncMembers = readNodeIds(response, "partition in-sync members"); return new Partition( partitionId, createdAt, @@ -169,7 +182,9 @@ currentOffset, size.toString(), messagesCount, - readPartitionContext(response)); + context, + members, + inSyncMembers); } public static Topic readTopic(ByteBuf response) { @@ -890,6 +905,19 @@ return buffer.readCharSequence(toInt(length), StandardCharsets.UTF_8).toString(); } + private static List<Integer> readNodeIds(ByteBuf buffer, String field) { + if (!buffer.isReadable()) { + throw new IggyMalformedResponseException("Missing count for " + field); + } + int count = validatedCollectionSize( + buffer.readUnsignedByte(), buffer.readableBytes(), Byte.BYTES, field + " count"); + List<Integer> nodeIds = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + nodeIds.add((int) buffer.readUnsignedByte()); + } + return nodeIds; + } + static int validatedCollectionSize(long count, int readableBytes, int entryBytes, String field) { if (count > readableBytes / entryBytes) { throw new IggyMalformedResponseException(
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptionKey.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptionKey.java index 23e6d6e..a197bf7 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptionKey.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptionKey.java
@@ -41,7 +41,8 @@ PARTITION_RESIZE_POLICY("partition_resize_policy", 0), MESSAGES_REQUIRED_TO_SAVE("messages_required_to_save", 0), SIZE_OF_MESSAGES_REQUIRED_TO_SAVE("size_of_messages_required_to_save", 0), - PREALLOCATE_SEGMENTS("preallocate_segments", 0); + PREALLOCATE_SEGMENTS("preallocate_segments", 0), + REPLICATION_FACTOR("replication_factor", 0); private final String value; private final int flags;
diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptions.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptions.java index 4e18043..6b5f8ff 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptions.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/topic/TopicOptions.java
@@ -165,6 +165,19 @@ return set(TopicOptionKey.PREALLOCATE_SEGMENTS, HeaderValue.fromBool(enabled)); } + /** + * Number of nodes that store each partition, 1 to 32 and at most the cluster's node count. Left unset, + * the server's {@code [cluster] default_replication_factor} applies: every node unless an operator set a + * number. + * + * <p>Two replicas keep two copies but stop accepting writes while either node is down, like Kafka with + * {@code min.insync.replicas=2}. The server refuses an out-of-range value with + * {@code INVALID_REPLICATION_FACTOR}. + */ + public Builder replicationFactor(long replicas) { + return set(TopicOptionKey.REPLICATION_FACTOR, HeaderValue.fromUint32(replicas)); + } + /** An option this build has no setter for. A typed setter wins on collision. */ public Builder raw(String key, HeaderValue value) { raw.put(key, value);
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/TopicsClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/TopicsClientBaseTest.java index 74bd1c6..11f06ea 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/TopicsClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/TopicsClientBaseTest.java
@@ -117,6 +117,29 @@ } @Test + void givenReplicationFactorWhenCreatingTopicShouldEchoItAndRefuseZero() { + // when + var created = topicsClient.createTopic( + STREAM_NAME, + "replicated-topic", + 1L, + TopicOptions.builder().replicationFactor(1).build()); + + // then + var topic = topicsClient.getTopic(STREAM_NAME, TopicId.of(created.id())).orElseThrow(); + assertThat(topic.options().get("replication_factor")).isEqualTo(HeaderValue.fromUint32(1)); + assertThatThrownBy(() -> topicsClient.createTopic( + STREAM_NAME, + "unreplicated-topic", + 1L, + TopicOptions.builder().replicationFactor(0).build())) + .isInstanceOfSatisfying( + IggyValidationException.class, + refusal -> + assertThat(refusal.getErrorCode()).isEqualTo(IggyErrorCode.INVALID_REPLICATION_FACTOR)); + } + + @Test void givenTopicResetWhenUpdatingShouldPreserveUnmentionedValues() { // given var created = topicsClient.createTopic(
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/ObjectMapperTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/ObjectMapperTest.java index 3398784..c252402 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/ObjectMapperTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/http/ObjectMapperTest.java
@@ -22,9 +22,11 @@ import org.apache.iggy.cluster.ClusterNode; import org.apache.iggy.message.Message; import org.apache.iggy.message.PolledMessages; +import org.apache.iggy.partition.Partition; import org.apache.iggy.partition.PartitionContext; import org.apache.iggy.system.CacheMetricsKey; import org.apache.iggy.system.Stats; +import org.apache.iggy.topic.TopicDetails; import org.assertj.core.data.Offset; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; @@ -402,6 +404,47 @@ """.formatted(base64Payload); } } + + @Nested + @DisplayName("TopicDetails") + class TopicDetailsDeserialization { + + @Test + void shouldReadPartitionMembersAndInSyncMembers() { + String json = """ + {"id": 2, "created_at": 0, "name": "topic", "size": "0 B", "messages_count": 0, + "partitions_count": 1, "options": {}, + "partitions": [ + {"id": 1, "created_at": 0, "segments_count": 1, "current_offset": 0, "size": "0 B", + "messages_count": 0, + "context": {"incarnation": 23, "owner_generation": 0, "metadata_op": 101}, + "members": [0, 1, 2], "in_sync_members": [0]} + ]} + """; + + var partition = objectMapper + .readValue(json, TopicDetails.class) + .partitions() + .get(0); + + assertThat(partition.members()).containsExactly(0, 1, 2); + assertThat(partition.inSyncMembers()).containsExactly(0); + } + + @Test + void shouldReadAbsentNodeListsAsEmpty() { + String json = """ + {"id": 1, "created_at": 0, "segments_count": 1, "current_offset": 0, "size": "0 B", + "messages_count": 0, + "context": {"incarnation": 23, "owner_generation": 0, "metadata_op": 101}} + """; + + var partition = objectMapper.readValue(json, Partition.class); + + assertThat(partition.members()).isEmpty(); + assertThat(partition.inSyncMembers()).isEmpty(); + } + } } @Nested
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java index cf4074f..0703f4b 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java
@@ -32,6 +32,7 @@ import org.apache.iggy.message.HeaderKind; import org.apache.iggy.message.HeaderValue; import org.apache.iggy.message.PolledMessages; +import org.apache.iggy.partition.Partition; import org.apache.iggy.partition.PartitionContext; import org.apache.iggy.stream.EncryptionKeySource; import org.apache.iggy.system.CacheMetricsKey; @@ -73,6 +74,7 @@ import static org.apache.iggy.serde.BytesDeserializer.readUserInfoDetails; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.tuple; class BytesDeserializerTest { @@ -87,9 +89,13 @@ } private static void writeTopicData(ByteBuf buffer) { + writeTopicData(buffer, 4); + } + + private static void writeTopicData(ByteBuf buffer, int partitionsCount) { buffer.writeIntLE(10); // topic ID writeU64(buffer, BigInteger.valueOf(1000)); // created at - buffer.writeIntLE(4); // partitions count + buffer.writeIntLE(partitionsCount); writeU64(buffer, BigInteger.valueOf(500)); // size writeU64(buffer, BigInteger.valueOf(50)); // messages count buffer.writeByte(4); // name length @@ -132,13 +138,22 @@ } private static void writePartitionData(ByteBuf buffer) { - buffer.writeIntLE(1); // partition ID + writePartitionData(buffer, 1, List.of(), List.of()); + } + + private static void writePartitionData( + ByteBuf buffer, int partitionId, List<Integer> members, List<Integer> inSyncMembers) { + buffer.writeIntLE(partitionId); writeU64(buffer, BigInteger.valueOf(1000)); // created at buffer.writeIntLE(5); // segments count writeU64(buffer, BigInteger.valueOf(99)); // current offset writeU64(buffer, BigInteger.valueOf(200)); // size writeU64(buffer, BigInteger.valueOf(20)); // messages count buffer.writeZero(24); + buffer.writeByte(members.size()); + members.forEach(buffer::writeByte); + buffer.writeByte(inSyncMembers.size()); + inSyncMembers.forEach(buffer::writeByte); } @Nested @@ -282,8 +297,7 @@ void shouldDeserializeTopicDetails() { // given ByteBuf buffer = Unpooled.buffer(); - writeTopicData(buffer); - // Write one partition + writeTopicData(buffer, 1); writePartitionData(buffer); // when @@ -293,10 +307,51 @@ assertThat(topicDetails.id()).isEqualTo(10L); assertThat(topicDetails.partitions()).hasSize(1); } + + @Test + void shouldAdvanceByEachPartitionsEncodedLength() { + ByteBuf buffer = Unpooled.buffer(); + writeTopicData(buffer, 3); + buffer.writeBytes(HexFormat.of().parseHex(PartitionDeserialization.PINNED_PARTITION)); + writePartitionData(buffer, 2, List.of(), List.of()); + writePartitionData(buffer, 3, List.of(0, 1, 2), List.of(0)); + + var topicDetails = readTopicDetails(buffer); + + assertThat(topicDetails.partitions()) + .extracting(Partition::id, Partition::members, Partition::inSyncMembers) + .containsExactly( + tuple(1L, List.of(1, 3, 4), List.of(1, 4)), + tuple(2L, List.of(), List.of()), + tuple(3L, List.of(0, 1, 2), List.of(0))); + assertThat(topicDetails.partitions()) + .extracting(Partition::segmentsCount) + .containsExactly(2L, 5L, 5L); + assertThat(buffer.readableBytes()).isZero(); + } + + @Test + void shouldRejectPartitionsCountBeyondPayload() { + ByteBuf buffer = Unpooled.buffer(); + writeTopicData(buffer, 2); + buffer.writeBytes(HexFormat.of().parseHex(PartitionDeserialization.PINNED_PARTITION)); + + assertThatThrownBy(() -> readTopicDetails(buffer)) + .isInstanceOf(IggyMalformedResponseException.class) + .hasMessageContaining("Topic partitions count 2"); + } } @Nested class PartitionDeserialization { + // The pinned bytes of the Rust encoder's sample partition: id 1, context (23, 0, 101), members [1, 3, 4] + // and in-sync members [1, 4]. + static final String PINNED_PARTITION = + "01000000004cf1238e0100000200000063000000000000000004000000000000640000000000000017000000000000000000" + + "000000000000650000000000000003010304020104"; + static final String EMPTY_LISTS_PARTITION = + "01000000004cf1238e0100000200000063000000000000000004000000000000640000000000000017000000000000000000" + + "00000000000065000000000000000000"; @Test void shouldDeserializePartition() { @@ -312,6 +367,67 @@ assertThat(partition.segmentsCount()).isEqualTo(5L); assertThat(partition.currentOffset()).isEqualTo(BigInteger.valueOf(99)); } + + @Test + void shouldDecodeThePinnedPartition() { + byte[] pinned = HexFormat.of().parseHex(PINNED_PARTITION); + ByteBuf buffer = Unpooled.wrappedBuffer(pinned); + + var partition = readPartition(buffer); + + assertThat(pinned).hasSize(71); + assertThat(buffer.readableBytes()).isZero(); + assertThat(partition.id()).isEqualTo(1L); + assertThat(partition.createdAt()).isEqualTo(BigInteger.valueOf(1_710_000_000_000L)); + assertThat(partition.segmentsCount()).isEqualTo(2L); + assertThat(partition.currentOffset()).isEqualTo(BigInteger.valueOf(99)); + assertThat(partition.size()).isEqualTo("1024"); + assertThat(partition.messagesCount()).isEqualTo(BigInteger.valueOf(100)); + assertThat(partition.context()) + .isEqualTo(new PartitionContext(BigInteger.valueOf(23), BigInteger.ZERO, BigInteger.valueOf(101))); + assertThat(partition.members()).containsExactly(1, 3, 4); + assertThat(partition.inSyncMembers()).containsExactly(1, 4); + } + + @Test + void shouldDecodeEmptyNodeListsFromOneCountByteEach() { + byte[] empty = HexFormat.of().parseHex(EMPTY_LISTS_PARTITION); + ByteBuf buffer = Unpooled.wrappedBuffer(empty); + + var partition = readPartition(buffer); + + assertThat(empty).hasSize(66); + assertThat(buffer.readableBytes()).isZero(); + assertThat(partition.messagesCount()).isEqualTo(BigInteger.valueOf(100)); + assertThat(partition.members()).isEmpty(); + assertThat(partition.inSyncMembers()).isEmpty(); + } + + @Test + void shouldDecodeNodeIdsAboveSignedByteRange() { + ByteBuf buffer = Unpooled.buffer(); + buffer.writeBytes(HexFormat.of().parseHex(EMPTY_LISTS_PARTITION), 0, 64); + buffer.writeByte(1).writeByte(200); + buffer.writeByte(1).writeByte(255); + + var partition = readPartition(buffer); + + assertThat(partition.members()).containsExactly(200); + assertThat(partition.inSyncMembers()).containsExactly(255); + } + + @Test + void shouldRejectTruncatedPartitions() { + byte[] pinned = HexFormat.of().parseHex(PINNED_PARTITION); + // Cut inside the counters, inside members, before the in-sync count and inside the in-sync list. + for (int length : new int[] {40, 66, 68, 69, 70}) { + ByteBuf buffer = Unpooled.wrappedBuffer(pinned, 0, length); + + assertThatThrownBy(() -> readPartition(buffer)) + .as("partition cut at %d bytes", length) + .isInstanceOf(IggyMalformedResponseException.class); + } + } } @Nested
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java index 778ff0a..f698bf6 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicDetailsTest.java
@@ -46,8 +46,16 @@ "compression_algorithm", HeaderValue.fromString("gzip")), Map.of("max_topic_size", HeaderValue.fromUint64(BigInteger.TWO))); - var partitions = List.of( - new Partition(1L, BigInteger.TEN, 2L, BigInteger.ZERO, "size", BigInteger.ONE, PartitionContext.EMPTY)); + var partitions = List.of(new Partition( + 1L, + BigInteger.TEN, + 2L, + BigInteger.ZERO, + "size", + BigInteger.ONE, + PartitionContext.EMPTY, + List.of(0, 1, 2), + List.of(0))); var topicDetails = new TopicDetails(topic, partitions);
diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicOptionsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicOptionsTest.java index 646b9b7..11624b1 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicOptionsTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/topic/TopicOptionsTest.java
@@ -54,6 +54,7 @@ .messagesRequiredToSave(1024) .sizeOfMessagesRequiredToSave(BigInteger.valueOf(1_048_576)) .preallocateSegments(true) + .replicationFactor(3) .build() .entries(); @@ -64,11 +65,14 @@ "consumer_offset_durability", "messages_required_to_save", "size_of_messages_required_to_save", - "preallocate_segments"); + "preallocate_segments", + "replication_factor"); assertThat(options.get("segment_size").kind()).isEqualTo(HeaderKind.Uint64); assertThat(options.get("messages_required_to_save").kind()).isEqualTo(HeaderKind.Uint32); assertThat(options.get("size_of_messages_required_to_save").kind()).isEqualTo(HeaderKind.Uint64); assertThat(options.get("preallocate_segments").kind()).isEqualTo(HeaderKind.Bool); + assertThat(options.get("replication_factor").kind()).isEqualTo(HeaderKind.Uint32); + assertThat(options.get("replication_factor").value()).containsExactly(3, 0, 0, 0); // Little-endian, so the low byte of 128 MiB leads and the high bytes are zero. assertThat(options.get("segment_size").value()).containsExactly(0, 0, 0, 8, 0, 0, 0, 0); }
diff --git a/foreign/java/java-sdk/src/test/resources/resource_options.json b/foreign/java/java-sdk/src/test/resources/resource_options.json index 202888f..3911499 100644 --- a/foreign/java/java-sdk/src/test/resources/resource_options.json +++ b/foreign/java/java-sdk/src/test/resources/resource_options.json
@@ -16,18 +16,18 @@ }, "constants": { "protocol_version": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "protocol_version_min": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "command_codes": { "create_stream": { @@ -343,6 +343,21 @@ "default": "false" }, { + "key": "replication_factor", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": null + }, + { "key": "encryption_key", "scopes": [ "stream" @@ -439,7 +454,7 @@ { "scope": "topic", "request_hex": "01", - "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "response_hex": "0c000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465127265706c69636174696f6e5f666163746f720b00000000007b0100004e6f64657320746861742073746f7265206561636820706172746974696f6e2c20312e2e3d333220616e64206174206d6f73742074686520636c75737465722773206e6f646520636f756e742e204f6d69747465642c20746865207365727665722773205b636c75737465725d2064656661756c745f7265706c69636174696f6e5f666163746f72206170706c6965733a206576657279206e6f646520756e6c65737320616e206f70657261746f72207365742061206e756d6265722e204120777269746520636f6d6d697473206f6e20612071756f72756d206f66207468657365206e6f6465732c20736f2032206b656570732074776f20636f70696573206275742073746f70732077726974696e67207768696c652065697468657220697320646f776e2c20616e642031206b65657073206f6e6520636f70792074686174207265706c696361746564206475726162696c6974792061636b6e6f776c6564676573206265666f72652069742072656163686573206469736b", "entries": [ { "key": "name", @@ -506,6 +521,12 @@ "kind": "bool", "flags": 0, "default": "false" + }, + { + "key": "replication_factor", + "kind": "uint32", + "flags": 0, + "default": null } ] },
diff --git a/foreign/node/package-lock.json b/foreign/node/package-lock.json index f7e4b88..81618ac 100644 --- a/foreign/node/package-lock.json +++ b/foreign/node/package-lock.json
@@ -1,12 +1,12 @@ { "name": "apache-iggy", - "version": "0.10.1-edge.6", + "version": "0.10.1-edge.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "apache-iggy", - "version": "0.10.1-edge.6", + "version": "0.10.1-edge.8", "license": "Apache-2.0", "dependencies": { "@node-rs/xxhash": "1.7.8",
diff --git a/foreign/node/package.json b/foreign/node/package.json index 1e95ad0..6cd4029 100644 --- a/foreign/node/package.json +++ b/foreign/node/package.json
@@ -1,7 +1,7 @@ { "name": "apache-iggy", "type": "module", - "version": "0.10.1-edge.6", + "version": "0.10.1-edge.8", "description": "Official Apache Iggy NodeJS SDK", "keywords": [ "iggy",
diff --git a/foreign/node/src/e2e/resource-options.suite.ts b/foreign/node/src/e2e/resource-options.suite.ts index 51f8f5b..08b44a9 100644 --- a/foreign/node/src/e2e/resource-options.suite.ts +++ b/foreign/node/src/e2e/resource-options.suite.ts
@@ -18,6 +18,8 @@ import { randomBytes } from 'node:crypto'; import { after, describe, it } from 'node:test'; import assert from 'node:assert/strict'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { isDeepStrictEqual } from 'node:util'; import type { Client } from '../client/client.js'; import { HeaderValue } from '../wire/message/header.utils.js'; import { ResponseError } from '../wire/error.utils.js'; @@ -32,6 +34,12 @@ const UNLIMITED = 18446744073709551615n; const UNSUPPORTED_OPTION_KEY = 4041; const OPTION_NOT_UPDATABLE = 4103; +const INVALID_REPLICATION_FACTOR = 2018; +/** + * A partition's primary reports who is in sync once it runs, so a read right + * after the create may list nobody. + */ +const IN_SYNC_TIMEOUT_MS = 15_000; type Credentials = { username: string, password: string }; @@ -298,6 +306,55 @@ } }); + it('given replication factor when creating topic should place every partition on one node', async () => { + const nodes = (await c.cluster.getClusterMetadata()).nodes.map(({ replicaId }) => replicaId); + const stream = await c.stream.create({ name: `${prefix}-factor` }); + try { + const created = await c.topic.create({ + streamId: stream.id, name: 'replicated', partitionCount: 2, replicationFactor: 1 + }); + const topic = await c.topic.get({ streamId: stream.id, topicId: created.id }); + assert.ok(topic); + assert.equal(topic.options.replication_factor, 1); + assert.deepEqual(topic.partitions.map(({ id }) => id), [0, 1]); + for (const { id, members } of topic.partitions) { + assert.equal(members.length, 1, `partition ${id} members ${members}`); + assert.ok(nodes.includes(members[0]), `partition ${id} member ${members[0]} is not in ${nodes}`); + } + + const members = topic.partitions.map((partition) => partition.members); + const inSyncMembers = async () => ( + await c.topic.get({ streamId: stream.id, topicId: created.id }) + )?.partitions.map((partition) => partition.inSyncMembers); + const deadline = Date.now() + IN_SYNC_TIMEOUT_MS; + let inSync = await inSyncMembers(); + while (!isDeepStrictEqual(inSync, members) && Date.now() < deadline) { + await sleep(100); + inSync = await inSyncMembers(); + } + assert.deepEqual(inSync, members); + } finally { + await c.stream.delete({ streamId: stream.id }); + } + }); + + it('given replication factor above node count when creating topic should refuse with 2018', async () => { + const nodes = (await c.cluster.getClusterMetadata()).nodes.length; + const stream = await c.stream.create({ name: `${prefix}-factor-refused` }); + try { + await assert.rejects( + c.topic.create({ + streamId: stream.id, name: 'over', partitionCount: 1, replicationFactor: nodes + 1 + }), + (error: unknown) => error instanceof ResponseError && + error.errorCode === INVALID_REPLICATION_FACTOR && error.fromServer + ); + assert.equal(await c.topic.get({ streamId: stream.id, topicId: 'over' }), null); + } finally { + await c.stream.delete({ streamId: stream.id }); + } + }); + it('given stream scope when describing options should report sensitivity and mutability', async () => { const specs = await c.system.describeOptions({ scope: OptionsScope.Stream }); const byKey = new Map(specs.map((spec) => [spec.key, spec]));
diff --git a/foreign/node/src/wire/fixtures/resource_options.json b/foreign/node/src/wire/fixtures/resource_options.json index 202888f..3911499 100644 --- a/foreign/node/src/wire/fixtures/resource_options.json +++ b/foreign/node/src/wire/fixtures/resource_options.json
@@ -16,18 +16,18 @@ }, "constants": { "protocol_version": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "protocol_version_min": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "command_codes": { "create_stream": { @@ -343,6 +343,21 @@ "default": "false" }, { + "key": "replication_factor", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": null + }, + { "key": "encryption_key", "scopes": [ "stream" @@ -439,7 +454,7 @@ { "scope": "topic", "request_hex": "01", - "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "response_hex": "0c000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465127265706c69636174696f6e5f666163746f720b00000000007b0100004e6f64657320746861742073746f7265206561636820706172746974696f6e2c20312e2e3d333220616e64206174206d6f73742074686520636c75737465722773206e6f646520636f756e742e204f6d69747465642c20746865207365727665722773205b636c75737465725d2064656661756c745f7265706c69636174696f6e5f666163746f72206170706c6965733a206576657279206e6f646520756e6c65737320616e206f70657261746f72207365742061206e756d6265722e204120777269746520636f6d6d697473206f6e20612071756f72756d206f66207468657365206e6f6465732c20736f2032206b656570732074776f20636f70696573206275742073746f70732077726974696e67207768696c652065697468657220697320646f776e2c20616e642031206b65657073206f6e6520636f70792074686174207265706c696361746564206475726162696c6974792061636b6e6f776c6564676573206265666f72652069742072656163686573206469736b", "entries": [ { "key": "name", @@ -506,6 +521,12 @@ "kind": "bool", "flags": 0, "default": "false" + }, + { + "key": "replication_factor", + "kind": "uint32", + "flags": 0, + "default": null } ] },
diff --git a/foreign/node/src/wire/options.utils.ts b/foreign/node/src/wire/options.utils.ts index 0572770..4e808ad 100644 --- a/foreign/node/src/wire/options.utils.ts +++ b/foreign/node/src/wire/options.utils.ts
@@ -67,7 +67,8 @@ PartitionResizePolicy: 'partition_resize_policy', MessagesRequiredToSave: 'messages_required_to_save', SizeOfMessagesRequiredToSave: 'size_of_messages_required_to_save', - PreallocateSegments: 'preallocate_segments' + PreallocateSegments: 'preallocate_segments', + ReplicationFactor: 'replication_factor' } as const; export type TopicOptionKey = typeof TopicOptionKey[keyof typeof TopicOptionKey]; @@ -107,6 +108,7 @@ ['messages_required_to_save', { scopes: [Topic], flags: 0, internal: false }], ['size_of_messages_required_to_save', { scopes: [Topic], flags: 0, internal: false }], ['preallocate_segments', { scopes: [Topic], flags: 0, internal: false }], + ['replication_factor', { scopes: [Topic], flags: 0, internal: false }], ['encryption_key', { scopes: [Stream], flags: Secret | Critical, internal: false }], ['encryption_key_wrapped', { scopes: [Stream], flags: Critical, internal: true }], ['encryption_key_source', { scopes: [Stream], flags: ReadOnly, internal: false }],
diff --git a/foreign/node/src/wire/system/update-resource.command.test.ts b/foreign/node/src/wire/system/update-resource.command.test.ts index b5ab031..8de30c3 100644 --- a/foreign/node/src/wire/system/update-resource.command.test.ts +++ b/foreign/node/src/wire/system/update-resource.command.test.ts
@@ -342,7 +342,8 @@ topicBlock({ partitionResizePolicy: PartitionResizePolicy.Fixed }), messages_required_to_save: () => topicBlock({ messagesRequiredToSave: 1 }), size_of_messages_required_to_save: () => topicBlock({ sizeOfMessagesRequiredToSave: 1n }), - preallocate_segments: () => topicBlock({ preallocateSegments: true }) + preallocate_segments: () => topicBlock({ preallocateSegments: true }), + replication_factor: () => topicBlock({ replicationFactor: 1 }) }, user: {} };
diff --git a/foreign/node/src/wire/topic/create-topic.command.test.ts b/foreign/node/src/wire/topic/create-topic.command.test.ts index 868d00a..c70aa7f 100644 --- a/foreign/node/src/wire/topic/create-topic.command.test.ts +++ b/foreign/node/src/wire/topic/create-topic.command.test.ts
@@ -94,6 +94,20 @@ ); }); + it('serialize replicationFactor as a uint32 option', () => { + const t = { ...t1, replicationFactor: 2 }; + const b = CREATE_TOPIC.serialize(t); + const key = Buffer.from('replication_factor'); + const keyAt = b.indexOf(key); + assert.ok(keyAt > fixedSize + t.name.length, 'replication_factor rides the options block'); + const value = b.subarray(keyAt + key.length); + const uint32Kind = 11; + assert.equal(value.readUInt8(0), uint32Kind); + assert.equal(value.readUInt32LE(1), 4); + assert.equal(value.readUInt32LE(5), 2); + assert.equal(deserializeOptions(b, fixedSize + t.name.length).replication_factor, 2); + }); + it('serialize caller-supplied option keys, typed fields winning', () => { const t = { ...t1,
diff --git a/foreign/node/src/wire/topic/create-topic.command.ts b/foreign/node/src/wire/topic/create-topic.command.ts index 1d3592e..18af6e4 100644 --- a/foreign/node/src/wire/topic/create-topic.command.ts +++ b/foreign/node/src/wire/topic/create-topic.command.ts
@@ -63,6 +63,16 @@ /** Preallocate segment files when the topic is created */ preallocateSegments?: boolean, /** + * Number of nodes that store each partition of the topic: 1 to 32 and at most + * the cluster's node count. Omitted, the server's `[cluster] + * default_replication_factor` applies: every node unless an operator set a + * number. A write commits on a quorum of those nodes, so 2 keeps two copies + * but stops taking writes while either node is down, like Kafka with + * `min.insync.replicas=2`. A value out of range is refused with error 2018, + * invalid replication factor. + */ + replicationFactor?: number, + /** * Option keys with no field of their own, for a key the server catalog gained * after this build shipped. A field above wins on collision, since the block * must not carry a key twice. Call `describeOptions` for the keys a server @@ -106,6 +116,7 @@ messagesRequiredToSave, sizeOfMessagesRequiredToSave, preallocateSegments, + replicationFactor, options: extraOptions = [] }: CreateTopic ) => { @@ -177,6 +188,11 @@ key: 'preallocate_segments', value: HeaderValue.Bool(preallocateSegments) }); + if (replicationFactor !== undefined) + options.push({ + key: 'replication_factor', + value: HeaderValue.Uint32(replicationFactor) + }); const b = Buffer.allocUnsafe(4 + 1); b.writeUInt32LE(partitionCount, 0); b.writeUInt8(bName.length, 4);
diff --git a/foreign/node/src/wire/topic/topic.utils.test.ts b/foreign/node/src/wire/topic/topic.utils.test.ts index fc781cd..c1fa53e 100644 --- a/foreign/node/src/wire/topic/topic.utils.test.ts +++ b/foreign/node/src/wire/topic/topic.utils.test.ts
@@ -17,9 +17,43 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { deserializeBaseTopic } from './topic.utils.js'; +import { deserializeBaseTopic, deserializePartition, deserializeTopic } from './topic.utils.js'; import { serializeOptions } from '../options.utils.js'; import { HeaderValue } from '../message/header.utils.js'; +import { DeserializeError } from '../error.utils.js'; + +/** Same bytes as the Rust `PartitionResponse` round-trip vector. */ +const PINNED_PARTITION = Buffer.from( + '01000000004cf1238e0100000200000063000000000000000004000000000000640000000000000017000000000000000000000000000000650000000000000003010304020104', + 'hex' +); + +const partitionEntry = (id: number, members: number[], inSyncMembers: number[]): Buffer => { + const counters = Buffer.alloc(64); + counters.writeUInt32LE(id, 0); + counters.writeBigUInt64LE(1_710_000_000_000n, 4); + counters.writeUInt32LE(id, 12); + counters.writeBigUInt64LE(BigInt(id) * 10n, 16); + counters.writeBigUInt64LE(BigInt(id) * 100n, 24); + counters.writeBigUInt64LE(BigInt(id) * 11n, 32); + counters.writeBigUInt64LE(BigInt(id) + 20n, 40); + counters.writeBigUInt64LE(BigInt(id) + 30n, 48); + counters.writeBigUInt64LE(BigInt(id) + 40n, 56); + return Buffer.concat([ + counters, + Buffer.from([members.length, ...members]), + Buffer.from([inSyncMembers.length, ...inSyncMembers]) + ]); +}; + +const topicHeader = (partitionsCount: number): Buffer => { + const fixed = Buffer.alloc(34); + fixed.writeUInt32LE(9, 0); + fixed.writeUInt32LE(partitionsCount, 12); + fixed.writeUInt8(1, 32); + fixed.write('t', 33); + return Buffer.concat([fixed, Buffer.alloc(8)]); +}; describe('deserializeBaseTopic', () => { it('uses the 33-byte fixed layout with settings in two options blocks', () => { @@ -83,3 +117,99 @@ assert.equal(topic.maxTopicSize, 0xffff_ffff_ffff_ffffn); }); }); + +describe('deserializePartition', () => { + it('decodes the pinned vector with its member and in-sync lists', () => { + assert.equal(PINNED_PARTITION.length, 71); + + const { bytesRead, data: partition } = deserializePartition(PINNED_PARTITION); + + assert.equal(bytesRead, 71); + assert.deepEqual(partition, { + id: 1, + createdAt: new Date(1_710_000_000), + segmentsCount: 2, + currentOffset: 99n, + sizeBytes: 1024n, + messagesCount: 100n, + context: { incarnation: 23n, ownerGeneration: 0n, metadataOp: 101n }, + members: [1, 3, 4], + inSyncMembers: [1, 4] + }); + }); + + it('reads two empty node lists as a 66-byte entry', () => { + const entry = partitionEntry(5, [], []); + assert.equal(entry.length, 66); + + const { bytesRead, data: partition } = deserializePartition(entry); + + assert.equal(bytesRead, 66); + assert.equal(partition.id, 5); + assert.deepEqual(partition.members, []); + assert.deepEqual(partition.inSyncMembers, []); + }); + + it('decodes from a non-zero position', () => { + const data = Buffer.concat([Buffer.from([0xaa, 0xbb, 0xcc]), PINNED_PARTITION]); + + const { bytesRead, data: partition } = deserializePartition(data, 3); + + assert.equal(bytesRead, 71); + assert.deepEqual(partition.members, [1, 3, 4]); + assert.deepEqual(partition.inSyncMembers, [1, 4]); + }); + + it('rejects an entry cut anywhere before its last byte', () => { + for (let length = 0; length < PINNED_PARTITION.length; length++) + assert.throws( + () => deserializePartition(PINNED_PARTITION.subarray(0, length)), + DeserializeError, + `accepted a ${length}-byte prefix` + ); + }); +}); + +describe('deserializeTopic', () => { + it('advances by the bytes each partition entry consumed', () => { + const partitions = [ + partitionEntry(1, [1, 3, 4], [1, 4]), + partitionEntry(2, [], []), + partitionEntry(3, [0, 1, 2], [0]) + ]; + assert.deepEqual(partitions.map((entry) => entry.length), [71, 66, 70]); + const data = Buffer.concat([topicHeader(3), ...partitions]); + + const { bytesRead, data: topic } = deserializeTopic(data); + + assert.equal(bytesRead, data.length); + assert.equal(topic.partitionsCount, 3); + assert.deepEqual( + topic.partitions.map(({ id, segmentsCount, messagesCount, context, members, inSyncMembers }) => + ({ id, segmentsCount, messagesCount, context, members, inSyncMembers })), + [ + { + id: 1, segmentsCount: 1, messagesCount: 11n, + context: { incarnation: 21n, ownerGeneration: 31n, metadataOp: 41n }, + members: [1, 3, 4], inSyncMembers: [1, 4] + }, + { + id: 2, segmentsCount: 2, messagesCount: 22n, + context: { incarnation: 22n, ownerGeneration: 32n, metadataOp: 42n }, + members: [], inSyncMembers: [] + }, + { + id: 3, segmentsCount: 3, messagesCount: 33n, + context: { incarnation: 23n, ownerGeneration: 33n, metadataOp: 43n }, + members: [0, 1, 2], inSyncMembers: [0] + } + ] + ); + }); + + it('rejects a response holding fewer partitions than the header counts', () => { + const data = Buffer.concat([topicHeader(2), PINNED_PARTITION]); + + assert.throws(() => deserializeTopic(data), DeserializeError); + }); +});
diff --git a/foreign/node/src/wire/topic/topic.utils.ts b/foreign/node/src/wire/topic/topic.utils.ts index c136cdc..e2367b6 100644 --- a/foreign/node/src/wire/topic/topic.utils.ts +++ b/foreign/node/src/wire/topic/topic.utils.ts
@@ -19,10 +19,17 @@ import { toDate } from '../serialize.utils.js'; import type { ValueOf } from '../../type.utils.js'; import { deserializePrefixedOptions, type ParsedOptions } from '../options.utils.js'; +import { deserializeError } from '../error.utils.js'; /** Default of `message_expiry` and `max_topic_size`: never expire, no cap. */ const UNLIMITED = 0xffff_ffff_ffff_ffffn; +/** Partition id, counters and context, before the two node lists. */ +const PARTITION_COUNTERS_SIZE = 40 + PARTITION_CONTEXT_SIZE; + +/** Smallest partition entry: the counters and two empty node lists. */ +const PARTITION_MIN_SIZE = PARTITION_COUNTERS_SIZE + 2; + /** * Basic topic information without partition details. */ @@ -68,7 +75,14 @@ /** Total size of the partition in bytes */ sizeBytes: bigint, /** Total number of messages in the partition */ - messagesCount: bigint + messagesCount: bigint, + /** Node ids that store the partition, in member order */ + members: number[], + /** + * Node ids among `members` that the partition's primary last reported in + * sync and that are up. Empty until the primary first reports. + */ + inSyncMembers: number[] }; /** Topic with partition details */ @@ -186,26 +200,61 @@ /** * Deserializes a partition from a buffer. + * Layout: `[id:u32][created_at:u64][segments_count:u32][current_offset:u64] + * [size_bytes:u64][messages_count:u64][context:24]` then two node lists, + * members and in-sync members, each a `u8` count followed by that many `u8` + * node ids. The entry is 66 bytes plus one per listed node. * * @param p - Buffer containing serialized partition data * @param pos - Starting position in the buffer * @returns Object with bytes read and deserialized partition data + * @throws DeserializeError if the buffer ends inside the entry */ export const deserializePartition = (p: Buffer, pos = 0): PartitionSerialized => { + if (p.length - pos < PARTITION_MIN_SIZE) + deserializeError('partition', pos, p.length, PARTITION_MIN_SIZE); + + const start = pos; + const id = p.readUInt32LE(pos); + const createdAt = toDate(p.readBigUint64LE(pos + 4)); + const segmentsCount = p.readUInt32LE(pos + 12); + const currentOffset = p.readBigUint64LE(pos + 16); + const sizeBytes = p.readBigUint64LE(pos + 24); + const messagesCount = p.readBigUint64LE(pos + 32); + const context = deserializePartitionContext(p, pos + 40); + pos += PARTITION_COUNTERS_SIZE; + + const members = deserializeNodeIds(p, pos, 'partition members'); + pos += 1 + members.length; + const inSyncMembers = deserializeNodeIds(p, pos, 'partition in-sync members'); + pos += 1 + inSyncMembers.length; + return { - bytesRead: 40 + PARTITION_CONTEXT_SIZE, + bytesRead: pos - start, data: { - id: p.readUInt32LE(pos), - createdAt: toDate(p.readBigUint64LE(pos + 4)), - segmentsCount: p.readUInt32LE(pos + 12), - currentOffset: p.readBigUint64LE(pos + 16), - sizeBytes: p.readBigUint64LE(pos + 24), - messagesCount: p.readBigUint64LE(pos + 32), - context: deserializePartitionContext(p, pos + 40), + id, + createdAt, + segmentsCount, + currentOffset, + sizeBytes, + messagesCount, + context, + members, + inSyncMembers, } } }; +/** Reads a `[count:u8][ids:count x u8]` node list at `pos`. */ +const deserializeNodeIds = (p: Buffer, pos: number, name: string): number[] => { + if (p.length - pos < 1) + deserializeError(name, pos, p.length, 1); + const count = p.readUInt8(pos); + if (p.length - pos - 1 < count) + deserializeError(name, pos, p.length, 1 + count, { count }); + return Array.from(p.subarray(pos + 1, pos + 1 + count)); +}; + /** * Deserializes a topic with partitions from a buffer.
diff --git a/foreign/node/src/wire/vsr/register.ts b/foreign/node/src/wire/vsr/register.ts index a09930b..53ae201 100644 --- a/foreign/node/src/wire/vsr/register.ts +++ b/foreign/node/src/wire/vsr/register.ts
@@ -30,11 +30,11 @@ /** * Packed protocol semver of the wire contract this port implements, - * `pack(0, 12, 1)` per `core/binary_protocol/src/version.rs`. Update with the + * `pack(0, 12, 2)` per `core/binary_protocol/src/version.rs`. Update with the * Rust constant during stable release preparation. Edge builds sharing this * number are not guaranteed to have compatible layouts. */ -export const IGGY_PROTOCOL_VERSION = (0 << 20) | (12 << 10) | 1; +export const IGGY_PROTOCOL_VERSION = (0 << 20) | (12 << 10) | 2; const SDK_NAME = 'node-sdk';
diff --git a/foreign/php/Cargo.toml b/foreign/php/Cargo.toml index 66145cc..cd25bff 100644 --- a/foreign/php/Cargo.toml +++ b/foreign/php/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy-php" -version = "0.1.8" +version = "0.1.10" edition = "2024" authors = ["Iggy Committers <dev@iggy.apache.org>"] license = "Apache-2.0"
diff --git a/foreign/php/README.md b/foreign/php/README.md index 2adfe70..dc6a0eb 100644 --- a/foreign/php/README.md +++ b/foreign/php/README.md
@@ -248,8 +248,10 @@ Reads report only `encryption_key_source` (`stream`, `server`, or `none` for a plaintext stream), `encryption_key_fingerprint` and `encryption_kek_fingerprint`, comma-separated during a server key rotation. `Iggy\Client::encryptionKeyFingerprint($key)` computes the fingerprint of a key you hold. Without a key, a server with encryption enabled generates one. - `updateStream()` renames a stream and `updateTopic()` renames a topic or changes `message_expiry_micros` and `max_topic_size`. Null arguments keep stored values. `reset` takes `Iggy\StreamOptionKey` or `Iggy\TopicOptionKey` cases and returns keys to their server defaults. Only the topic's message expiry and size cap are resettable, every other case is refused with the server's error. - `raw` on create and update sends string-valued keys this build does not know, and `raw_reset` resets keys by name. The server decides whether each one is accepted. A name listed twice across `reset` and `raw_reset` is refused as `Invalid command`. +- `createTopic()` takes `replication_factor`, how many nodes store each partition: 1 to 32 and at most the cluster's node count. Null lets the server's `[cluster] default_replication_factor` apply. The factor is create-only, and a value out of range is refused as `Invalid replication factor` (code 2018). +- A topic read's `partitions` holds one `Iggy\Partition` per partition with `id`, `segments_count`, `current_offset`, `size_bytes`, `messages_count`, `members` (the node ids that store the partition) and `in_sync_members` (the members its primary last reported in sync and that are up, empty until it first reports). - `describeOptions(Iggy\OptionsScope)` returns the server catalog as `Iggy\OptionSpec` objects with `key`, `kind`, `updatable`, `resettable`, `secret`, `read_only`, `critical`, `default_value` and `description`. -- SDK exceptions carry the server error code in `getCode()`, for example 4103 (option cannot be updated), 4104 (option cannot be reset), 4105 (read-only option), 4108 (root user cannot be changed by this caller) and 4041 (unsupported option key). +- SDK exceptions carry the server error code in `getCode()`, for example 4103 (option cannot be updated), 4104 (option cannot be reset), 4105 (read-only option), 4108 (root user cannot be changed by this caller), 4041 (unsupported option key) and 2018 (invalid replication factor). ## Metadata Changes
diff --git a/foreign/php/iggy-php.stubs.php b/foreign/php/iggy-php.stubs.php index 301f516..f6a12ce 100644 --- a/foreign/php/iggy-php.stubs.php +++ b/foreign/php/iggy-php.stubs.php
@@ -169,6 +169,12 @@ * know, and the server decides whether to accept them. It is marked * `#[\SensitiveParameter]`, because an unknown key may hold a secret. * + * `replication_factor` is how many nodes store each partition, 1 to 32 and at most the + * cluster's node count. Null lets the server's `[cluster] default_replication_factor` apply: + * every node unless an operator set a number. A write commits on a quorum of these nodes, so + * 2 keeps two copies but stops accepting writes while either node is down. The server + * rejects an out-of-range value with `InvalidReplicationFactor`. + * * @param mixed $stream * @param string $name * @param int $partitions_count @@ -182,10 +188,11 @@ * @param mixed $size_of_messages_required_to_save * @param bool|null $preallocate_segments * @param string|null $partition_resize_policy + * @param mixed $replication_factor * @param array|null $raw * @return void */ - public function createTopic(mixed $stream, string $name, int $partitions_count, ?string $compression_algorithm = null, mixed $message_expiry_micros = null, mixed $max_topic_size = null, mixed $segment_size = null, ?\Iggy\Durability $durability = null, ?\Iggy\Durability $consumer_offset_durability = null, mixed $messages_required_to_save = null, mixed $size_of_messages_required_to_save = null, ?bool $preallocate_segments = null, ?string $partition_resize_policy = null, ?array $raw = null): void {} + public function createTopic(mixed $stream, string $name, int $partitions_count, ?string $compression_algorithm = null, mixed $message_expiry_micros = null, mixed $max_topic_size = null, mixed $segment_size = null, ?\Iggy\Durability $durability = null, ?\Iggy\Durability $consumer_offset_durability = null, mixed $messages_required_to_save = null, mixed $size_of_messages_required_to_save = null, ?bool $preallocate_segments = null, ?string $partition_resize_policy = null, mixed $replication_factor = null, ?array $raw = null): void {} /** * Deletes the offset stored on `partition_id` for a standalone consumer, or for a consumer @@ -2102,6 +2109,40 @@ } /** + * One partition of a topic. + */ + class Partition { + public readonly int $current_offset; + + public readonly int $id; + + /** + * Node ids among `members` that the partition's primary last reported in sync and that + * are up. Empty until the primary first reports, and while no member that is up can + * lead it. + * + * @var array + */ + public readonly array $in_sync_members; + + /** + * Node ids that store the partition, in member order. While one is down, a write still + * commits if a quorum of the others is in sync. + * + * @var array + */ + public readonly array $members; + + public readonly int $messages_count; + + public readonly int $segments_count; + + public readonly int $size_bytes; + + public function __construct() {} + } + + /** * Reads the partitions of a topic in batches, with routing, prefetch and the position of each * partition inside. Build one with Client::reader() or Client::groupReader(). * @@ -2636,6 +2677,16 @@ */ public readonly array $options; + /** + * One `Iggy\Partition` per partition of the topic. + * + * The partitions are rebuilt on each getter call; cache the result in PHP if they will + * be read repeatedly. + * + * @var array + */ + public readonly array $partitions; + public readonly int $partitions_count; public function __construct() {} @@ -2659,6 +2710,7 @@ case MessagesRequiredToSave = 'messages_required_to_save'; case SizeOfMessagesRequiredToSave = 'size_of_messages_required_to_save'; case PreallocateSegments = 'preallocate_segments'; + case ReplicationFactor = 'replication_factor'; } }
diff --git a/foreign/php/src/client.rs b/foreign/php/src/client.rs index dc26f86..8312d74 100644 --- a/foreign/php/src/client.rs +++ b/foreign/php/src/client.rs
@@ -304,6 +304,12 @@ /// (18446744073709551615) can be sent. `raw` sends string-valued keys this build does not /// know, and the server decides whether to accept them. It is marked /// `#[\SensitiveParameter]`, because an unknown key may hold a secret. + /// + /// `replication_factor` is how many nodes store each partition, 1 to 32 and at most the + /// cluster's node count. Null lets the server's `[cluster] default_replication_factor` apply: + /// every node unless an operator set a number. A write commits on a quorum of these nodes, so + /// 2 keeps two copies but stops accepting writes while either node is down. The server + /// rejects an out-of-range value with `InvalidReplicationFactor`. #[allow(clippy::too_many_arguments)] pub fn create_topic( &self, @@ -320,6 +326,7 @@ size_of_messages_required_to_save: Option<&Zval>, preallocate_segments: Option<bool>, partition_resize_policy: Option<String>, + replication_factor: Option<&Zval>, raw: Option<BTreeMap<String, String>>, ) -> PhpResult { let message_expiry_micros = optional_u64(message_expiry_micros, "message_expiry_micros")?; @@ -331,6 +338,8 @@ size_of_messages_required_to_save, "size_of_messages_required_to_save", )?; + let replication_factor = + optional_unsigned::<u32>(replication_factor, "replication_factor")?; let compression_algorithm = match compression_algorithm { Some(value) => CompressionAlgorithm::from_str(&value).map_err(to_php_exception)?, None => CompressionAlgorithm::default(), @@ -363,6 +372,7 @@ size_of_messages_required_to_save: size_of_messages_required_to_save .map(IggyByteSize::from), preallocate_segments, + replication_factor, raw: raw.unwrap_or_default(), };
diff --git a/foreign/php/src/lib.rs b/foreign/php/src/lib.rs index 5acc204..ff22eaf 100644 --- a/foreign/php/src/lib.rs +++ b/foreign/php/src/lib.rs
@@ -61,7 +61,7 @@ use crate::receive_message::{Checkpoint, PolledMessages, PollingStrategy, ReceiveMessage}; use crate::send_message::{SendMessage, SendMessagesConfirmation, SendMessagesResponse}; use crate::stream::StreamDetails; -use crate::topic::TopicDetails; +use crate::topic::{Partition, TopicDetails}; #[php_module] pub fn get_module(module: ModuleBuilder) -> ModuleBuilder { @@ -109,6 +109,7 @@ .class::<SendMessagesResponse>() .class::<StreamDetails>() .class::<TopicDetails>() + .class::<Partition>() .class::<OptionSpec>() .class::<MetadataChanges>() .class::<MetadataChangeRecord>()
diff --git a/foreign/php/src/options.rs b/foreign/php/src/options.rs index 411db32..410c331 100644 --- a/foreign/php/src/options.rs +++ b/foreign/php/src/options.rs
@@ -108,6 +108,8 @@ SizeOfMessagesRequiredToSave, #[php(value = "preallocate_segments")] PreallocateSegments, + #[php(value = "replication_factor")] + ReplicationFactor, } impl From<TopicOptionKey> for OptionKey { @@ -124,6 +126,7 @@ TopicOptionKey::MessagesRequiredToSave => Self::MessagesRequiredToSave, TopicOptionKey::SizeOfMessagesRequiredToSave => Self::SizeOfMessagesRequiredToSave, TopicOptionKey::PreallocateSegments => Self::PreallocateSegments, + TopicOptionKey::ReplicationFactor => Self::ReplicationFactor, } } } @@ -143,6 +146,7 @@ Self::MessagesRequiredToSave => "MessagesRequiredToSave", Self::SizeOfMessagesRequiredToSave => "SizeOfMessagesRequiredToSave", Self::PreallocateSegments => "PreallocateSegments", + Self::ReplicationFactor => "ReplicationFactor", }; format!("\\Iggy\\TopicOptionKey::{case}") }
diff --git a/foreign/php/src/sensitive_parameters.rs b/foreign/php/src/sensitive_parameters.rs index 73aeea1..7f10e75 100644 --- a/foreign/php/src/sensitive_parameters.rs +++ b/foreign/php/src/sensitive_parameters.rs
@@ -57,7 +57,7 @@ ("Iggy\\Client", "createStream", 1), ("Iggy\\Client", "createStream", 2), ("Iggy\\Client", "updateStream", 3), - ("Iggy\\Client", "createTopic", 13), + ("Iggy\\Client", "createTopic", 14), ("Iggy\\Client", "updateTopic", 6), ("Iggy\\Client", "encryptionKeyFingerprint", 0), ];
diff --git a/foreign/php/src/topic.rs b/foreign/php/src/topic.rs index 7708b7a..3d1e951 100644 --- a/foreign/php/src/topic.rs +++ b/foreign/php/src/topic.rs
@@ -16,7 +16,7 @@ // under the License. use ext_php_rs::{boxed::ZBox, exception::PhpResult, php_class, php_impl, types::ZendHashTable}; -use iggy::prelude::TopicDetails as RustTopicDetails; +use iggy::prelude::{Partition as RustPartition, TopicDetails as RustTopicDetails}; use crate::options::options_array; @@ -54,6 +54,20 @@ self.inner.partitions_count } + /// One `Iggy\Partition` per partition of the topic. + /// + /// The partitions are rebuilt on each getter call; cache the result in PHP if they will + /// be read repeatedly. + #[php(getter)] + pub fn partitions(&self) -> Vec<Partition> { + self.inner + .partitions + .iter() + .cloned() + .map(Partition::from) + .collect() + } + /// Options the creating or updating client set, as option name to scalar value. /// Integers above `PHP_INT_MAX`, such as never-expire and unlimited /// (18446744073709551615), are decimal strings. @@ -68,3 +82,59 @@ options_array(&self.inner.options, false) } } + +/// One partition of a topic. +#[php_class] +#[php(name = "Iggy\\Partition")] +pub struct Partition { + pub(crate) inner: RustPartition, +} + +impl From<RustPartition> for Partition { + fn from(inner: RustPartition) -> Self { + Self { inner } + } +} + +#[php_impl] +impl Partition { + #[php(getter)] + pub fn id(&self) -> u32 { + self.inner.id + } + + #[php(getter, change_case = "snake_case")] + pub fn segments_count(&self) -> u32 { + self.inner.segments_count + } + + #[php(getter, change_case = "snake_case")] + pub fn current_offset(&self) -> u64 { + self.inner.current_offset + } + + #[php(getter, change_case = "snake_case")] + pub fn size_bytes(&self) -> u64 { + self.inner.size.as_bytes_u64() + } + + #[php(getter, change_case = "snake_case")] + pub fn messages_count(&self) -> u64 { + self.inner.messages_count + } + + /// Node ids that store the partition, in member order. While one is down, a write still + /// commits if a quorum of the others is in sync. + #[php(getter)] + pub fn members(&self) -> Vec<u8> { + self.inner.members.clone() + } + + /// Node ids among `members` that the partition's primary last reported in sync and that + /// are up. Empty until the primary first reports, and while no member that is up can + /// lead it. + #[php(getter, change_case = "snake_case")] + pub fn in_sync_members(&self) -> Vec<u8> { + self.inner.in_sync_members.clone() + } +}
diff --git a/foreign/php/tests/IggySdkTest.php b/foreign/php/tests/IggySdkTest.php index 321189d..6f381e8 100644 --- a/foreign/php/tests/IggySdkTest.php +++ b/foreign/php/tests/IggySdkTest.php
@@ -25,6 +25,7 @@ use Iggy\CommitPolicy; use Iggy\Exception\NotFoundException; use Iggy\Exception\PendingOperationException; +use Iggy\Partition; use Iggy\PolledMessages; use Iggy\PollingStrategy; use Iggy\PollOptions; @@ -197,6 +198,10 @@ assert_same($topicName, $topic->name); assert_true($topic->id >= 0, 'expected non-negative topic id'); assert_same(2, $topic->partitions_count); + assert_same([0, 1], array_map(static fn (Partition $partition): int => $partition->id, $topic->partitions)); + foreach ($topic->partitions as $partition) { + assert_same(0, $partition->messages_count, "partition {$partition->id}"); + } } finally { cleanup_stream_with_topics($client, $streamName, [$topicName]); }
diff --git a/foreign/php/tests/RawCommandTest.php b/foreign/php/tests/RawCommandTest.php index bd632cb..ce5a1cd 100644 --- a/foreign/php/tests/RawCommandTest.php +++ b/foreign/php/tests/RawCommandTest.php
@@ -220,9 +220,10 @@ 'compression_algorithm' => null, 'message_expiry_micros' => null, 'max_topic_size' => null, 'segment_size' => null, 'durability' => null, 'consumer_offset_durability' => null, 'messages_required_to_save' => null, - 'size_of_messages_required_to_save' => null, 'preallocate_segments' => null], + 'size_of_messages_required_to_save' => null, 'preallocate_segments' => null, + 'replication_factor' => null], ['message_expiry_micros', 'max_topic_size', 'segment_size', - 'messages_required_to_save', 'size_of_messages_required_to_save'], + 'messages_required_to_save', 'size_of_messages_required_to_save', 'replication_factor'], ], 'consumerGroup' => [ ['name' => 'g', 'stream' => 's', 'topic' => 't', 'polling_strategy' => null, @@ -263,8 +264,10 @@ assert_throws(static fn () => $client->$method(...$arguments), $field); } } - $arguments = array_replace($methods['createTopic'][0], ['messages_required_to_save' => 4294967296]); - assert_throws(static fn () => $client->createTopic(...$arguments), 'messages_required_to_save'); + foreach (['messages_required_to_save', 'replication_factor'] as $field) { + $arguments = array_replace($methods['createTopic'][0], [$field => 4294967296]); + assert_throws(static fn () => $client->createTopic(...$arguments), $field); + } } public function testPollOptionsKeepDefaultsAndRefuseInvalidLimits(): void
diff --git a/foreign/php/tests/ResourceOptionsTest.php b/foreign/php/tests/ResourceOptionsTest.php index 92daca4..4b4588d 100644 --- a/foreign/php/tests/ResourceOptionsTest.php +++ b/foreign/php/tests/ResourceOptionsTest.php
@@ -23,6 +23,7 @@ use Iggy\Exception\IggyException; use Iggy\OptionSpec; use Iggy\OptionsScope; +use Iggy\Partition; use Iggy\PollingStrategy; use Iggy\PollOptions; use Iggy\SendMessage; @@ -54,6 +55,12 @@ private const NEVER = '18446744073709551615'; // Both durabilities are always sent, so a topic created without options stores them as explicit. private const DURABILITY_DEFAULTS = ['consumer_offset_durability' => 'replicated', 'durability' => 'replicated']; + // The e2e lanes run a single-node server. + private const SERVER_NODES = 1; + private const INVALID_REPLICATION_FACTOR = 2018; + // A partition's primary reports who is in sync once it runs, so a read right after the + // create may list nobody. + private const IN_SYNC_WAIT_SECONDS = 15.0; private const STREAM_DERIVED_KEYS = ['encryption_kek_fingerprint', 'encryption_key_fingerprint', 'encryption_key_source']; // Create and update arguments named after the unit they take rather than the option key. private const PARAMETER_NAMES = ['message_expiry' => 'message_expiry_micros']; @@ -404,10 +411,13 @@ assert_same($explicit, self::sorted($defaults->options)); $expected = []; foreach (self::catalogRows('topic') as $key => $row) { - if ($row['storage'] === 'map' && !array_key_exists($key, $explicit)) { + if ($row['storage'] === 'map' && $row['default'] !== null && !array_key_exists($key, $explicit)) { $expected[$key] = self::phpScalar($row['default'], $row['kind']); } } + // The catalog has no replication factor default: the server's + // default_replication_factor applies, which is every node unless configured. + $expected['replication_factor'] = self::SERVER_NODES; ksort($expected); $derived = $defaults->derived_options; ksort($derived); @@ -421,6 +431,59 @@ ); } + #[DataProvider('transports')] + #[TestDox('A topic created with a replication factor reads it back and lists node 0 for every partition on $transport')] + public function testGivenReplicationFactorWhenCreatingShouldListMembersOfEveryPartition(string $transport): void + { + $client = new_transport_client($transport); + $stream = $this->createStream($client, unique_name('replication-factor')); + + $client->createTopic($stream->id, 'events', 2, null, null, null, null, replication_factor: 1); + + $topic = $client->getTopic($stream->id, 'events'); + assert_not_null($topic); + assert_same(1, $topic->options['replication_factor']); + assert_same([0, 1], array_map(static fn (Partition $partition): int => $partition->id, $topic->partitions)); + foreach ($topic->partitions as $partition) { + assert_same([0], $partition->members, "partition {$partition->id}"); + } + $inSync = read_until( + static fn (): array => array_map( + static fn (Partition $partition): array => $partition->in_sync_members, + $client->getTopic($stream->id, 'events')?->partitions ?? [], + ), + [[0], [0]], + self::IN_SYNC_WAIT_SECONDS, + ); + assert_same([[0], [0]], $inSync); + } + + #[DataProvider('transports')] + #[TestDox('A replication factor above the node count is refused with code 2018 and creates no topic on $transport')] + public function testGivenReplicationFactorAboveNodeCountWhenCreatingShouldRefuseWithItsCode(string $transport): void + { + $client = new_transport_client($transport); + $stream = $this->createStream($client, unique_name('replication-factor-refused')); + + $error = assert_throws( + static fn () => $client->createTopic( + $stream->id, + 'events', + 1, + null, + null, + null, + null, + replication_factor: self::SERVER_NODES + 1, + ), + 'Invalid replication factor', + ); + + Assert::assertInstanceOf(IggyException::class, $error, $error->getMessage()); + assert_same(self::INVALID_REPLICATION_FACTOR, $error->getCode()); + assert_null($client->getTopic($stream->id, 'events')); + } + #[TestDox('describeOptions reports every described catalog row with its kind and flags')] public function testGivenEveryScopeWhenDescribingOptionsShouldMatchCatalog(): void {
diff --git a/foreign/php/tests/SensitiveParameterTest.php b/foreign/php/tests/SensitiveParameterTest.php index 96073e7..664c81f 100644 --- a/foreign/php/tests/SensitiveParameterTest.php +++ b/foreign/php/tests/SensitiveParameterTest.php
@@ -50,7 +50,7 @@ return [ 'createStream' => ['createStream', 2, static fn (IggyClient $client, array $raw) => $client->createStream('never-sent', null, $raw, 'unexpected')], 'updateStream' => ['updateStream', 3, static fn (IggyClient $client, array $raw) => $client->updateStream('never-sent', null, [], $raw, null, 'unexpected')], - 'createTopic' => ['createTopic', 13, static fn (IggyClient $client, array $raw) => $client->createTopic('never-sent', 'never-sent', 1, null, null, null, null, null, null, null, null, null, null, $raw, 'unexpected')], + 'createTopic' => ['createTopic', 14, static fn (IggyClient $client, array $raw) => $client->createTopic('never-sent', 'never-sent', 1, null, null, null, null, null, null, null, null, null, null, null, $raw, 'unexpected')], 'updateTopic' => ['updateTopic', 6, static fn (IggyClient $client, array $raw) => $client->updateTopic('never-sent', 'never-sent', null, null, null, [], $raw, null, 'unexpected')], ]; }
diff --git a/foreign/php/tests/fixtures/resource_options.json b/foreign/php/tests/fixtures/resource_options.json index 202888f..3911499 100644 --- a/foreign/php/tests/fixtures/resource_options.json +++ b/foreign/php/tests/fixtures/resource_options.json
@@ -16,18 +16,18 @@ }, "constants": { "protocol_version": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "protocol_version_min": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "command_codes": { "create_stream": { @@ -343,6 +343,21 @@ "default": "false" }, { + "key": "replication_factor", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": null + }, + { "key": "encryption_key", "scopes": [ "stream" @@ -439,7 +454,7 @@ { "scope": "topic", "request_hex": "01", - "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "response_hex": "0c000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465127265706c69636174696f6e5f666163746f720b00000000007b0100004e6f64657320746861742073746f7265206561636820706172746974696f6e2c20312e2e3d333220616e64206174206d6f73742074686520636c75737465722773206e6f646520636f756e742e204f6d69747465642c20746865207365727665722773205b636c75737465725d2064656661756c745f7265706c69636174696f6e5f666163746f72206170706c6965733a206576657279206e6f646520756e6c65737320616e206f70657261746f72207365742061206e756d6265722e204120777269746520636f6d6d697473206f6e20612071756f72756d206f66207468657365206e6f6465732c20736f2032206b656570732074776f20636f70696573206275742073746f70732077726974696e67207768696c652065697468657220697320646f776e2c20616e642031206b65657073206f6e6520636f70792074686174207265706c696361746564206475726162696c6974792061636b6e6f776c6564676573206265666f72652069742072656163686573206469736b", "entries": [ { "key": "name", @@ -506,6 +521,12 @@ "kind": "bool", "flags": 0, "default": "false" + }, + { + "key": "replication_factor", + "kind": "uint32", + "flags": 0, + "default": null } ] },
diff --git a/foreign/python/Cargo.toml b/foreign/python/Cargo.toml index 78ce2b3..7c3adc5 100644 --- a/foreign/python/Cargo.toml +++ b/foreign/python/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "apache-iggy" -version = "0.9.1-dev6" +version = "0.9.1-dev8" edition = "2024" authors = ["Iggy Committers <dev@iggy.apache.org>"] license = "Apache-2.0" @@ -37,7 +37,7 @@ [dependencies] bytes = "1.12.1" futures = "0.3.34" -iggy = { path = "../../core/sdk", version = "0.11.1-edge.6" } +iggy = { path = "../../core/sdk", version = "0.11.1-edge.7" } paste = "1" pyo3 = { version = "0.29.3", features = ["experimental-async"] } pyo3-async-runtimes = { version = "0.29.0", features = [
diff --git a/foreign/python/apache_iggy.pyi b/foreign/python/apache_iggy.pyi index 844febf..349f7be 100644 --- a/foreign/python/apache_iggy.pyi +++ b/foreign/python/apache_iggy.pyi
@@ -3693,6 +3693,10 @@ r""" Topic segment preallocation, fixed at creation. """ + ReplicationFactor: typing.ClassVar[OptionKey] + r""" + Topic replication factor, fixed at creation. + """ EncryptionKey: typing.ClassVar[OptionKey] r""" Stream encryption key, write-only and fixed at creation. @@ -3811,6 +3815,19 @@ The partition incarnation and the metadata operation the details were read at. The owner generation is `0` because the details belong to no consumer group. """ + @property + def members(self) -> builtins.list[builtins.int]: + r""" + Node ids that store the partition, in member order. While one is down, a write + still commits if a quorum of the others is in sync. + """ + @property + def in_sync_members(self) -> builtins.list[builtins.int]: + r""" + Node ids among `members` that the partition's primary last reported in sync and + that are up. Empty until the primary first reports, and while no member that is + up can lead it. + """ @typing.final class PartitionContext: @@ -5321,6 +5338,7 @@ messages_required_to_save: builtins.int | None = None, size_of_messages_required_to_save: builtins.int | None = None, preallocate_segments: builtins.bool | None = None, + replication_factor: builtins.int | None = None, raw: builtins.dict[builtins.str, builtins.str] | None = None, ) -> TopicOptions: r""" @@ -5350,6 +5368,14 @@ size_of_messages_required_to_save: Byte flush threshold as `int | None`. preallocate_segments: Reserve segment bytes on open as `bool | None`. + replication_factor: Nodes that store each partition as + `int | None`, 1 to 32 and at most the cluster's node count. + `None` applies the server's `[cluster] default_replication_factor`, + every node unless an operator set a number. A write commits on + a quorum of these nodes, so 2 keeps two copies but stops + accepting writes while either node is down. The server rejects + an out-of-range value with an `OptionError` of kind + `invalid_replication_factor`. raw: Additional option keys as `dict[str, str]`, sent as string values so a key newer than this build can be set. A typed field wins on collision.
diff --git a/foreign/python/pyproject.toml b/foreign/python/pyproject.toml index 2027b7d..9a5e496 100644 --- a/foreign/python/pyproject.toml +++ b/foreign/python/pyproject.toml
@@ -22,7 +22,7 @@ [project] name = "apache-iggy" requires-python = ">=3.11" -version = "0.9.1.dev6" +version = "0.9.1.dev8" description = "Apache Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." readme = "README.md" license = { file = "LICENSE" }
diff --git a/foreign/python/src/client.rs b/foreign/python/src/client.rs index cc2c80d..e91ed73 100644 --- a/foreign/python/src/client.rs +++ b/foreign/python/src/client.rs
@@ -290,6 +290,7 @@ | IggyError::StreamEncryptionUnavailable | IggyError::UnsupportedCriticalOption(_) | IggyError::CannotModifyRootUser + | IggyError::InvalidReplicationFactor ) } @@ -2404,6 +2405,32 @@ } #[test] + fn given_replication_factor_refusal_when_mapping_should_raise_option_error_on_every_transport() + { + Python::initialize(); + Python::attach(|py| { + let binary = to_runtime_error(IggyError::InvalidReplicationFactor); + let http = to_runtime_error(IggyError::HttpResponseError( + 400, + r#"{"id":2018,"code":"invalid_replication_factor","reason":"Invalid replication factor","field":null}"# + .to_owned(), + )); + for error in [binary, http] { + assert!(error.is_instance_of::<OptionError>(py)); + let value = error.value(py); + assert_eq!( + value.getattr("code").unwrap().extract::<u32>().unwrap(), + 2018 + ); + assert_eq!( + value.getattr("kind").unwrap().extract::<String>().unwrap(), + "invalid_replication_factor" + ); + } + }); + } + + #[test] fn pending_operation_errors_preserve_exact_identity_and_kind() { Python::initialize(); Python::attach(|py| {
diff --git a/foreign/python/src/resource_options.rs b/foreign/python/src/resource_options.rs index a5fd650..352341c 100644 --- a/foreign/python/src/resource_options.rs +++ b/foreign/python/src/resource_options.rs
@@ -93,6 +93,8 @@ SizeOfMessagesRequiredToSave, /// Topic segment preallocation, fixed at creation. PreallocateSegments, + /// Topic replication factor, fixed at creation. + ReplicationFactor, /// Stream encryption key, write-only and fixed at creation. EncryptionKey, /// Who chose the stream's key, computed by the server. @@ -223,6 +225,14 @@ /// size_of_messages_required_to_save: Byte flush threshold as /// `int | None`. /// preallocate_segments: Reserve segment bytes on open as `bool | None`. + /// replication_factor: Nodes that store each partition as + /// `int | None`, 1 to 32 and at most the cluster's node count. + /// `None` applies the server's `[cluster] default_replication_factor`, + /// every node unless an operator set a number. A write commits on + /// a quorum of these nodes, so 2 keeps two copies but stops + /// accepting writes while either node is down. The server rejects + /// an out-of-range value with an `OptionError` of kind + /// `invalid_replication_factor`. /// raw: Additional option keys as `dict[str, str]`, sent as string /// values so a key newer than this build can be set. A typed field /// wins on collision. @@ -233,7 +243,7 @@ /// TypeError: If a durability is not a `Durability`. #[new] #[pyo3( - signature = (*, partitions_count = 1, compression_algorithm = None, message_expiry = None, max_topic_size = None, segment_size = None, durability = None, consumer_offset_durability = None, partition_resize_policy = None, messages_required_to_save = None, size_of_messages_required_to_save = None, preallocate_segments = None, raw = None) + signature = (*, partitions_count = 1, compression_algorithm = None, message_expiry = None, max_topic_size = None, segment_size = None, durability = None, consumer_offset_durability = None, partition_resize_policy = None, messages_required_to_save = None, size_of_messages_required_to_save = None, preallocate_segments = None, replication_factor = None, raw = None) )] #[allow(clippy::too_many_arguments)] fn new( @@ -262,6 +272,9 @@ #[gen_stub(override_type(type_repr = "builtins.bool | None"))] preallocate_segments: Option< bool, >, + #[gen_stub(override_type(type_repr = "builtins.int | None"))] replication_factor: Option< + u32, + >, #[gen_stub(override_type(type_repr = "builtins.dict[builtins.str, builtins.str] | None"))] raw: Option<BTreeMap<String, String>>, ) -> PyResult<Self> { @@ -293,6 +306,7 @@ size_of_messages_required_to_save: size_of_messages_required_to_save .map(IggyByteSize::from), preallocate_segments, + replication_factor, raw: raw.unwrap_or_default(), }, }) @@ -367,6 +381,7 @@ OptionKey::MessagesRequiredToSave => Self::MessagesRequiredToSave, OptionKey::SizeOfMessagesRequiredToSave => Self::SizeOfMessagesRequiredToSave, OptionKey::PreallocateSegments => Self::PreallocateSegments, + OptionKey::ReplicationFactor => Self::ReplicationFactor, OptionKey::EncryptionKey => Self::EncryptionKey, OptionKey::EncryptionKeySource => Self::EncryptionKeySource, OptionKey::EncryptionKeyFingerprint => Self::EncryptionKeyFingerprint,
diff --git a/foreign/python/src/topic.rs b/foreign/python/src/topic.rs index e7c6815..bb840cb 100644 --- a/foreign/python/src/topic.rs +++ b/foreign/python/src/topic.rs
@@ -22,7 +22,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use pyo3::types::PyDelta; +use pyo3::types::{PyDelta, PyList}; use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pyclass_complex_enum, gen_stub_pymethods}; use crate::duration::{iggy_duration_to_py_delta, py_delta_to_iggy_duration}; @@ -431,4 +431,58 @@ pub fn context(&self) -> PartitionContext { self.inner.context.into() } + + /// Node ids that store the partition, in member order. While one is down, a write + /// still commits if a quorum of the others is in sync. + #[gen_stub(override_return_type(type_repr = "builtins.list[builtins.int]"))] + #[getter] + pub fn members<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, PyList>> { + node_id_list(py, &self.inner.members) + } + + /// Node ids among `members` that the partition's primary last reported in sync and + /// that are up. Empty until the primary first reports, and while no member that is + /// up can lead it. + #[gen_stub(override_return_type(type_repr = "builtins.list[builtins.int]"))] + #[getter] + pub fn in_sync_members<'a>(&self, py: Python<'a>) -> PyResult<Bound<'a, PyList>> { + node_id_list(py, &self.inner.in_sync_members) + } +} + +/// A `Vec<u8>` return would convert to `bytes`, so node ids are built as a list of `int`. +fn node_id_list<'a>(py: Python<'a>, node_ids: &[u8]) -> PyResult<Bound<'a, PyList>> { + PyList::new(py, node_ids) +} + +#[cfg(test)] +mod tests { + use super::*; + use iggy::prelude::{IggyTimestamp, PartitionContext as RustPartitionContext}; + + #[test] + fn given_partition_with_members_when_converting_should_expose_node_ids_as_int_lists() { + Python::initialize(); + let partition = Partition::from(RustPartition { + id: 1, + created_at: IggyTimestamp::from(11), + segments_count: 1, + current_offset: 2, + size: IggyByteSize::from(13), + messages_count: 3, + context: RustPartitionContext::default(), + members: vec![1, 3, 4], + in_sync_members: vec![1, 4], + }); + + Python::attach(|py| { + let members = partition.members(py).expect("build members list"); + let in_sync_members = partition + .in_sync_members(py) + .expect("build in-sync members list"); + + assert_eq!(members.extract::<Vec<u8>>().unwrap(), [1, 3, 4]); + assert_eq!(in_sync_members.extract::<Vec<u8>>().unwrap(), [1, 4]); + }); + } }
diff --git a/foreign/python/tests/fixtures/resource_options.json b/foreign/python/tests/fixtures/resource_options.json index 202888f..3911499 100644 --- a/foreign/python/tests/fixtures/resource_options.json +++ b/foreign/python/tests/fixtures/resource_options.json
@@ -16,18 +16,18 @@ }, "constants": { "protocol_version": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "protocol_version_min": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "command_codes": { "create_stream": { @@ -343,6 +343,21 @@ "default": "false" }, { + "key": "replication_factor", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": null + }, + { "key": "encryption_key", "scopes": [ "stream" @@ -439,7 +454,7 @@ { "scope": "topic", "request_hex": "01", - "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "response_hex": "0c000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465127265706c69636174696f6e5f666163746f720b00000000007b0100004e6f64657320746861742073746f7265206561636820706172746974696f6e2c20312e2e3d333220616e64206174206d6f73742074686520636c75737465722773206e6f646520636f756e742e204f6d69747465642c20746865207365727665722773205b636c75737465725d2064656661756c745f7265706c69636174696f6e5f666163746f72206170706c6965733a206576657279206e6f646520756e6c65737320616e206f70657261746f72207365742061206e756d6265722e204120777269746520636f6d6d697473206f6e20612071756f72756d206f66207468657365206e6f6465732c20736f2032206b656570732074776f20636f70696573206275742073746f70732077726974696e67207768696c652065697468657220697320646f776e2c20616e642031206b65657073206f6e6520636f70792074686174207265706c696361746564206475726162696c6974792061636b6e6f776c6564676573206265666f72652069742072656163686573206469736b", "entries": [ { "key": "name", @@ -506,6 +521,12 @@ "kind": "bool", "flags": 0, "default": "false" + }, + { + "key": "replication_factor", + "kind": "uint32", + "flags": 0, + "default": null } ] },
diff --git a/foreign/python/tests/test_resource_options.py b/foreign/python/tests/test_resource_options.py index 0b886c4..a9e70e5 100644 --- a/foreign/python/tests/test_resource_options.py +++ b/foreign/python/tests/test_resource_options.py
@@ -258,7 +258,7 @@ """Typed topic create options and generic topic updates.""" @pytest.mark.asyncio - async def test_given_keyword_topic_options_when_creating_should_preserve_all_ten_settings( # noqa: E501 + async def test_given_keyword_topic_options_when_creating_should_preserve_all_eleven_settings( # noqa: E501 self, transport_client: IggyClient, unique_name ): """Every known create setting has a typed field and reads back explicit.""" @@ -281,6 +281,7 @@ messages_required_to_save=7, size_of_messages_required_to_save=4096, preallocate_segments=True, + replication_factor=1, ), ) @@ -298,6 +299,7 @@ "messages_required_to_save": 7, "size_of_messages_required_to_save": 4096, "preallocate_segments": True, + "replication_factor": 1, } assert topic.derived_options.to_scalar_dict() == {} assert topic.compression_algorithm == "gzip"
diff --git a/foreign/python/uv.lock b/foreign/python/uv.lock index 702163a..a1ce1e6 100644 --- a/foreign/python/uv.lock +++ b/foreign/python/uv.lock
@@ -11,7 +11,7 @@ [[package]] name = "apache-iggy" -version = "0.9.1.dev6" +version = "0.9.1.dev8" source = { editable = "." } [package.optional-dependencies]
diff --git a/foreign/swift/Sources/Iggy/Iggy.swift b/foreign/swift/Sources/Iggy/Iggy.swift index 8a87cf1..ddcbde6 100644 --- a/foreign/swift/Sources/Iggy/Iggy.swift +++ b/foreign/swift/Sources/Iggy/Iggy.swift
@@ -21,5 +21,5 @@ public static let sdkName = "swift-sdk" /// Version of this SDK as reported to the server. Provisional until the /// release wiring lands; the release tooling reads this constant. - public static let sdkVersion = "0.1.5" + public static let sdkVersion = "0.1.7" }
diff --git a/foreign/swift/Sources/Iggy/Models/Resources.swift b/foreign/swift/Sources/Iggy/Models/Resources.swift index dbbda04..7827d32 100644 --- a/foreign/swift/Sources/Iggy/Models/Resources.swift +++ b/foreign/swift/Sources/Iggy/Models/Resources.swift
@@ -233,10 +233,15 @@ public var currentOffset: UInt64 public var sizeBytes: UInt64 public var messagesCount: UInt64 + /// Node ids that store the partition, in member order. + public var members: [UInt8] + /// The members the partition's primary last reported in sync and that are + /// up. Empty until the primary first reports. + public var inSyncMembers: [UInt8] public init( id: UInt32, createdAt: IggyTimestamp, segmentsCount: UInt32, currentOffset: UInt64, sizeBytes: UInt64, messagesCount: UInt64, - context: PartitionContext = .empty + context: PartitionContext = .empty, members: [UInt8] = [], inSyncMembers: [UInt8] = [] ) { self.context = context self.id = id @@ -245,6 +250,8 @@ self.currentOffset = currentOffset self.sizeBytes = sizeBytes self.messagesCount = messagesCount + self.members = members + self.inSyncMembers = inSyncMembers } }
diff --git a/foreign/swift/Sources/Iggy/Models/TopicOptions.swift b/foreign/swift/Sources/Iggy/Models/TopicOptions.swift index e6699c0..545092f 100644 --- a/foreign/swift/Sources/Iggy/Models/TopicOptions.swift +++ b/foreign/swift/Sources/Iggy/Models/TopicOptions.swift
@@ -32,6 +32,7 @@ case messagesRequiredToSave = "messages_required_to_save" case sizeOfMessagesRequiredToSave = "size_of_messages_required_to_save" case preallocateSegments = "preallocate_segments" + case replicationFactor = "replication_factor" } /// How far a write has to get before the server completes it. @@ -80,13 +81,24 @@ public var messagesRequiredToSave: UInt32? public var sizeOfMessagesRequiredToSaveBytes: UInt64? public var preallocateSegments: Bool? + /// Number of nodes that store each partition, 1 to 32 and at most the + /// cluster's node count. Nil lets the server's + /// `[cluster] default_replication_factor` apply: every node unless an + /// operator set a number. Create-only. + /// + /// Two replicas keep two copies but stop accepting writes while either + /// node is down, like Kafka with `min.insync.replicas=2`. The server + /// refuses an out-of-range value with + /// ``IggyErrorCode/invalidReplicationFactor``. + public var replicationFactor: UInt32? public var raw: [String: String] public init( partitionsCount: UInt32? = nil, compressionAlgorithm: CompressionAlgorithm? = nil, messageExpiry: IggyExpiry? = nil, maxTopicSize: MaxTopicSize? = nil, segmentSizeBytes: UInt64? = nil, durability: Durability = .replicated, consumerOffsetDurability: Durability = .replicated, partitionResizePolicy: PartitionResizePolicy? = nil, messagesRequiredToSave: UInt32? = nil, - sizeOfMessagesRequiredToSaveBytes: UInt64? = nil, preallocateSegments: Bool? = nil, raw: [String: String] = [:] + sizeOfMessagesRequiredToSaveBytes: UInt64? = nil, preallocateSegments: Bool? = nil, replicationFactor: UInt32? = nil, + raw: [String: String] = [:] ) { self.partitionsCount = partitionsCount self.compressionAlgorithm = compressionAlgorithm @@ -99,6 +111,7 @@ self.messagesRequiredToSave = messagesRequiredToSave self.sizeOfMessagesRequiredToSaveBytes = sizeOfMessagesRequiredToSaveBytes self.preallocateSegments = preallocateSegments + self.replicationFactor = replicationFactor self.raw = raw } @@ -147,6 +160,9 @@ if let preallocateSegments { options[TopicOptionKey.preallocateSegments.rawValue] = .explicit(.bool(preallocateSegments)) } + if let replicationFactor { + options[TopicOptionKey.replicationFactor.rawValue] = .explicit(.uint32(replicationFactor)) + } return options }
diff --git a/foreign/swift/Sources/Iggy/Protocol/ProtocolVersion.swift b/foreign/swift/Sources/Iggy/Protocol/ProtocolVersion.swift index 37360ad..eb87018 100644 --- a/foreign/swift/Sources/Iggy/Protocol/ProtocolVersion.swift +++ b/foreign/swift/Sources/Iggy/Protocol/ProtocolVersion.swift
@@ -32,7 +32,7 @@ /// The protocol version this SDK implements: the version of the /// `iggy_binary_protocol` crate it was ported from. The golden fixture /// carries the crate's value and a test fails when the two diverge. - public static let current = ProtocolVersion(major: 0, minor: 12, patch: 1) + public static let current = ProtocolVersion(major: 0, minor: 12, patch: 2) public var major: UInt32 { packed >> 20 } public var minor: UInt32 { (packed >> 10) & 0x3FF }
diff --git a/foreign/swift/Sources/Iggy/Protocol/Responses.swift b/foreign/swift/Sources/Iggy/Protocol/Responses.swift index 3c06573..8b90ea2 100644 --- a/foreign/swift/Sources/Iggy/Protocol/Responses.swift +++ b/foreign/swift/Sources/Iggy/Protocol/Responses.swift
@@ -98,10 +98,19 @@ let sizeBytes = try reader.readUInt64() let messagesCount = try reader.readUInt64() let context = try PartitionContext.decode(from: &reader) + let members = try nodes(from: &reader) + let inSyncMembers = try nodes(from: &reader) return Partition( id: id, createdAt: IggyTimestamp(microseconds: createdAt), segmentsCount: segmentsCount, currentOffset: currentOffset, - sizeBytes: sizeBytes, messagesCount: messagesCount, context: context) + sizeBytes: sizeBytes, messagesCount: messagesCount, context: context, + members: members, inSyncMembers: inSyncMembers) + } + + /// `[count: u8][ids: count x u8]`, one node list of a partition. + private static func nodes(from reader: inout ByteReader) throws -> [UInt8] { + let count = Int(try reader.readUInt8()) + return Array(try reader.readBytes(count)) } static func topicDetails(_ body: ArraySlice<UInt8>) throws -> TopicDetails {
diff --git a/foreign/swift/Tests/IggyE2ETests/ResourceOptionsE2ETests.swift b/foreign/swift/Tests/IggyE2ETests/ResourceOptionsE2ETests.swift index f58fc8a..3fb3e88 100644 --- a/foreign/swift/Tests/IggyE2ETests/ResourceOptionsE2ETests.swift +++ b/foreign/swift/Tests/IggyE2ETests/ResourceOptionsE2ETests.swift
@@ -187,6 +187,53 @@ } } + @Test func givenReplicationFactorWhenCreatingTopicShouldPlaceEveryPartitionOnOneNode() async throws { + try await E2EEnvironment.withSession() { session in + let client = session.client + let nodes = try await client.getClusterMetadata().nodes.map(\.replicaID) + let name = E2EEnvironment.uniqueName("swift-factor") + let stream = try await client.createStream(name: name) + let streamID = Identifier(numeric: stream.id) + await session.track(stream: try Identifier(named: name)) + + let created = try await client.createTopic( + streamID: streamID, name: name, options: TopicCreateOptions(partitionsCount: 2, replicationFactor: 1)) + let topicID = Identifier(numeric: created.id) + let topic = try #require(try await client.getTopic(streamID: streamID, topicID: topicID)) + #expect(topic.options[TopicOptionKey.replicationFactor.rawValue] == .explicit(.uint32(1))) + #expect(topic.partitions.map(\.id) == [0, 1]) + for partition in topic.partitions { + #expect(partition.members.count == 1, "partition \(partition.id) members \(partition.members)") + #expect(partition.members.allSatisfy { nodes.contains($0) }, "partition \(partition.id) is not on \(nodes)") + } + + // A partition's primary reports who is in sync once it runs, so a + // read right after the create may list nobody. + let members = topic.partitions.map(\.members) + let inSync = try await E2EEnvironment.eventually { + try await client.getTopic(streamID: streamID, topicID: topicID)?.partitions.map(\.inSyncMembers) == members + } + #expect(inSync, "in-sync members never matched \(members)") + } + } + + @Test func givenReplicationFactorAboveNodeCountWhenCreatingTopicShouldRefuseIt() async throws { + try await E2EEnvironment.withSession() { session in + let client = session.client + let nodes = try await client.getClusterMetadata().nodes.count + let name = E2EEnvironment.uniqueName("swift-factor-refused") + let stream = try await client.createStream(name: name) + let streamID = Identifier(numeric: stream.id) + await session.track(stream: try Identifier(named: name)) + + await expectReplicatedIggyError(.invalidReplicationFactor) { + _ = try await client.createTopic( + streamID: streamID, name: name, options: TopicCreateOptions(replicationFactor: UInt32(nodes) + 1)) + } + #expect(try await client.getTopic(streamID: streamID, topicID: Identifier(named: name)) == nil) + } + } + @Test func givenUserOptionsWhenCreatingAndUpdatingShouldPreserveUserBehavior() async throws { try await E2EEnvironment.withSession() { session in let client = session.client
diff --git a/foreign/swift/Tests/IggyTests/Fixtures/golden.json b/foreign/swift/Tests/IggyTests/Fixtures/golden.json index 1aed727..b753c4c 100644 --- a/foreign/swift/Tests/IggyTests/Fixtures/golden.json +++ b/foreign/swift/Tests/IggyTests/Fixtures/golden.json
@@ -1,6 +1,6 @@ { - "protocol_version": 12289, - "protocol_version_min": 12289, + "protocol_version": 12290, + "protocol_version_min": 12290, "no_assigned_partition": 4294967294, "resync_required_partition": 4294967295, "xxh3_64": { @@ -915,7 +915,7 @@ "options.golden": "020a0000006475726162696c6974790209000000706572736973746564020c0000007365676d656e745f73697a650c080000000000004000000000", "options.golden_block": "0214000000707265616c6c6f636174655f7365676d656e7473030100000001020c0000007365676d656e745f73697a650c080000000000004000000000020a0000006475726162696c6974790209000000706572736973746564", "options.stream_create.key": "020e000000656e6372797074696f6e5f6b65790120000000000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", - "options.topic_create.full": "0215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c6974790209000000706572736973746564020e0000006d61785f746f7069635f73697a650c080000000000004000000000020e0000006d6573736167655f6578706972790c0800000000a0e3d08c00000002190000006d657373616765735f72657175697265645f746f5f736176650b04000000070000000214000000707265616c6c6f636174655f7365676d656e7473030100000000020c0000007365676d656e745f73697a650c080000000000100000000000022100000073697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c080000000010000000000000", + "options.topic_create.full": "0215000000636f6d7072657373696f6e5f616c676f726974686d0204000000677a6970021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c6974790209000000706572736973746564020e0000006d61785f746f7069635f73697a650c080000000000004000000000020e0000006d6573736167655f6578706972790c0800000000a0e3d08c00000002190000006d657373616765735f72657175697265645f746f5f736176650b04000000070000000214000000707265616c6c6f636174655f7365676d656e747303010000000002120000007265706c69636174696f6e5f666163746f720b0400000002000000020c0000007365676d656e745f73697a650c080000000000100000000000022100000073697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c080000000010000000000000", "options.topic_create.raw": "021a000000636f6e73756d65725f6f66667365745f6475726162696c697479020a0000007265706c696361746564020a0000006475726162696c6974790209000000706572736973746564", "options.topic_update.full": "020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff020e0000006d6573736167655f6578706972790c08000000ffffffffffffffff", "partition.context": "110000000000000009000000000000003400000000000000", @@ -930,7 +930,7 @@ "polling.next": "050000000000000000", "polling.offset.100": "016400000000000000", "polling.timestamp.1700000000000": "020068e5cf8b010000", - "request.bind_session": "013000000973776966742d73646b05302e312e3501000000000000000000000000000000640000000000000078000000000000005a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a", + "request.bind_session": "023000000973776966742d73646b05302e312e3701000000000000000000000000000000640000000000000078000000000000005a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a", "request.change_password": "010401000000086f6c642d706173730c6e65772d706173732d313233", "request.create_consumer_group": "01040100000001040200000003677270", "request.create_partitions": "01040100000001040200000005000000", @@ -971,8 +971,8 @@ "request.get_users": "", "request.join_consumer_group": "010401000000010402000000010403000000", "request.leave_consumer_group": "010401000000010402000000020167", - "request.login_register": "013000000973776966742d73646b05302e312e355a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a0469676779046967677900000000", - "request.login_register_with_pat": "013000000973776966742d73646b05302e312e355a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a107061742d61626331323364656634353600000000", + "request.login_register": "023000000973776966742d73646b05302e312e375a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a0469676779046967677900000000", + "request.login_register_with_pat": "023000000973776966742d73646b05302e312e375a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a107061742d61626331323364656634353600000000", "request.logout_user": "", "request.ping": "", "request.poll_messages": "0101040100000001040a00000001041400000001050000000164000000000000003200000001", @@ -1005,12 +1005,12 @@ "response.get_personal_access_tokens": "07746f6b656e2d6100e090663c13060007746f6b656e2d620000000000000000", "response.get_stream": "0100000000e090663c130600020000000008000000000000c800000000000000096d792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d0100000000e090663c130600030000000010000000000000640000000000000007746f7069632d615b000000020a0000006475726162696c6974790209000000706572736973746564020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000004000000000430000000215000000636f6d7072657373696f6e5f616c676f726974686d02040000006e6f6e65020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff0200000000e090663c130600030000000010000000000000640000000000000007746f7069632d625b000000020a0000006475726162696c6974790209000000706572736973746564020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000004000000000430000000215000000636f6d7072657373696f6e5f616c676f726974686d02040000006e6f6e65020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff", "response.get_streams": "0100000000e090663c130600020000000008000000000000c800000000000000096d792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d0200000000e090663c130600000000000008000000000000c800000000000000096d792d73747265616d000000009e000000021a000000656e6372797074696f6e5f6b656b5f66696e6765727072696e740221000000333965346630663935633336393363372c64353564343739383239626332393336021a000000656e6372797074696f6e5f6b65795f66696e6765727072696e740210000000383434316533383533386362666135370215000000656e6372797074696f6e5f6b65795f736f75726365020600000073747265616d", - "response.get_topic": "0100000000e090663c1306000300000000100000000000006400000000000000086d792d746f7069635b000000020a0000006475726162696c6974790209000000706572736973746564020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000004000000000430000000215000000636f6d7072657373696f6e5f616c676f726974686d02040000006e6f6e65020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff0000000000e090663c130600020000006300000000000000000400000000000064000000000000000000000000000000000000000000000000000000000000000100000001e090663c130600010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000200000002e090663c13060003000000070000000000000000020000000000000800000000000000000000000000000000000000000000000000000000000000", + "response.get_topic": "0100000000e090663c1306000300000000100000000000006400000000000000086d792d746f7069635b000000020a0000006475726162696c6974790209000000706572736973746564020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000004000000000430000000215000000636f6d7072657373696f6e5f616c676f726974686d02040000006e6f6e65020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff0000000000e090663c13060002000000630000000000000000040000000000006400000000000000000000000000000000000000000000000000000000000000030001020200010100000001e090663c1306000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003010200000200000002e090663c13060003000000070000000000000000020000000000000800000000000000000000000000000000000000000000000000000000000000030200010102", "response.get_topics": "020000000100000000e090663c1306000300000000100000000000006400000000000000066576656e74735b000000020a0000006475726162696c6974790209000000706572736973746564020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000004000000000430000000215000000636f6d7072657373696f6e5f616c676f726974686d02040000006e6f6e65020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff0200000000e090663c1306000300000000100000000000006400000000000000046c6f67735b000000020a0000006475726162696c6974790209000000706572736973746564020e0000006d6573736167655f6578706972790c0800000000a0e3d08c000000020c0000007365676d656e745f73697a650c080000000000004000000000430000000215000000636f6d7072657373696f6e5f616c676f726974686d02040000006e6f6e65020e0000006d61785f746f7069635f73697a650c08000000ffffffffffffffff", "response.get_topics.empty": "00000000", "response.get_users": "0100000000e090663c130600010561646d696e000000000000000002000000c8000000000000000205616c6963650000000000000000", "response.identity": "2a000000", - "response.login_register": "2a00000064000000000000000130000006302e31312e30", + "response.login_register": "2a00000064000000000000000230000006302e31312e30", "response.partition_route": "07000000000000000a00000000000000030000000400000002", "response.poll_messages": "07000000660000000000000003000000000000000000000000000000000000000000000000000000070000000000000064000000000000008813000000000000e8030000000000006e01000000000000f576a2492e1328b302000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000fa7fe16d19356ec20b00000000000000000000000000000000000000000000000000000001000000000000000000000061b7f159184e9d8ac20c000000000000000000000000000000010000000a0000000c0000000100000000000000000000006202010000006b020100000076070000000000000066000000000000007017000000000000d00700000000000031010000000000006739773bc7d38608010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009377fb7d5180f51e0d00000000000000000000000000000000000000000000000000000001000000000000000000000063", "response.poll_messages.empty": "00000000000000000000000000000000000000000000000000000000000000000000000000000000", @@ -1026,8 +1026,8 @@ "response.user_details.no_permissions": "0100000000e090663c130600010561646d696e000000000000000000000000", "stream_key.fingerprint": "8441e38538cbfa57", "user_headers.sample": "0207000000617474656d70740b0400000003000000020800000074726163652d69640203000000616263", - "version_info": "013000000973776966742d73646b05302e312e35", - "vsr.eviction.incompatible_protocol": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab89674523010130000001300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e", + "version_info": "023000000973776966742d73646b05302e312e37", + "vsr.eviction.incompatible_protocol": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab89674523010230000002300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e", "vsr.eviction.stale_client": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab89674523010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000d", "vsr.reply.committed": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c0100000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab896745230100000000000000000c0000000000000000000000000000000700000000000000800000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000", "vsr.reply.denied": "00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000020000000000000029000000000000000000000000000000000000000000000000000000000000000000000000000000",
diff --git a/foreign/swift/Tests/IggyTests/Fixtures/resource_options.json b/foreign/swift/Tests/IggyTests/Fixtures/resource_options.json index 202888f..3911499 100644 --- a/foreign/swift/Tests/IggyTests/Fixtures/resource_options.json +++ b/foreign/swift/Tests/IggyTests/Fixtures/resource_options.json
@@ -16,18 +16,18 @@ }, "constants": { "protocol_version": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "protocol_version_min": { - "packed": 12289, - "text": "0.12.1", + "packed": 12290, + "text": "0.12.2", "major": 0, "minor": 12, - "patch": 1 + "patch": 2 }, "command_codes": { "create_stream": { @@ -343,6 +343,21 @@ "default": "false" }, { + "key": "replication_factor", + "scopes": [ + "topic" + ], + "kind": "uint32", + "kind_code": 11, + "storage": "map", + "mutability": "create_only", + "visibility": "public", + "critical": false, + "flags": 0, + "described": true, + "default": null + }, + { "key": "encryption_key", "scopes": [ "stream" @@ -439,7 +454,7 @@ { "scope": "topic", "request_hex": "01", - "response_hex": "0b000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465", + "response_hex": "0c000000046e616d650201000000007e0000005265736f75726365206e616d652c203120746f203235352062797465732c20756e697175652077697468696e2069747320706172656e742e20536574206279207468652063726561746520636f6d6d616e642773206e616d65206669656c6420616e64206368616e676564207468726f75676820616e207570646174652e15636f6d7072657373696f6e5f616c676f726974686d0200040000006e6f6e6550000000436f6d7072657373696f6e20616c676f726974686d20286e6f6e652c20677a6970292e2053746f72656420616e64207265706f727465642c206e6f74206170706c69656420746f206d657373616765730e6d6573736167655f6578706972790c0308000000ffffffffffffffff430000004d6573736167652065787069727920696e206d6963726f7365636f6e64732c206f7220612068756d616e74696d6520737472696e672028652e672e20372064617973290e6d61785f746f7069635f73697a650c0308000000ffffffffffffffff9d000000546f7069632d77696465207365616c65642d7365676d656e742073697a65206361702c2073706c6974206163726f737320616c6c20706172746974696f6e732c20696e206279746573206f72206120627974652d73697a6520737472696e672028652e672e203120476942293b2066696e6974652076616c756573206d757374206265206174206c6561737420746865207365676d656e742073697a650c7365676d656e745f73697a650c000800000000000040000000006c0000005365676d656e742073697a6520696e2062797465732c206f72206120627974652d73697a6520737472696e672028652e672e20313238204d6942293b2061203531322d62797465206d756c7469706c652077697468696e20313034383537362e2e3d313037333734313832340a6475726162696c69747902000a0000007265706c6963617465643e0100004d65737361676520636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20412073696e676c65746f6e2071756f72756d20686173206f6e6520636f70792e20496e207265706c6963617465642067726f7570732c20706572736973746564206d65737361676573207573652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c2057414c207265636c616d6174696f6e2e205468652066756c6c20626f64792073697a65207374696c6c20636f756e747320616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e1a636f6e73756d65725f6f66667365745f6475726162696c69747902000a0000007265706c6963617465646e0100004578706c69636974206f666673657420636f6d706c6574696f6e3a207265706c696361746564206f72207065727369737465642e20496e646570656e64656e746c792064656661756c747320746f207265706c6963617465642e20506f6c6c206175746f2d636f6d6d69742072656d61696e73206173796e6368726f6e6f75732e20496e207265706c6963617465642067726f7570732c20706572736973746564206f66667365747320616c736f20656e61626c652057414c207265666572656e63657320746f207365676d656e7420626f646965732c2072657461696e696e6720746865697220696e6f6465732062792068617264206c696e6b20756e74696c207265636c616d6174696f6e2c206576656e2077697468207265706c696361746564206d657373616765206475726162696c6974792e2046756c6c20626f64792073697a657320636f756e7420616761696e737420706172746974696f6e2e77616c5f62797465735f6d61782e17706172746974696f6e5f726573697a655f706f6c6963790200070000006d757461626c65c8000000496d6d757461626c65206166746572206372656174696f6e3a206d757461626c65207065726d69747320706172746974696f6e206164646974696f6e2f64656c6574696f6e20776974686f7574206120626f756e646564206375746f766572206f72207065722d6b6579206f72646572696e67206163726f7373206c61796f7574733b2066697865642072656a6563747320626f74682e20526574656e74696f6e20616e6420746f7069632064656c6574696f6e2072656d61696e2064657374727563746976652e196d657373616765735f72657175697265645f746f5f736176650b000400000000040000870000004f7264696e617279206d6573736167652d636f756e7420666c75736820747269676765723b20312e2e3d31363737373231362e2052657175697265642070657273697374656e63652c2063617061636974792070726573737572652c20616e64206c6966656379636c65206f7065726174696f6e732063616e20666c757368206561726c6965722173697a655f6f665f6d657373616765735f72657175697265645f746f5f736176650c0008000000000010000000000083000000466c75736820746865206a6f75726e616c206f6e636520697420686f6c64732074686973206d616e792062797465732c206f72206120627974652d73697a6520737472696e673b20776869636865766572207468726573686f6c6420747269707320666972737420666c75736865732e204174206d6f7374203130373337343138323414707265616c6c6f636174655f7365676d656e7473030001000000001201000052657175657374207365676d656e745f73697a65206279746573206f66206469736b207265736572766174696f6e207768656e2065616368207365676d656e74206f70656e732e20556e737570706f72746564206f72206661696c6564207265736572766174696f6e732066616c6c206261636b20746f206f7264696e61727920616c6c6f636174696f6e20776974682061207761726e696e672e205265736572766174696f6e2072756e7320696e6c696e65206f6e20746865206f776e696e672073686172642e207365676d656e745f73697a65202a20706172746974696f6e735f636f756e74206973206361707065642061742036383731393437363733362062797465732070657220637265617465127265706c69636174696f6e5f666163746f720b00000000007b0100004e6f64657320746861742073746f7265206561636820706172746974696f6e2c20312e2e3d333220616e64206174206d6f73742074686520636c75737465722773206e6f646520636f756e742e204f6d69747465642c20746865207365727665722773205b636c75737465725d2064656661756c745f7265706c69636174696f6e5f666163746f72206170706c6965733a206576657279206e6f646520756e6c65737320616e206f70657261746f72207365742061206e756d6265722e204120777269746520636f6d6d697473206f6e20612071756f72756d206f66207468657365206e6f6465732c20736f2032206b656570732074776f20636f70696573206275742073746f70732077726974696e67207768696c652065697468657220697320646f776e2c20616e642031206b65657073206f6e6520636f70792074686174207265706c696361746564206475726162696c6974792061636b6e6f776c6564676573206265666f72652069742072656163686573206469736b", "entries": [ { "key": "name", @@ -506,6 +521,12 @@ "kind": "bool", "flags": 0, "default": "false" + }, + { + "key": "replication_factor", + "kind": "uint32", + "flags": 0, + "default": null } ] },
diff --git a/foreign/swift/Tests/IggyTests/ModelVectorTests.swift b/foreign/swift/Tests/IggyTests/ModelVectorTests.swift index 2511c81..1791744 100644 --- a/foreign/swift/Tests/IggyTests/ModelVectorTests.swift +++ b/foreign/swift/Tests/IggyTests/ModelVectorTests.swift
@@ -130,7 +130,7 @@ partitionsCount: 3, compressionAlgorithm: .gzip, messageExpiry: .after(.seconds(604_800)), maxTopicSize: .bytes(1_073_741_824), segmentSizeBytes: 1_048_576, durability: .persisted, consumerOffsetDurability: .replicated, messagesRequiredToSave: 7, sizeOfMessagesRequiredToSaveBytes: 4096, - preallocateSegments: false) + preallocateSegments: false, replicationFactor: 2) #expect(try create.encode() == golden.bytes("options.topic_create.full")) let update = TopicUpdateOptions(messageExpiry: .never, maxTopicSize: .unlimited) #expect(try OptionsBlock.encode(update.toPatch().set) == golden.bytes("options.topic_update.full"))
diff --git a/foreign/swift/Tests/IggyTests/ResourceOptionsTests.swift b/foreign/swift/Tests/IggyTests/ResourceOptionsTests.swift index bb6410d..142cb63 100644 --- a/foreign/swift/Tests/IggyTests/ResourceOptionsTests.swift +++ b/foreign/swift/Tests/IggyTests/ResourceOptionsTests.swift
@@ -300,7 +300,7 @@ let topicCreate = TopicCreateOptions( partitionsCount: 3, compressionAlgorithm: .gzip, messageExpiry: .never, maxTopicSize: .bytes(1 << 30), segmentSizeBytes: 1 << 20, durability: .persisted, consumerOffsetDurability: .persisted, partitionResizePolicy: .fixed, messagesRequiredToSave: 10, - sizeOfMessagesRequiredToSaveBytes: 1 << 10, preallocateSegments: true) + sizeOfMessagesRequiredToSaveBytes: 1 << 10, preallocateSegments: true, replicationFactor: 3) #expect(unsetFields(streamCreate, except: ["raw"]).isEmpty) #expect(unsetFields(topicCreate, except: ["raw"]).isEmpty) #expect(Set(try streamCreate.toResourceOptions().keys) == creatable("stream"))
diff --git a/foreign/swift/Tests/IggyTests/ResponseVectorTests.swift b/foreign/swift/Tests/IggyTests/ResponseVectorTests.swift index efcb1c2..18b62f2 100644 --- a/foreign/swift/Tests/IggyTests/ResponseVectorTests.swift +++ b/foreign/swift/Tests/IggyTests/ResponseVectorTests.swift
@@ -25,11 +25,28 @@ struct ResponseVectorTests { let golden = GoldenFixture.shared let createdAt = IggyTimestamp(microseconds: 1_710_000_000_000_000) + /// `get_topic.rs` pins these bytes for its sample partition, so every SDK + /// decodes the same entry. + let pinnedPartition = [UInt8]( + hex: "01000000004cf1238e0100000200000063000000000000000004000000000000640000000000000017000000000000000000000000000000650000000000000003010304020104" + ) func body(_ name: String) -> ArraySlice<UInt8> { golden.bytes(name)[...] } + /// The pinned partition's counters and context under `id`, then the two + /// node lists. + func partitionEntry(id: UInt32, members: [UInt8], inSyncMembers: [UInt8]) -> [UInt8] { + var entry = Array(pinnedPartition.prefix(64)) + entry.replaceSubrange(0..<4, with: id.littleEndianBytes) + entry.append(UInt8(members.count)) + entry += members + entry.append(UInt8(inSyncMembers.count)) + entry += inSyncMembers + return entry + } + func expectSampleTopic(_ topic: Topic, id: UInt32, name: String) { #expect(topic.id == id) #expect(topic.name == name) @@ -84,20 +101,71 @@ #expect(details.partitions[0].segmentsCount == 2) #expect(details.partitions[2].messagesCount == 8) #expect(details.partitions.map(\.context) == [.empty, .empty, .empty]) + #expect(details.partitions.map(\.members) == [[0, 1, 2], [1, 2, 0], [2, 0, 1]]) + #expect(details.partitions.map(\.inSyncMembers) == [[0, 1], [], [2]]) - // The partitions are the body's tail, 64 bytes each, context last. + // The last partition ends with its context and then its node lists, + // [2, 0, 1] and [2], which take six bytes. var fenced = golden.bytes("response.get_topic") let context = PartitionContext(incarnation: 17, ownerGeneration: 9, metadataOp: 52) - fenced.replaceSubrange(fenced.count - 24..<fenced.count, with: golden.bytes("partition.context")) + let contextEnd = fenced.count - 6 + fenced.replaceSubrange(contextEnd - 24..<contextEnd, with: golden.bytes("partition.context")) let fencedPartitions = try Responses.topicDetails(fenced[...]).partitions #expect(fencedPartitions.map(\.context) == [.empty, .empty, context]) #expect(fencedPartitions[2].messagesCount == 8) + #expect(fencedPartitions[2].members == [2, 0, 1]) let topics = try Responses.topics(body("response.get_topics")) #expect(topics.map(\.name) == ["events", "logs"]) #expect(try Responses.topics(body("response.get_topics.empty")).isEmpty) } + @Test func partitionNodeLists() throws { + #expect(pinnedPartition.count == 71) + var reader = ByteReader(pinnedPartition) + let partition = try Responses.partition(from: &reader) + #expect(reader.isAtEnd) + #expect(partition.id == 1) + #expect(partition.createdAt == IggyTimestamp(microseconds: 1_710_000_000_000)) + #expect(partition.segmentsCount == 2) + #expect(partition.currentOffset == 99) + #expect(partition.sizeBytes == 1024) + #expect(partition.messagesCount == 100) + #expect(partition.context == PartitionContext(incarnation: 23, ownerGeneration: 0, metadataOp: 101)) + #expect(partition.members == [1, 3, 4]) + #expect(partition.inSyncMembers == [1, 4]) + + let unlisted = partitionEntry(id: 1, members: [], inSyncMembers: []) + #expect(unlisted.count == 66) + var unlistedReader = ByteReader(unlisted) + let bare = try Responses.partition(from: &unlistedReader) + #expect(unlistedReader.isAtEnd) + #expect(bare.context == partition.context) + #expect(bare.members.isEmpty) + #expect(bare.inSyncMembers.isEmpty) + + for length in 0..<pinnedPartition.count { + var truncated = ByteReader(Array(pinnedPartition.prefix(length))) + #expect(throws: WireError.self, "\(length)") { try Responses.partition(from: &truncated) } + } + } + + @Test func topicPartitionsAdvanceByTheirOwnLength() throws { + let entries = [ + partitionEntry(id: 0, members: [0, 1, 2], inSyncMembers: [0, 1, 2]), + partitionEntry(id: 1, members: [], inSyncMembers: []), + partitionEntry(id: 2, members: [1, 3, 4], inSyncMembers: [1, 4]), + ] + #expect(entries.map(\.count) == [72, 66, 71]) + let topic = golden.bytes("response.topic_header") + entries.joined() + let partitions = try Responses.topicDetails(topic[...]).partitions + #expect(partitions.map(\.id) == [0, 1, 2]) + #expect(partitions.map(\.members) == [[0, 1, 2], [], [1, 3, 4]]) + #expect(partitions.map(\.inSyncMembers) == [[0, 1, 2], [], [1, 4]]) + #expect(partitions.allSatisfy { $0.messagesCount == 100 && $0.context.metadataOp == 101 }) + #expect(throws: IggyError(.invalidCommand)) { try Responses.topicDetails(topic.dropLast()) } + } + @Test func consumerGroups() throws { let details = try Responses.consumerGroupDetails(body("response.consumer_group_details")) #expect(details.id == 1)
diff --git a/foreign/swift/Tests/IggyTests/Support/FakeVSRServer.swift b/foreign/swift/Tests/IggyTests/Support/FakeVSRServer.swift index 70f2277..618b96f 100644 --- a/foreign/swift/Tests/IggyTests/Support/FakeVSRServer.swift +++ b/foreign/swift/Tests/IggyTests/Support/FakeVSRServer.swift
@@ -313,6 +313,8 @@ writer.write(UInt64(0)) writer.write(UInt64(0)) context.encode(into: &writer) + writer.write(UInt8(0)) + writer.write(UInt8(0)) } return writer.bytes }
diff --git a/foreign/swift/Tools/golden-vectors/src/main.rs b/foreign/swift/Tools/golden-vectors/src/main.rs index c7cfdd3..1390b68 100644 --- a/foreign/swift/Tools/golden-vectors/src/main.rs +++ b/foreign/swift/Tools/golden-vectors/src/main.rs
@@ -110,6 +110,11 @@ use std::collections::{BTreeMap, BTreeSet}; use twox_hash::{XxHash3_64, XxHash32}; +/// The SDK sends `sdkVersion` in every login handshake, so the handshake +/// vectors take it from the Swift source instead of a copy that a version +/// bump would leave behind. +const IGGY_SWIFT_SOURCE: &str = include_str!("../../../Sources/Iggy/Iggy.swift"); + /// Hash and error tables are maps keyed by length and code, so each entry is /// one line of the pretty-printed fixture and the file stays reviewable. #[derive(Serialize)] @@ -211,10 +216,21 @@ ClientVersionInfo { protocol_version: IGGY_PROTOCOL_VERSION, sdk_name: name("swift-sdk"), - sdk_version: name("0.1.5"), + sdk_version: name(swift_sdk_version()), } } +fn swift_sdk_version() -> &'static str { + IGGY_SWIFT_SOURCE + .lines() + .find_map(|line| { + line.trim() + .strip_prefix("public static let sdkVersion = \"")? + .strip_suffix('"') + }) + .expect("Iggy.swift declares sdkVersion") +} + fn sample_permissions() -> WirePermissions { WirePermissions { global: WireGlobalPermissions { @@ -1332,6 +1348,8 @@ current_offset: 99, size_bytes: 1024, messages_count: 100, + members: vec![0, 1, 2], + in_sync_members: vec![0, 1], }, PartitionResponse { context: Default::default(), @@ -1341,6 +1359,8 @@ current_offset: 0, size_bytes: 0, messages_count: 0, + members: vec![1, 2, 0], + in_sync_members: vec![], }, PartitionResponse { context: Default::default(), @@ -1350,6 +1370,8 @@ current_offset: 7, size_bytes: 512, messages_count: 8, + members: vec![2, 0, 1], + in_sync_members: vec![2], }, ]; add( @@ -1817,6 +1839,7 @@ messages_required_to_save: Some(7), size_of_messages_required_to_save: Some(IggyByteSize::from(4096)), preallocate_segments: Some(false), + replication_factor: Some(2), raw: BTreeMap::new(), }; add(
diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml index 3e22b33..113598c 100644 --- a/gateways/kafka/Cargo.toml +++ b/gateways/kafka/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy-gateway-kafka" -version = "0.1.0-edge.2" +version = "0.1.0-edge.3" description = "Kafka wire protocol gateway foundation for Apache Iggy" edition = "2024" license = "Apache-2.0"
diff --git a/gateways/kafka/src/bridge/iggy_bridge/fetch.rs b/gateways/kafka/src/bridge/iggy_bridge/fetch.rs index 30c91e2..eae169e 100644 --- a/gateways/kafka/src/bridge/iggy_bridge/fetch.rs +++ b/gateways/kafka/src/bridge/iggy_bridge/fetch.rs
@@ -662,6 +662,8 @@ size: IggyByteSize::from(size), messages_count, context: iggy::prelude::PartitionContext::default(), + members: Vec::new(), + in_sync_members: Vec::new(), } }