feat(partitions)!: add idempotent producers to Iggy (#4691) Relates to #4513
diff --git a/Cargo.lock b/Cargo.lock index 231d6a0..d84261e 100644 --- a/Cargo.lock +++ b/Cargo.lock
@@ -7091,7 +7091,7 @@ [[package]] name = "iggy" -version = "0.11.1-edge.7" +version = "0.11.1-edge.8" dependencies = [ "async-broadcast", "async-dropper", @@ -7188,7 +7188,7 @@ [[package]] name = "iggy-cli" -version = "0.14.1-edge.6" +version = "0.14.1-edge.7" dependencies = [ "anyhow", "apple-native-keyring-store", @@ -7354,7 +7354,7 @@ [[package]] name = "iggy_binary_protocol" -version = "0.11.1-edge.7" +version = "0.11.1-edge.8" dependencies = [ "aligned-vec", "bytemuck", @@ -7369,7 +7369,7 @@ [[package]] name = "iggy_common" -version = "0.11.1-edge.8" +version = "0.11.1-edge.9" dependencies = [ "aes-gcm 0.11.1", "aes-gcm-siv",
diff --git a/Cargo.toml b/Cargo.toml index a8515ae..e07b16a 100644 --- a/Cargo.toml +++ b/Cargo.toml
@@ -214,11 +214,11 @@ iceberg = "0.10.1" iceberg-catalog-rest = "0.10.1" iceberg-storage-opendal = "0.10.1" -iggy = { path = "core/sdk", version = "0.11.1-edge.7" } -iggy-cli = { path = "core/cli", version = "0.14.1-edge.6" } +iggy = { path = "core/sdk", version = "0.11.1-edge.8" } +iggy-cli = { path = "core/cli", version = "0.14.1-edge.7" } iggy-gateway-kafka = { path = "gateways/kafka" } -iggy_binary_protocol = { path = "core/binary_protocol", version = "0.11.1-edge.7" } -iggy_common = { path = "core/common", version = "0.11.1-edge.8" } +iggy_binary_protocol = { path = "core/binary_protocol", version = "0.11.1-edge.8" } +iggy_common = { path = "core/common", version = "0.11.1-edge.9" } iggy_connector_doris_sink = { path = "core/connectors/sinks/doris_sink" } iggy_connector_sdk = { path = "core/connectors/sdk", version = "0.5.1-edge.2" } iggy_kafka_wire = { path = "core/kafka_wire" }
diff --git a/bdd/python/uv.lock b/bdd/python/uv.lock index f9cc365..e766e0e 100644 --- a/bdd/python/uv.lock +++ b/bdd/python/uv.lock
@@ -8,7 +8,7 @@ [[package]] name = "apache-iggy" -version = "0.9.1.dev8" +version = "0.9.1.dev9" source = { directory = "../../foreign/python" } [package.metadata]
diff --git a/core/bench/src/actors/producer/client/high_level.rs b/core/bench/src/actors/producer/client/high_level.rs index 36ff273..be3c568 100644 --- a/core/bench/src/actors/producer/client/high_level.rs +++ b/core/bench/src/actors/producer/client/high_level.rs
@@ -65,12 +65,16 @@ let user_data_bytes = batch.user_data_bytes; let total_bytes = batch.total_bytes; + // Boxed because the send future is above the `clippy::large_futures` limit. The box is + // allocated before the clock starts, so the measured latency leaves it out. + let send = Box::pin( + self.producer + .as_mut() + .expect("Producer not initialized") + .send(batch.messages), + ); let before_send = Instant::now(); - self.producer - .as_mut() - .expect("Producer not initialized") - .send(batch.messages) - .await?; + send.await?; let latency = before_send.elapsed(); Ok(Some(BatchMetrics {
diff --git a/core/binary_protocol/Cargo.toml b/core/binary_protocol/Cargo.toml index 7c9ce5f..cb968ab 100644 --- a/core/binary_protocol/Cargo.toml +++ b/core/binary_protocol/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy_binary_protocol" -version = "0.11.1-edge.7" +version = "0.11.1-edge.8" description = "Wire protocol types and codec for the Iggy binary protocol. Shared between server and SDK." edition = "2024" rust-version.workspace = true
diff --git a/core/binary_protocol/src/batch.rs b/core/binary_protocol/src/batch.rs index 995abd1..9c8a2e7 100644 --- a/core/binary_protocol/src/batch.rs +++ b/core/binary_protocol/src/batch.rs
@@ -46,14 +46,103 @@ /// Byte offset of `message_count` inside the batch header. pub const BATCH_MESSAGE_COUNT_OFFSET: usize = 48; +/// Byte offset of the producer section of the batch header: a flags byte, +/// then the [`BatchProducer`] fields. +/// +/// ```text +/// [52] flags: bit 0 = producer present [53..56] zero +/// [56..64] producer_id [64..66] epoch [66..68] zero [68..72] base_sequence +/// ``` +pub const BATCH_PRODUCER_OFFSET: usize = 52; + /// Byte offset where the reserved region of the batch header starts. -pub const BATCH_RESERVED_OFFSET: usize = 52; +pub const BATCH_RESERVED_OFFSET: usize = 72; + +const PRODUCER_PRESENT: u8 = 1; +const PRODUCER_ID_OFFSET: usize = 56; +const PRODUCER_EPOCH_OFFSET: usize = 64; +const PRODUCER_SEQUENCE_OFFSET: usize = 68; /// Upper bound on a message's `timestamp_delta`: the field is a `u32` /// microsecond delta against the batch `origin_timestamp`, so a single batch /// spans at most ~71.6 minutes of producer clock. pub const MAX_TIMESTAMP_DELTA_MICROS: u64 = u32::MAX as u64; +/// The idempotent producer that wrote a batch: its id, its epoch, and the +/// sequence number of the batch's first message. +/// +/// The limits are Kafka's ranges, so the Kafka gateway passes the epoch and +/// the sequence of a Kafka batch through unchanged. [`Self::new`] enforces +/// them, so the fields are private. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct BatchProducer { + producer_id: u64, + epoch: u16, + base_sequence: u32, +} + +impl BatchProducer { + pub const PRODUCER_ID_MAX: u64 = i64::MAX as u64; + pub const EPOCH_MAX: u16 = i16::MAX as u16; + /// Sequences wrap to 0 after this value. + pub const SEQUENCE_MAX: u32 = i32::MAX as u32; + + /// `None` when a field is above its maximum. + #[must_use] + pub const fn new(producer_id: u64, epoch: u16, base_sequence: u32) -> Option<Self> { + if producer_id > Self::PRODUCER_ID_MAX + || epoch > Self::EPOCH_MAX + || base_sequence > Self::SEQUENCE_MAX + { + return None; + } + Some(Self { + producer_id, + epoch, + base_sequence, + }) + } + + #[must_use] + pub const fn producer_id(&self) -> u64 { + self.producer_id + } + + #[must_use] + pub const fn epoch(&self) -> u16 { + self.epoch + } + + #[must_use] + pub const fn base_sequence(&self) -> u32 { + self.base_sequence + } + + /// The producer of the batch after this one, when this one holds `count` + /// messages. + #[must_use] + pub const fn next(&self, count: u32) -> Self { + Self { + base_sequence: Self::advance(self.base_sequence, count), + ..*self + } + } + + /// The sequence number of the last message of a batch of `count` + /// messages, with the wrap after [`Self::SEQUENCE_MAX`]. + #[must_use] + pub const fn last_sequence(&self, count: u32) -> u32 { + Self::advance(self.base_sequence, count.saturating_sub(1)) + } + + /// `sequence` moved forward by `by`, with the wrap. + #[must_use] + #[allow(clippy::cast_possible_truncation)] + pub const fn advance(sequence: u32, by: u32) -> u32 { + ((sequence as u64 + by as u64) % (Self::SEQUENCE_MAX as u64 + 1)) as u32 + } +} + /// The batch header. /// /// A producer encodes it with `partition_id`, `base_offset`, and @@ -70,6 +159,9 @@ pub batch_length: u64, pub batch_checksum: u64, pub message_count: u32, + /// Set only for a batch of an idempotent producer. The partition writes + /// a batch of one producer once, also when it is sent again. + pub producer: Option<BatchProducer>, } impl BatchHeader { @@ -88,6 +180,7 @@ batch_length, batch_checksum: 0, message_count, + producer: None, } } @@ -130,6 +223,7 @@ batch_length, batch_checksum: read_u64(bytes, BATCH_CHECKSUM_OFFSET), message_count: read_u32(bytes, BATCH_MESSAGE_COUNT_OFFSET), + producer: decode_producer(bytes)?, }) } @@ -147,6 +241,15 @@ .copy_from_slice(&self.batch_checksum.to_le_bytes()); bytes[BATCH_MESSAGE_COUNT_OFFSET..BATCH_MESSAGE_COUNT_OFFSET + 4] .copy_from_slice(&self.message_count.to_le_bytes()); + if let Some(producer) = self.producer { + bytes[BATCH_PRODUCER_OFFSET] = PRODUCER_PRESENT; + bytes[PRODUCER_ID_OFFSET..PRODUCER_ID_OFFSET + 8] + .copy_from_slice(&producer.producer_id.to_le_bytes()); + bytes[PRODUCER_EPOCH_OFFSET..PRODUCER_EPOCH_OFFSET + 2] + .copy_from_slice(&producer.epoch.to_le_bytes()); + bytes[PRODUCER_SEQUENCE_OFFSET..PRODUCER_SEQUENCE_OFFSET + 4] + .copy_from_slice(&producer.base_sequence.to_le_bytes()); + } } /// Total batch size in bytes (header + blob). @@ -456,6 +559,47 @@ hasher.write(&header.origin_timestamp.to_le_bytes()); hasher.write(&header.batch_length.to_le_bytes()); hasher.write(&header.message_count.to_le_bytes()); + // Hashed only when present, so a batch without a producer keeps the + // checksum every earlier build computed for it. + if let Some(producer) = header.producer { + hasher.write(&[PRODUCER_PRESENT]); + hasher.write(&producer.producer_id.to_le_bytes()); + hasher.write(&producer.epoch.to_le_bytes()); + hasher.write(&producer.base_sequence.to_le_bytes()); + } +} + +/// The producer section of a header whose length `decode` already checked. +fn decode_producer(bytes: &[u8]) -> Result<Option<BatchProducer>, WireError> { + let invalid = || { + WireError::Validation(std::borrow::Cow::Borrowed( + "batch header producer section is malformed", + )) + }; + let section = &bytes[BATCH_PRODUCER_OFFSET..BATCH_RESERVED_OFFSET]; + match bytes[BATCH_PRODUCER_OFFSET] { + 0 if section.iter().all(|&byte| byte == 0) => Ok(None), + PRODUCER_PRESENT + if bytes[BATCH_PRODUCER_OFFSET + 1..PRODUCER_ID_OFFSET] + .iter() + .chain(&bytes[PRODUCER_EPOCH_OFFSET + 2..PRODUCER_SEQUENCE_OFFSET]) + .all(|&byte| byte == 0) => + { + let epoch = u16::from_le_bytes( + bytes[PRODUCER_EPOCH_OFFSET..PRODUCER_EPOCH_OFFSET + 2] + .try_into() + .expect("2-byte slice"), + ); + BatchProducer::new( + read_u64(bytes, PRODUCER_ID_OFFSET), + epoch, + read_u32(bytes, PRODUCER_SEQUENCE_OFFSET), + ) + .map(Some) + .ok_or_else(invalid) + } + _ => Err(invalid()), + } } /// Verify every per-message checksum in `batch` and return the recomputed @@ -660,4 +804,112 @@ bytes[BATCH_HEADER_SIZE - 1] = 1; assert!(BatchHeader::decode(&bytes).is_err()); } + + fn producer() -> BatchProducer { + BatchProducer::new(4_000, 3, 17).unwrap() + } + + fn producer_batch_bytes(frames: &[Vec<u8>]) -> Vec<u8> { + let mut bytes = batch_bytes(frames); + let mut header = BatchHeader::decode(&bytes).unwrap(); + header.producer = Some(producer()); + header.batch_checksum = calculate_batch_checksum(&header, &bytes[BATCH_HEADER_SIZE..]); + header.encode_into(&mut bytes[..BATCH_HEADER_SIZE]); + bytes + } + + #[test] + fn given_a_producer_when_round_tripping_should_keep_it_and_verify() { + let bytes = producer_batch_bytes(&[frame(1, 0, 0, b"payload")]); + + let batch = decode_batch_slice(&bytes).unwrap(); + + assert_eq!(batch.header.producer, Some(producer())); + assert_eq!(bytes[BATCH_PRODUCER_OFFSET], PRODUCER_PRESENT); + } + + #[test] + fn given_no_producer_when_checksumming_should_match_the_earlier_format() { + let bytes = batch_bytes(&[frame(1, 0, 0, b"payload")]); + let header = BatchHeader::decode(&bytes).unwrap(); + let mut hasher = XxHash3_64::new(); + for field in [ + header.partition_id, + header.base_offset, + header.base_timestamp, + header.origin_timestamp, + header.batch_length, + ] { + hasher.write(&field.to_le_bytes()); + } + hasher.write(&header.message_count.to_le_bytes()); + hasher.write(&bytes[BATCH_HEADER_SIZE..BATCH_HEADER_SIZE + 8]); + + assert_eq!(header.batch_checksum, hasher.finish()); + } + + #[test] + fn given_a_changed_producer_field_when_verifying_should_refuse_the_batch() { + let mut bytes = producer_batch_bytes(&[frame(1, 0, 0, b"payload")]); + bytes[PRODUCER_SEQUENCE_OFFSET] ^= 1; + assert!(matches!( + decode_batch_slice(&bytes), + Err(WireError::InvalidBatchChecksum { .. }) + )); + + let mut bytes = producer_batch_bytes(&[frame(1, 0, 0, b"payload")]); + bytes[BATCH_PRODUCER_OFFSET..BATCH_RESERVED_OFFSET].fill(0); + assert!(matches!( + decode_batch_slice(&bytes), + Err(WireError::InvalidBatchChecksum { .. }) + )); + } + + #[test] + fn given_a_malformed_producer_section_when_decoding_should_refuse_it() { + let valid = producer_batch_bytes(&[frame(1, 0, 0, b"payload")]); + let mutations: [(usize, u8); 6] = [ + (BATCH_PRODUCER_OFFSET, 2), + (BATCH_PRODUCER_OFFSET, 3), + (BATCH_PRODUCER_OFFSET + 1, 1), + (PRODUCER_EPOCH_OFFSET + 2, 1), + (PRODUCER_ID_OFFSET + 7, 0x80), + (PRODUCER_SEQUENCE_OFFSET + 3, 0x80), + ]; + for (offset, value) in mutations { + let mut bytes = valid.clone(); + bytes[offset] = value; + assert!( + matches!(BatchHeader::decode(&bytes), Err(WireError::Validation(_))), + "byte {offset} = {value:#x} must be refused" + ); + } + + let mut absent = batch_bytes(&[frame(1, 0, 0, b"payload")]); + absent[PRODUCER_ID_OFFSET] = 1; + assert!(BatchHeader::decode(&absent).is_err()); + } + + #[test] + fn given_fields_above_their_maximum_when_building_a_producer_should_refuse() { + assert!(BatchProducer::new(BatchProducer::PRODUCER_ID_MAX + 1, 0, 0).is_none()); + assert!(BatchProducer::new(0, BatchProducer::EPOCH_MAX + 1, 0).is_none()); + assert!(BatchProducer::new(0, 0, BatchProducer::SEQUENCE_MAX + 1).is_none()); + assert!( + BatchProducer::new( + BatchProducer::PRODUCER_ID_MAX, + BatchProducer::EPOCH_MAX, + BatchProducer::SEQUENCE_MAX, + ) + .is_some() + ); + } + + #[test] + fn given_a_batch_across_the_sequence_limit_when_computing_its_last_sequence_should_wrap() { + let producer = BatchProducer::new(1, 0, BatchProducer::SEQUENCE_MAX - 1).unwrap(); + assert_eq!(producer.last_sequence(2), BatchProducer::SEQUENCE_MAX); + assert_eq!(producer.last_sequence(4), 1); + assert_eq!(producer.last_sequence(1), BatchProducer::SEQUENCE_MAX - 1); + } }
diff --git a/core/binary_protocol/src/requests/messages/send_messages.rs b/core/binary_protocol/src/requests/messages/send_messages.rs index 4d4007f..2ce4e18 100644 --- a/core/binary_protocol/src/requests/messages/send_messages.rs +++ b/core/binary_protocol/src/requests/messages/send_messages.rs
@@ -22,8 +22,8 @@ //! persists, and serves back to polls. use crate::batch::{ - BATCH_HEADER_SIZE, BATCH_MESSAGE_HEADER_SIZE, BatchHeader, MAX_TIMESTAMP_DELTA_MICROS, - calculate_batch_checksum, + BATCH_HEADER_SIZE, BATCH_MESSAGE_HEADER_SIZE, BatchHeader, BatchProducer, + MAX_TIMESTAMP_DELTA_MICROS, calculate_batch_checksum, }; use crate::codec::{WireDecode, WireEncode, read_u32_le}; use crate::error::WireError; @@ -93,6 +93,21 @@ partitioning: &WirePartitioning, messages: &[RawMessage<'_>], ) -> Result<(), WireError> { + Self::encode_with_producer(buf, stream_id, topic_id, partitioning, None, messages) + } + + /// [`Self::encode`] for a batch of an idempotent producer. + /// + /// # Errors + /// See [`Self::encode`]. + pub fn encode_with_producer( + buf: &mut BytesMut, + stream_id: &WireIdentifier, + topic_id: &WireIdentifier, + partitioning: &WirePartitioning, + producer: Option<BatchProducer>, + messages: &[RawMessage<'_>], + ) -> Result<(), WireError> { if messages.is_empty() { return Err(WireError::Validation(std::borrow::Cow::Borrowed( "cannot encode an empty message batch", @@ -174,6 +189,7 @@ } let batch_length = batch_size as u64; let mut batch_header = BatchHeader::new(0, origin_timestamp, batch_length, message_count); + batch_header.producer = producer; batch_header.batch_checksum = calculate_batch_checksum(&batch_header, &buf[blob_start..]); let header_bytes: &mut [u8] = &mut buf[header_start..header_start + BATCH_HEADER_SIZE]; batch_header.encode_into(header_bytes);
diff --git a/core/binary_protocol/src/responses/messages/fetch_messages.rs b/core/binary_protocol/src/responses/messages/fetch_messages.rs index 723c48b..f37f2e6 100644 --- a/core/binary_protocol/src/responses/messages/fetch_messages.rs +++ b/core/binary_protocol/src/responses/messages/fetch_messages.rs
@@ -296,8 +296,8 @@ mod tests { use super::*; use crate::batch::{ - BATCH_HEADER_SIZE, BATCH_RESERVED_OFFSET, BatchHeader, calculate_batch_checksum, - decode_batch_slice, + BATCH_HEADER_SIZE, BATCH_PRODUCER_OFFSET, BATCH_RESERVED_OFFSET, BatchHeader, + calculate_batch_checksum, decode_batch_slice, }; use crate::requests::messages::{ FetchMessagesRequest, FetchMessagesTarget, MIN_DEFERRED_POLL_MAX_BYTES, @@ -613,6 +613,7 @@ .collect::<Vec<_>>() .concat(); let empty_context = "00".repeat(PartitionContext::ENCODED_SIZE); + let no_producer = "00".repeat(BATCH_RESERVED_OFFSET - BATCH_PRODUCER_OFFSET); let reserved = "00".repeat(BATCH_HEADER_SIZE - BATCH_RESERVED_OFFSET); let expected = [ "02000000", @@ -639,6 +640,7 @@ "3501000000000000", "bd2d787d3cdb745c", "01000000", + &no_producer, &reserved, "8e5ca59edbbdbfb9", "01000000000000000000000000000000",
diff --git a/core/cli/Cargo.toml b/core/cli/Cargo.toml index 88e1cfd..e8c7d40 100644 --- a/core/cli/Cargo.toml +++ b/core/cli/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy-cli" -version = "0.14.1-edge.6" +version = "0.14.1-edge.7" edition = "2024" rust-version.workspace = true authors = ["bartosz.ciesla@gmail.com"]
diff --git a/core/common/Cargo.toml b/core/common/Cargo.toml index e90f9e7..5c267dc 100644 --- a/core/common/Cargo.toml +++ b/core/common/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy_common" -version = "0.11.1-edge.8" +version = "0.11.1-edge.9" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2024" rust-version.workspace = true
diff --git a/core/common/src/error/iggy_error.rs b/core/common/src/error/iggy_error.rs index be13e4a..04ed85a 100644 --- a/core/common/src/error/iggy_error.rs +++ b/core/common/src/error/iggy_error.rs
@@ -197,6 +197,29 @@ /// No poll progress was applied. Back off without invalidating the partition route. #[error("Deferred poll capacity exceeded; retry after backoff on the same replica")] DeferredPollCapacityExceeded = 90, + /// This client handed out every producer id of its session. Another + /// client, or this one after it signs in again, gets new ids. + #[error("This client has no producer ids left in its session")] + ProducerIdsExhausted = 91, + /// The batch does not follow the last batch that the partition holds for + /// its producer. Nothing was written. + #[error("The producer batch does not follow the last batch of its producer")] + ProducerSequenceOutOfOrder = 92, + /// A newer epoch of the producer wrote to the partition. Nothing was + /// written. + #[error("A newer epoch of the producer fenced this batch")] + ProducerEpochFenced = 93, + /// The partition holds no state for the producer, and the batch does not + /// start at sequence 0. Nothing was written. The producer starts a new + /// epoch at sequence 0. + #[error( + "The partition does not know this producer, and the batch does not start at sequence 0" + )] + ProducerUnknown = 94, + /// A batch of an idempotent producer names no partition. Its sequence + /// counts per partition, so `Balanced` and a messages key cannot pick one. + #[error("A producer batch must name its partition")] + ProducerPartitionRequired = 95, #[error("Client with ID: {0} was not found.")] ClientNotFound(u32) = 100, #[error("Invalid client ID")]
diff --git a/core/common/src/lib.rs b/core/common/src/lib.rs index e1db4cb..e676a24 100644 --- a/core/common/src/lib.rs +++ b/core/common/src/lib.rs
@@ -77,6 +77,7 @@ pub use http::system::*; pub use http::topics::*; pub use http::users::*; +pub use iggy_binary_protocol::batch::BatchProducer; pub use iggy_binary_protocol::requests::messages::{ MAX_DEFERRED_POLL_TIMEOUT_US, MAX_DEFERRED_POLL_WAIT_US, MIN_DEFERRED_POLL_MAX_BYTES, MIN_DEFERRED_POLL_REPLY_MARGIN_US, @@ -108,7 +109,9 @@ pub use traits::user_client::UserClient; pub use traits::validatable::Validatable; #[doc(hidden)] -pub use traits::{BinarySendPartitioning, fetch_binary_messages, send_binary_messages}; +pub use traits::{ + BinarySendPartitioning, fetch_binary_messages, select_send_partition, send_binary_messages, +}; pub use traits::{binary_sign_in, rust_sdk_version_info, sign_in_request}; pub use traits::{decode_send_confirmations, group_cache_key, sync_group_assignment}; pub use types::args::*;
diff --git a/core/common/src/traits/binary_impls/messages.rs b/core/common/src/traits/binary_impls/messages.rs index 57ea348..f91d856 100644 --- a/core/common/src/traits/binary_impls/messages.rs +++ b/core/common/src/traits/binary_impls/messages.rs
@@ -15,16 +15,16 @@ // specific language governing permissions and limitations // under the License. -use crate::BinaryClient; use crate::traits::binary_auth::fail_if_not_authenticated; use crate::wire_conversions::{ consumer_to_wire, identifier_to_wire, partitioning_to_wire, polling_strategy_to_wire, }; use crate::{ - CommitPolicy, Consumer, DecodeError, DeferredPollOptions, FetchedSection, Identifier, - IggyError, IggyMessage, MessageClient, Partitioning, PollOptions, PolledMessages, + BatchProducer, CommitPolicy, Consumer, DecodeError, DeferredPollOptions, FetchedSection, + Identifier, IggyError, IggyMessage, MessageClient, Partitioning, PollOptions, PolledMessages, PollingStrategy, SendMessagesResponse, }; +use crate::{BinaryClient, VsrSessionControl}; use crate::{ PartitioningKind, TopicClient, binary_operation_deadline, calculate_32, with_binary_deadline, with_binary_lineage, @@ -157,11 +157,13 @@ /// A chunk keeps its selected partition while an earlier lane blocker resolves. /// Only a definitive missing-partition result permits one fresh selection. +/// An idempotent producer rides along, so every send of the chunk carries it. #[doc(hidden)] pub struct BinarySendPartitioning<'a> { original: &'a Partitioning, selected: Option<Cow<'a, Partitioning>>, refreshed: bool, + producer: Option<BatchProducer>, } impl<'a> BinarySendPartitioning<'a> { @@ -170,13 +172,27 @@ original, selected: None, refreshed: false, + producer: None, } } + /// The producer counts its sequence per partition, so `original` must + /// name the partition. + #[must_use] + pub const fn with_producer(mut self, producer: BatchProducer) -> Self { + debug_assert!(matches!(self.original.kind, PartitioningKind::PartitionId)); + self.producer = Some(producer); + self + } + pub fn original(&self) -> &Partitioning { self.original } + pub const fn producer(&self) -> Option<BatchProducer> { + self.producer + } + pub fn is_resolved(&self) -> bool { self.selected.is_some() } @@ -199,13 +215,13 @@ PartitioningKind::PartitionId => Cow::Borrowed(partitioning.original), _ => { let count = topic_partition_count(client, stream_id, topic_id).await?; - Cow::Owned(select_partition( + Cow::Owned(Partitioning::partition_id(select_partition( client, stream_id, topic_id, partitioning.original, count, - )?) + )?)) } }); } @@ -214,7 +230,13 @@ .selected .as_deref() .ok_or(IggyError::InvalidCommand)?; - let payload = encode_messages(stream_id, topic_id, selected, messages)?; + let payload = encode_messages( + stream_id, + topic_id, + selected, + partitioning.producer, + messages, + )?; match client .send_raw_with_context( SEND_MESSAGES_CODE, @@ -230,13 +252,14 @@ partitioning.refreshed = true; let count = fresh_topic_partition_count(client, stream_id, topic_id).await?; - partitioning.selected = Some(Cow::Owned(select_partition( - client, - stream_id, - topic_id, - partitioning.original, - count, - )?)); + partitioning.selected = + Some(Cow::Owned(Partitioning::partition_id(select_partition( + client, + stream_id, + topic_id, + partitioning.original, + count, + )?))); } result => { return result.map(|response| committed_send_confirmations(&response)); @@ -248,13 +271,40 @@ .await } +/// The partition that a send with `partitioning` goes to, picked as +/// [`send_binary_messages`] picks it. With `fresh`, the partition count comes +/// from the topic details instead of a cached route. +#[doc(hidden)] +pub async fn select_send_partition<B: BinaryClient>( + client: &B, + stream_id: &Identifier, + topic_id: &Identifier, + partitioning: &Partitioning, + fresh: bool, +) -> Result<u32, IggyError> { + if partitioning.kind == PartitioningKind::PartitionId { + return partitioning + .value + .get(..4) + .and_then(|bytes| bytes.try_into().ok()) + .map(u32::from_le_bytes) + .ok_or(IggyError::InvalidCommand); + } + let count = if fresh { + fresh_topic_partition_count(client, stream_id, topic_id).await? + } else { + topic_partition_count(client, stream_id, topic_id).await? + }; + select_partition(client, stream_id, topic_id, partitioning, count) +} + fn select_partition<B: BinaryClient>( client: &B, stream_id: &Identifier, topic_id: &Identifier, partitioning: &Partitioning, count: u32, -) -> Result<Partitioning, IggyError> { +) -> Result<u32, IggyError> { if count == 0 { return Err(IggyError::TopicIdNotFound( topic_id.clone(), @@ -268,7 +318,7 @@ PartitioningKind::MessagesKey => calculate_32(&partitioning.value) % count, PartitioningKind::PartitionId => return Err(IggyError::InvalidCommand), }; - Ok(Partitioning::partition_id(partition)) + Ok(partition) } /// Read up to `count` messages from several partitions in one request to their primary, and @@ -510,12 +560,36 @@ ) .await } + + async fn send_producer_messages( + &self, + stream_id: &Identifier, + topic_id: &Identifier, + partition_id: u32, + producer: BatchProducer, + messages: &mut [IggyMessage], + ) -> Result<SendMessagesResponse, IggyError> { + send_binary_messages( + self, + stream_id, + topic_id, + &mut BinarySendPartitioning::new(&Partitioning::partition_id(partition_id)) + .with_producer(producer), + messages, + ) + .await + } + + async fn next_producer_id(&self) -> Result<u64, IggyError> { + VsrSessionControl::next_producer_id(self) + } } fn encode_messages( stream_id: &Identifier, topic_id: &Identifier, partitioning: &Partitioning, + producer: Option<BatchProducer>, messages: &mut [IggyMessage], ) -> Result<Bytes, IggyError> { let wire_stream_id = identifier_to_wire(stream_id)?; @@ -545,11 +619,12 @@ &raw_messages, ); let mut buf = BytesMut::with_capacity(size); - SendMessagesEncoder::encode( + SendMessagesEncoder::encode_with_producer( &mut buf, &wire_stream_id, &wire_topic_id, &wire_partitioning, + producer, &raw_messages, ) .map_err(|error| match error {
diff --git a/core/common/src/traits/binary_impls/mod.rs b/core/common/src/traits/binary_impls/mod.rs index 4cd3606..b9ef6db 100644 --- a/core/common/src/traits/binary_impls/mod.rs +++ b/core/common/src/traits/binary_impls/mod.rs
@@ -28,7 +28,8 @@ mod users; pub use messages::{ - BinarySendPartitioning, decode_send_confirmations, fetch_binary_messages, send_binary_messages, + BinarySendPartitioning, decode_send_confirmations, fetch_binary_messages, + select_send_partition, send_binary_messages, }; pub use messages::{group_cache_key, sync_group_assignment};
diff --git a/core/common/src/traits/binary_transport.rs b/core/common/src/traits/binary_transport.rs index a74de78..4f88c2a 100644 --- a/core/common/src/traits/binary_transport.rs +++ b/core/common/src/traits/binary_transport.rs
@@ -378,6 +378,12 @@ /// Implemented by the transports so the value is the SDK crate's own /// `CARGO_PKG_VERSION` (`iggy` for Rust), not `iggy_common`'s. fn sdk_version(&self) -> &'static str; + /// The next producer id of this client's session. See + /// [`crate::MessageClient::next_producer_id`]. + #[doc(hidden)] + fn next_producer_id(&self) -> Result<u64, IggyError> { + Err(IggyError::FeatureUnavailable) + } } pub use vsr_session_sealed::Sealed as VsrSessionSealed;
diff --git a/core/common/src/traits/message_client.rs b/core/common/src/traits/message_client.rs index ba4abf9..8feabce 100644 --- a/core/common/src/traits/message_client.rs +++ b/core/common/src/traits/message_client.rs
@@ -16,8 +16,8 @@ // under the License. use crate::{ - Consumer, DecodeError, Identifier, IggyError, IggyMessage, Partitioning, PollOptions, - PolledMessages, PollingStrategy, SendMessagesResponse, + BatchProducer, Consumer, DecodeError, Identifier, IggyError, IggyMessage, Partitioning, + PollOptions, PolledMessages, PollingStrategy, SendMessagesResponse, }; use async_trait::async_trait; @@ -151,4 +151,39 @@ partitioning: &Partitioning, messages: &mut [IggyMessage], ) -> Result<SendMessagesResponse, IggyError>; + + /// [`Self::send_messages`] for one batch of an idempotent producer, to + /// the partition `partition_id`. The producer counts its sequence per + /// partition, so the caller names the partition. + /// + /// The partition keeps the last batches of each producer. A batch it + /// already holds is not written again: the reply names the offset of its + /// first write. Only binary clients send a producer, and HTTP returns + /// [`IggyError::FeatureUnavailable`]. + /// + /// A client with an encryptor encrypts `messages` in place. To resend a + /// batch, pass its plaintext messages again. + async fn send_producer_messages( + &self, + stream_id: &Identifier, + topic_id: &Identifier, + partition_id: u32, + producer: BatchProducer, + messages: &mut [IggyMessage], + ) -> Result<SendMessagesResponse, IggyError> { + let _ = (stream_id, topic_id, partition_id, producer, messages); + Err(IggyError::FeatureUnavailable) + } + + /// A producer id that no other client of the cluster gets. It is built + /// from this client's session number, the op of its `Register`, so no + /// request goes to the server. The ids of a session stay unique after the + /// session ends, so a producer can store its id and use it again. + /// + /// Fails with [`IggyError::ProducerIdsExhausted`] after 2^20 ids from one + /// session, and with [`IggyError::Unauthenticated`] before a sign-in. + /// HTTP returns [`IggyError::FeatureUnavailable`]. + async fn next_producer_id(&self) -> Result<u64, IggyError> { + Err(IggyError::FeatureUnavailable) + } }
diff --git a/core/common/src/traits/mod.rs b/core/common/src/traits/mod.rs index db7fab3..ee7d8e5 100644 --- a/core/common/src/traits/mod.rs +++ b/core/common/src/traits/mod.rs
@@ -36,7 +36,8 @@ pub(crate) mod validatable; pub use binary_impls::{ - BinarySendPartitioning, decode_send_confirmations, fetch_binary_messages, send_binary_messages, + BinarySendPartitioning, decode_send_confirmations, fetch_binary_messages, + select_send_partition, send_binary_messages, }; pub use binary_impls::{binary_sign_in, rust_sdk_version_info, sign_in_request}; pub use binary_impls::{group_cache_key, sync_group_assignment};
diff --git a/core/connectors/runtime/README.md b/core/connectors/runtime/README.md index 4a40e18..1a68bc5 100644 --- a/core/connectors/runtime/README.md +++ b/core/connectors/runtime/README.md
@@ -134,6 +134,47 @@ The runtime does not delete remote state. Disabling, removing, or restarting a source preserves its checkpoint. Deletion is an explicit administrative action on the backing store because automatic deletion would make a connector lifecycle operation silently rewind the source. +## Exactly-once sources + +If a source sets `exactly_once = true`, the runtime writes each batch of the source once. This holds after a lost reply, a NACK, or a crash of the runtime. Each batch goes to one partition as a batch of an idempotent producer. The [SDK README](../../sdk/README.md#idempotent-producers) describes idempotent producers. + +```toml +exactly_once = true + +[[streams]] +stream = "orders" +topic = "created" +schema = "json" +partition_id = 0 # the partition of every batch, 0 when unset +``` + +The runtime keeps a producer record in the state of the source, in front of the plugin's bytes. The record holds these values: + +- The producer id. The runtime gets it from the session of its Iggy client. +- The epoch. After each start, batches keep the stored epoch until the first save after a batch, which moves to the next epoch. The new epoch fences an older copy of the runtime. +- The partition, and the first sequence of the next batch in it. + +The runtime saves the state after each batch for which the plugin returns new state. The save moves the next sequence past the batch and stores the plugin's new bytes. A batch without new state is not saved. The plugin gets only its own bytes. A new record, a new epoch or a new partition is also saved before its first batch goes. The `iggy_send` stage histogram includes that save. + +After a NACK or a restart, the plugin produces a batch again from its saved state. No save moved the record past that batch, so the runtime sends it with the same producer id, epoch and sequence. The partition answers with the offset of the first write, and the batch lands once. + +These limits apply: + +- All batches go to one partition. Balanced and messages-key partitioning are not available. +- Each batch of the plugin goes to Iggy in one request. `batch_length` and `linger_time` do not apply. +- The plugin must produce the same messages from the same state. The partition keeps the first copy. +- If the plugin produces another message count, the partition refuses the batch as out of order, and the runtime moves to a new epoch and logs a warning. The unsaved batch can then land twice. +- A partition drops a producer that sent no batch for 24 hours. Above 4096 producers, it drops the one idle the longest. +- If the partition dropped the producer, the runtime moves to a new epoch. The unsaved batch can then land twice. +- The partition keeps the last five batches of a producer. If the plugin returns no state for more than five batches in a row, a restart can write those batches twice. +- If you change `partition_id`, the unsaved batch can land twice. +- Give the topic `persisted` durability. With `replicated`, a quorum of nodes that crash at the same time can lose acknowledged batches. +- The record starts with the bytes `\0IGGYEO\x01`. Plugin state that starts with these bytes is not supported. + +You can turn `exactly_once` on or off for a source that has state. The runtime reads both state formats. +An older runtime does not know the record: its plugin fails to read the state and starts again without its cursor. To roll back, turn `exactly_once` off first and wait until the source saves its state. +Builds released before idempotent producers cannot poll the batches of an exactly-once source. These are the Rust SDK and every client that embeds it (the Python, C++ and PHP SDKs, the CLI, the Kafka gateway and older connector runtimes), and the Swift SDK. Older Iggy nodes cannot store these batches. Upgrade every Iggy node and every consumer of the topic first. + ## Logging By default, the runtime emits human-readable text logs via the `tracing` crate. Switching `logging.format` to `json` produces structured JSON lines (one event per line, machine-parseable). When telemetry is enabled, the chosen format applies to the local stdout layer; OpenTelemetry export is unaffected.
diff --git a/core/connectors/runtime/src/configs/connectors.rs b/core/connectors/runtime/src/configs/connectors.rs index 5e1ac4b..41167d7 100644 --- a/core/connectors/runtime/src/configs/connectors.rs +++ b/core/connectors/runtime/src/configs/connectors.rs
@@ -210,6 +210,8 @@ pub verbose: bool, #[serde(default)] pub benchmark: bool, + #[serde(default)] + pub exactly_once: bool, } impl CreateSourceConfig { @@ -226,6 +228,7 @@ plugin_config: self.plugin_config, verbose: self.verbose, benchmark: self.benchmark, + exactly_once: self.exactly_once, } } } @@ -251,6 +254,10 @@ pub verbose: bool, #[serde(default)] pub benchmark: bool, + /// Each batch goes to one partition with the source's producer, and lands once, also after a + /// crash of the runtime. Needs a plugin that produces the same batch again from its state. + #[serde(default)] + pub exactly_once: bool, } #[derive(Debug, Clone, Default, Serialize, Deserialize)] @@ -288,6 +295,8 @@ pub avro_schema_path: Option<PathBuf>, pub batch_length: Option<u32>, pub linger_time: Option<String>, + /// The partition every message goes to. An exactly-once source uses 0 when it is unset. + pub partition_id: Option<u32>, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -416,7 +425,7 @@ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ enabled: {}, name: {}, path: {}, transforms: {:?}, streams: [{}], plugin_config_format: {:?}, verbose: {}, benchmark: {} }}", + "{{ enabled: {}, name: {}, path: {}, transforms: {:?}, streams: [{}], plugin_config_format: {:?}, verbose: {}, benchmark: {}, exactly_once: {} }}", self.enabled, self.name, self.path, @@ -429,6 +438,7 @@ self.plugin_config_format, self.verbose, self.benchmark, + self.exactly_once, ) } } @@ -471,14 +481,15 @@ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ stream: {}, topic: {}, schema: {:?}, avro_schema_json: {:?}, avro_schema_path: {:?}, batch_length: {:?}, linger_time: {:?} }}", + "{{ stream: {}, topic: {}, schema: {:?}, avro_schema_json: {:?}, avro_schema_path: {:?}, batch_length: {:?}, linger_time: {:?}, partition_id: {:?} }}", self.stream, self.topic, self.schema, self.avro_schema_json, self.avro_schema_path, self.batch_length, - self.linger_time + self.linger_time, + self.partition_id ) } }
diff --git a/core/connectors/runtime/src/exactly_once.rs b/core/connectors/runtime/src/exactly_once.rs new file mode 100644 index 0000000..2f66b8c --- /dev/null +++ b/core/connectors/runtime/src/exactly_once.rs
@@ -0,0 +1,613 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Exactly-once delivery for a source. +//! +//! Each batch goes to one partition with a producer section: the source's producer id, an epoch, +//! and the sequence of its first message. A partition writes a batch of one producer once, so a +//! batch sent again after a lost reply or a crash lands once. +//! +//! The runtime keeps the producer in the source's state, around the plugin's own bytes. The record +//! names the epoch and the sequence of the next batch, and the save of the plugin's new bytes after +//! a batch moves the sequence past it. A batch without new plugin bytes is not saved. After a NACK +//! or a restart, the plugin produces the unsaved batches again from its saved state, and they go +//! with the same sections, so the partition answers them with their first offsets. A new record or +//! a new epoch is saved before its first batch goes. The first save after a batch of a restarted +//! source moves to the next epoch, which fences an older copy of the runtime. + +use std::time::Instant; + +use iggy::prelude::{ + BatchProducer, Client, Identifier, IggyClient, IggyError, IggyMessage, MessageClient, +}; +use iggy_common::BINARY_OPERATION_TIMEOUT; +use iggy_connector_sdk::{ConnectorState, Error as SdkError}; +use tracing::{debug, error, info, warn}; + +use crate::state::StateStorage; + +/// Starts the state of an exactly-once source. No state that a plugin writes starts with these +/// bytes, and the last one is the layout version. +const MAGIC: [u8; 8] = *b"\0IGGYEO\x01"; + +/// Magic, the record, and the plugin flag, before the plugin's bytes. +const HEADER_SIZE: usize = MAGIC.len() + 8 + 2 + 4 + 4 + 1; + +/// Sends of one batch after an unknown outcome or a lost producer, before the batch fails. +const RESENDS: usize = 3; + +/// Where an exactly-once source writes. +#[derive(Debug, Clone)] +pub(crate) struct ExactlyOnceTarget { + pub stream: Identifier, + pub topic: Identifier, + pub partition_id: u32, +} + +/// The producer of a source, as its state keeps it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct ProducerRecord { + pub producer_id: u64, + pub epoch: u16, + /// The partition that `next_sequence` counts for. + pub partition_id: u32, + /// The first sequence of the next batch. A batch that no save settled goes again with it. + pub next_sequence: u32, +} + +/// A stored state, split into the plugin's bytes and the producer record. +#[derive(Debug, Default)] +pub(crate) struct StoredState { + pub plugin: Option<ConnectorState>, + pub record: Option<ProducerRecord>, +} + +impl StoredState { + /// Splits a stored state. A state without the marker is the plugin's own, as a source without + /// exactly-once delivery writes it. + pub(crate) fn split(state: Option<ConnectorState>) -> Result<Self, SdkError> { + let Some(state) = state else { + return Ok(Self::default()); + }; + if !state.0.starts_with(&MAGIC) { + return Ok(Self { + plugin: Some(state), + record: None, + }); + } + let bytes = &state.0; + if bytes.len() < HEADER_SIZE { + return Err(SdkError::InvalidState); + } + let mut reader = Reader { + bytes, + at: MAGIC.len(), + }; + let record = ProducerRecord { + producer_id: reader.u64(), + epoch: reader.u16(), + partition_id: reader.u32(), + next_sequence: reader.u32(), + }; + if record.producer().is_err() { + return Err(SdkError::InvalidState); + } + let has_plugin = reader.flag()?; + let rest = &bytes[HEADER_SIZE..]; + if !has_plugin && !rest.is_empty() { + return Err(SdkError::InvalidState); + } + Ok(Self { + plugin: has_plugin.then(|| ConnectorState(rest.to_vec())), + record: Some(record), + }) + } + + /// A copy of the plugin's bytes, without the record, for the plugin's init. + pub(crate) fn plugin_state(&self) -> Option<ConnectorState> { + self.plugin + .as_ref() + .map(|state| ConnectorState(state.0.clone())) + } +} + +/// The state that an exactly-once source saves: the record, then the plugin's bytes. +pub(crate) fn encode(record: &ProducerRecord, plugin: Option<&[u8]>) -> ConnectorState { + let mut bytes = Vec::with_capacity(HEADER_SIZE + plugin.map_or(0, <[u8]>::len)); + bytes.extend_from_slice(&MAGIC); + bytes.extend_from_slice(&record.producer_id.to_le_bytes()); + bytes.extend_from_slice(&record.epoch.to_le_bytes()); + bytes.extend_from_slice(&record.partition_id.to_le_bytes()); + bytes.extend_from_slice(&record.next_sequence.to_le_bytes()); + bytes.push(u8::from(plugin.is_some())); + if let Some(plugin) = plugin { + bytes.extend_from_slice(plugin); + } + ConnectorState(bytes) +} + +impl ProducerRecord { + /// The producer section of the next batch. + fn producer(&self) -> Result<BatchProducer, IggyError> { + BatchProducer::new(self.producer_id, self.epoch, self.next_sequence) + .ok_or(IggyError::InvalidCommand) + } + + /// The record with the next epoch, whose sequences start at 0. `None` after the last epoch. + fn next_epoch(self) -> Option<Self> { + let epoch = self + .epoch + .checked_add(1) + .filter(|epoch| *epoch <= BatchProducer::EPOCH_MAX)?; + Some(Self { + epoch, + next_sequence: 0, + ..self + }) + } + + /// The record after `producer` wrote a batch of `count` messages. + fn settled(self, producer: BatchProducer, count: u32) -> Self { + Self { + next_sequence: producer.next(count).base_sequence(), + ..self + } + } +} + +/// The exactly-once state of one running source. +#[derive(Debug)] +pub(crate) struct ExactlyOnce { + key: String, + target: ExactlyOnceTarget, + /// The record that the state held at init. The first batch starts from it. + stored: Option<ProducerRecord>, + /// The record of this run, set at the first batch. + record: Option<ProducerRecord>, + /// The plugin's bytes of the last save. A save of a new record keeps them. + plugin: Option<Vec<u8>>, + /// The batch that went out and its message count, until a save settles it. + sent: Option<(BatchProducer, u32)>, + /// An earlier run wrote with the record, so the save after the first batch of this run moves + /// to the next epoch. + fence: bool, +} + +impl ExactlyOnce { + /// Starts from `stored`, the state that init gave the plugin. A reload at the first batch + /// would take the version of another runtime's save, so its conflict would go unseen. + pub(crate) fn new(key: &str, target: ExactlyOnceTarget, stored: StoredState) -> Self { + Self { + key: key.to_owned(), + target, + stored: stored.record, + record: None, + plugin: stored.plugin.map(|state| state.0), + sent: None, + fence: false, + } + } + + /// Sends `messages` as one batch with the producer section of the source. A batch that a crash + /// or a lost reply leaves in doubt goes again with the same section, because no save moved the + /// record past it. + pub(crate) async fn send( + &mut self, + client: &IggyClient, + storage: &StateStorage, + messages: &mut [IggyMessage], + ) -> Result<(), IggyError> { + if messages.is_empty() { + return Ok(()); + } + let count = u32::try_from(messages.len()).map_err(|_| IggyError::InvalidMessagesCount)?; + let mut record = self.record(client, storage).await?; + let mut producer = record.producer()?; + let mut attempts = 0; + loop { + attempts += 1; + match client + .send_producer_messages( + &self.target.stream, + &self.target.topic, + self.target.partition_id, + producer, + messages, + ) + .await + { + Ok(_) => { + self.sent = Some((producer, count)); + return Ok(()); + } + // The same section makes a resend safe, whether or not the first one landed. + Err(IggyError::OperationUncertain(op)) if attempts <= RESENDS => { + if let Err(error) = client.abandon_operation(op).await { + debug!( + "Exactly-once source {} could not release {op}: {error}", + self.key + ); + } + } + Err(IggyError::LaneBlocked(op)) if attempts <= RESENDS => { + if let Err(error) = client + .resolve_operation(op, Instant::now() + BINARY_OPERATION_TIMEOUT) + .await + { + debug!( + "Exactly-once source {} could not resolve {op}: {error}", + self.key + ); + } + } + // The partition forgot the producer, or holds other batches than the state + // names. The next epoch starts at sequence 0, and a batch that the last run wrote + // can then land twice. + Err( + error @ (IggyError::ProducerUnknown | IggyError::ProducerSequenceOutOfOrder), + ) if attempts <= RESENDS => { + warn!( + "Exactly-once source {} moves to a new epoch: {error}", + self.key + ); + record = self.next_epoch(client, record).await?; + self.store(storage, &record).await?; + self.fence = false; + producer = record.producer()?; + } + Err(error) => return Err(error), + } + } + } + + /// Saves the state after a batch: the plugin's new bytes, and the record past the batch that + /// went out. `None` when the plugin gave no new bytes, so nothing is saved. + pub(crate) async fn commit( + &mut self, + client: &IggyClient, + storage: &StateStorage, + plugin: Option<ConnectorState>, + ) -> Option<Result<(), SdkError>> { + let sent = self.sent.take(); + let Some(plugin) = plugin else { + // A restart starts the plugin from its saved bytes, which come before the batch, so + // the stored record keeps the batch's sequence. Only this run moves past it. + if let Some((producer, count)) = sent + && let Some(record) = self.record + { + self.record = Some(record.settled(producer, count)); + } + return None; + }; + let record = match self.record(client, storage).await { + Ok(record) => record, + Err(error) => return Some(Err(SdkError::TransientState(error.to_string()))), + }; + let mut record = match sent { + Some((producer, count)) => record.settled(producer, count), + None => record, + }; + // The batches of this run before this save went at the epoch of the last run, so a replay + // found their first writes. Later batches take the next epoch, which fences an older copy. + let fences = sent.is_some() && self.fence; + if fences { + record = match self.next_epoch(client, record).await { + Ok(record) => record, + Err(error) => return Some(Err(SdkError::TransientState(error.to_string()))), + }; + } + let saved = storage.save(encode(&record, Some(&plugin.0))).await; + if saved.is_ok() { + if fences { + self.fence = false; + info!( + "Exactly-once source {} writes as producer {} at epoch {} to partition {}.", + self.key, record.producer_id, record.epoch, record.partition_id + ); + } + self.record = Some(record); + self.plugin = Some(plugin.0); + } + Some(saved) + } + + /// The record of this run, from the state at init. A new source gets a new producer id. A + /// source that ran before keeps its record, so the batch that no save settled goes again with + /// the same section. + async fn record( + &mut self, + client: &IggyClient, + storage: &StateStorage, + ) -> Result<ProducerRecord, IggyError> { + if let Some(record) = self.record { + return Ok(record); + } + let record = match self.stored { + Some(record) if record.partition_id == self.target.partition_id => { + self.fence = true; + info!( + "Exactly-once source {} resumes as producer {} at epoch {} to partition {}.", + self.key, record.producer_id, record.epoch, record.partition_id + ); + self.record = Some(record); + return Ok(record); + } + Some(record) => { + warn!( + "Exactly-once source {} moved from partition {} to {}. Its unsaved batch can land twice.", + self.key, record.partition_id, self.target.partition_id + ); + ProducerRecord { + partition_id: self.target.partition_id, + ..self.next_epoch(client, record).await? + } + } + None => ProducerRecord { + producer_id: client.next_producer_id().await?, + epoch: 0, + partition_id: self.target.partition_id, + next_sequence: 0, + }, + }; + // A restart must find the producer of a batch that went out before any save settled it. + self.store(storage, &record).await?; + info!( + "Exactly-once source {} writes as producer {} at epoch {} to partition {}.", + self.key, record.producer_id, record.epoch, record.partition_id + ); + Ok(record) + } + + /// `record` at the next epoch, or with a new producer id after the last epoch. + async fn next_epoch( + &self, + client: &IggyClient, + record: ProducerRecord, + ) -> Result<ProducerRecord, IggyError> { + match record.next_epoch() { + Some(record) => Ok(record), + None => Ok(ProducerRecord { + producer_id: client.next_producer_id().await?, + epoch: 0, + next_sequence: 0, + ..record + }), + } + } + + /// Saves `record` with the plugin's last bytes, before a batch goes out with it. + async fn store( + &mut self, + storage: &StateStorage, + record: &ProducerRecord, + ) -> Result<(), IggyError> { + storage + .save(encode(record, self.plugin.as_deref())) + .await + .map_err(|error| { + error!( + "Exactly-once source {} could not store its producer before the send: {error}", + self.key + ); + IggyError::Error + })?; + self.record = Some(*record); + Ok(()) + } +} + +/// Reads the fixed fields of a stored record. The caller checked the length. +struct Reader<'a> { + bytes: &'a [u8], + at: usize, +} + +impl Reader<'_> { + fn take<const N: usize>(&mut self) -> [u8; N] { + let mut field = [0; N]; + field.copy_from_slice(&self.bytes[self.at..self.at + N]); + self.at += N; + field + } + + fn u64(&mut self) -> u64 { + u64::from_le_bytes(self.take()) + } + + fn u32(&mut self) -> u32 { + u32::from_le_bytes(self.take()) + } + + fn u16(&mut self) -> u16 { + u16::from_le_bytes(self.take()) + } + + fn flag(&mut self) -> Result<bool, SdkError> { + match self.take::<1>() { + [0] => Ok(false), + [1] => Ok(true), + _ => Err(SdkError::InvalidState), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::state::FileStateProvider; + + fn record() -> ProducerRecord { + ProducerRecord { + producer_id: 7 << 20, + epoch: 2, + partition_id: 1, + next_sequence: 9, + } + } + + #[test] + fn given_a_record_and_plugin_bytes_when_round_tripped_should_split_back() { + for plugin in [None, Some(&[][..]), Some(&b"\x91\x05"[..])] { + let stored = StoredState::split(Some(encode(&record(), plugin))).unwrap(); + + assert_eq!(stored.record, Some(record())); + assert_eq!( + stored.plugin.map(|state| state.0), + plugin.map(<[u8]>::to_vec) + ); + } + } + + #[test] + fn given_plugin_bytes_without_the_marker_when_split_should_pass_them_through() { + let stored = StoredState::split(Some(ConnectorState(b"\x91\x05".to_vec()))).unwrap(); + + assert_eq!(stored.record, None); + assert_eq!( + stored.plugin.map(|state| state.0), + Some(b"\x91\x05".to_vec()) + ); + assert!(StoredState::split(None).unwrap().plugin.is_none()); + } + + #[test] + fn given_a_damaged_record_when_split_should_refuse_it() { + let state = encode(&record(), Some(b"plugin")).0; + + let short = ConnectorState(state[..HEADER_SIZE - 1].to_vec()); + assert!(matches!( + StoredState::split(Some(short)), + Err(SdkError::InvalidState) + )); + let mut bad_flag = state.clone(); + bad_flag[HEADER_SIZE - 1] = 2; + assert!(matches!( + StoredState::split(Some(ConnectorState(bad_flag))), + Err(SdkError::InvalidState) + )); + let over_max = ProducerRecord { + epoch: BatchProducer::EPOCH_MAX + 1, + ..record() + }; + assert!(matches!( + StoredState::split(Some(encode(&over_max, None))), + Err(SdkError::InvalidState) + )); + let no_plugin = encode(&record(), None).0; + let mut trailing = no_plugin.clone(); + trailing.push(0); + assert!(matches!( + StoredState::split(Some(ConnectorState(trailing))), + Err(SdkError::InvalidState) + )); + } + + #[test] + fn given_a_written_batch_when_settled_should_name_the_sequence_after_it() { + let producer = record().producer().unwrap(); + + assert_eq!( + record().settled(producer, 3), + ProducerRecord { + next_sequence: 12, + ..record() + } + ); + } + + #[test] + fn given_the_last_epoch_when_moving_on_should_need_a_new_id() { + let next = record().next_epoch().unwrap(); + assert_eq!((next.epoch, next.next_sequence), (3, 0)); + + let last = ProducerRecord { + epoch: BatchProducer::EPOCH_MAX, + ..record() + }; + assert_eq!(last.next_epoch(), None); + } + + #[tokio::test] + async fn given_a_batch_without_plugin_state_when_committed_should_keep_the_stored_record() { + let dir = tempfile::TempDir::new().unwrap(); + let storage = StateStorage::File(FileStateProvider::new( + dir.path().join("state").to_string_lossy().to_string(), + )); + storage + .save(encode(&record(), Some(b"saved"))) + .await + .unwrap(); + let init = StoredState::split(storage.load().await.unwrap()).unwrap(); + let target = ExactlyOnceTarget { + stream: Identifier::named("stream").unwrap(), + topic: Identifier::named("topic").unwrap(), + partition_id: record().partition_id, + }; + let mut exactly_once = ExactlyOnce::new("source", target, init); + let client = IggyClient::default(); + let resumed = exactly_once.record(&client, &storage).await.unwrap(); + exactly_once.sent = Some((resumed.producer().unwrap(), 3)); + + assert!(exactly_once.commit(&client, &storage, None).await.is_none()); + + // A restart produces the batch again from the saved bytes, with the same sequence. + let stored = StoredState::split(storage.load().await.unwrap()).unwrap(); + assert_eq!(stored.record, Some(record())); + assert_eq!(stored.plugin.map(|state| state.0), Some(b"saved".to_vec())); + let next = exactly_once.record.unwrap(); + assert_eq!(next.next_sequence, 12); + + exactly_once.sent = Some((next.producer().unwrap(), 2)); + let saved = exactly_once + .commit(&client, &storage, Some(ConnectorState(b"later".to_vec()))) + .await; + + assert!(matches!(saved, Some(Ok(())))); + let stored = StoredState::split(storage.load().await.unwrap()).unwrap(); + assert_eq!(stored.record, record().next_epoch()); + assert_eq!(stored.plugin.map(|state| state.0), Some(b"later".to_vec())); + } + + #[tokio::test] + async fn given_a_save_after_init_when_the_first_batch_goes_should_start_from_the_init_state() { + let dir = tempfile::TempDir::new().unwrap(); + let storage = StateStorage::File(FileStateProvider::new( + dir.path().join("state").to_string_lossy().to_string(), + )); + // Another runtime saved after this one read the state at init. + let other = ProducerRecord { + producer_id: 9 << 20, + ..record() + }; + storage.save(encode(&other, Some(b"other"))).await.unwrap(); + let init = StoredState::split(Some(encode(&record(), Some(b"mine")))).unwrap(); + let target = ExactlyOnceTarget { + stream: Identifier::named("stream").unwrap(), + topic: Identifier::named("topic").unwrap(), + partition_id: record().partition_id, + }; + let mut exactly_once = ExactlyOnce::new("source", target, init); + + let resumed = exactly_once + .record(&IggyClient::default(), &storage) + .await + .unwrap(); + + assert_eq!(resumed, record()); + assert_eq!(exactly_once.plugin.as_deref(), Some(&b"mine"[..])); + } +}
diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index 91c00f6..0b9a98a 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs
@@ -57,6 +57,7 @@ pub(crate) mod configs; pub(crate) mod context; pub(crate) mod error; +mod exactly_once; mod log; mod manager; pub(crate) mod metrics; @@ -584,6 +585,8 @@ struct SourceConnectorProducer { encoder: Arc<dyn StreamEncoder>, producer: IggyProducer, + /// Set when the source delivers exactly once. See `exactly_once`. + exactly_once: Option<exactly_once::ExactlyOnce>, } struct SourceConnectorWrapper {
diff --git a/core/connectors/runtime/src/manager/source.rs b/core/connectors/runtime/src/manager/source.rs index dd1d14f..ddbfb25 100644 --- a/core/connectors/runtime/src/manager/source.rs +++ b/core/connectors/runtime/src/manager/source.rs
@@ -254,12 +254,15 @@ }, other => RuntimeError::ConnectorSdkError(other), })?; + // An exactly-once source keeps its producer around the plugin's bytes. + let stored = crate::exactly_once::StoredState::split(state) + .map_err(RuntimeError::ConnectorSdkError)?; source::init_source( &container, &config.plugin_config.clone().unwrap_or_default(), plugin_id, - state, + stored.plugin_state(), )?; info!("Source connector with ID: {plugin_id} for plugin: {key} initialized successfully."); // Armed from here until the id is recorded below. `PluginInstanceGuard` @@ -267,7 +270,7 @@ let instance_guard = PluginInstanceGuard::for_source(container.clone(), plugin_id, key); let (producer, transforms) = - match source::setup_source_producer(key, config, iggy_client).await { + match source::setup_source_producer(key, config, iggy_client, stored).await { Ok(parts) => parts, Err(error) => { // Awaited rather than left to `drop`, so this error reaches
diff --git a/core/connectors/runtime/src/source.rs b/core/connectors/runtime/src/source.rs index 560d867..c169a58 100644 --- a/core/connectors/runtime/src/source.rs +++ b/core/connectors/runtime/src/source.rs
@@ -21,7 +21,7 @@ use iggy::prelude::StreamCreateOptions; use iggy::prelude::{ Client, DirectConfig, HeaderKey, HeaderValue, Identifier, IggyClient, IggyDuration, IggyError, - IggyMessage, StreamClient, TopicClient, TopicCreateOptions, + IggyMessage, Partitioning, StreamClient, TopicClient, TopicCreateOptions, }; use iggy_common::{BINARY_OPERATION_TIMEOUT, Durability, TopicRuntimeOptions}; use iggy_connector_sdk::encoders::avro::{AvroEncoderConfig, AvroStreamEncoder}; @@ -46,6 +46,7 @@ use crate::benchmark; use crate::configs::connectors::SourceConfig; use crate::context::RuntimeContext; +use crate::exactly_once::{ExactlyOnce, ExactlyOnceTarget, StoredState}; use crate::log::LOG_CALLBACK; use crate::manager::source::StopReport; use crate::metrics::SourceLabels; @@ -182,6 +183,26 @@ } }; + // An exactly-once source keeps its producer around the plugin's bytes. The plugin gets + // only its own. + let stored = match StoredState::split(state) { + Ok(stored) => stored, + Err(error) => { + let message = format!("Failed to read source state: {error}"); + error!("Source: {name} ({key}) - {message}"); + failed_plugins.push(FailedPlugin::new( + plugin_id, + &key, + &name, + &config.path, + config.plugin_config_format, + config.enabled, + message, + )); + continue; + } + }; + if !source_connectors.contains_key(&path) { let container = match unsafe { Container::<SourceApi>::load(&path) } { Ok(container) => container, @@ -220,7 +241,7 @@ &connector.container, &config.plugin_config.clone().unwrap_or_default(), plugin_id, - state, + stored.plugin_state(), ) .err() .map(|error| error.to_string()); @@ -254,7 +275,7 @@ PluginInstanceGuard::for_source(connector.container.clone(), plugin_id, &key) }; - match setup_source_producer(&key, &config, iggy_client).await { + match setup_source_producer(&key, &config, iggy_client, stored).await { Ok((producer, transforms)) => { let connector = source_connectors .get_mut(&path) @@ -332,10 +353,12 @@ } /// Returns the last stream's producer with its encoder, and the transforms. +/// `stored` is the state that init gave the plugin. An exactly-once source starts from it. pub(crate) async fn setup_source_producer( key: &str, config: &SourceConfig, iggy_client: &IggyClient, + stored: StoredState, ) -> Result<(SourceConnectorProducer, Vec<Arc<dyn Transform>>), RuntimeError> { let transforms = if let Some(transforms_config) = &config.transforms { let loaded = transform::load(transforms_config).map_err(|error| { @@ -351,6 +374,7 @@ let mut last_producer = None; let mut last_encoder = None; + let mut last_target = None; for stream in config.streams.iter() { let linger_time = IggyDuration::from_str(stream.linger_time.as_deref().unwrap_or("5ms")) .map_err(|error| { @@ -358,7 +382,7 @@ })?; ensure_durable_source_topic(iggy_client, &stream.stream, &stream.topic).await?; let batch_length = stream.batch_length.unwrap_or(1000); - let producer = iggy_client + let mut builder = iggy_client .producer(&stream.stream, &stream.topic)? // The topic was validated above. If it disappears before init, // recreating it with producer defaults would drop the durability guarantee. @@ -369,9 +393,21 @@ .batch_length(batch_length) .linger_time(linger_time) .build(), - ) - .build(); + ); + if let Some(partition_id) = stream.partition_id { + builder = builder.partitioning(Partitioning::partition_id(partition_id)); + } + let producer = builder.build(); producer.init().await?; + last_target = if config.exactly_once { + Some(ExactlyOnceTarget { + stream: Identifier::named(&stream.stream)?, + topic: Identifier::named(&stream.topic)?, + partition_id: stream.partition_id.unwrap_or(0), + }) + } else { + None + }; let encoder: Arc<dyn StreamEncoder> = match stream.schema { Schema::Avro => { let config = AvroEncoderConfig { @@ -399,7 +435,14 @@ RuntimeError::InvalidConfiguration("No encoder configured for source".to_string()) })?; - Ok((SourceConnectorProducer { encoder, producer }, transforms)) + Ok(( + SourceConnectorProducer { + encoder, + producer, + exactly_once: last_target.map(|target| ExactlyOnce::new(key, target, stored)), + }, + transforms, + )) } async fn ensure_durable_source_topic( @@ -470,7 +513,11 @@ context: Arc<RuntimeContext>, labels: Arc<SourceLabels>, ) { - let SourceConnectorProducer { encoder, producer } = producer; + let SourceConnectorProducer { + encoder, + producer, + mut exactly_once, + } = producer; info!("Source connector with ID: {plugin_id} started."); if benchmark { info!( @@ -582,19 +629,33 @@ let send_result = if state_unavailable { Err(IggyError::Error) } else if processing_errors == 0 { - send_with_failed_tail_retries( - processed.messages, - plugin_id, - &context.iggy_clients.producer, - |messages| producer.send(messages), - |op| { - context - .iggy_clients - .producer - .resolve_operation(op, Instant::now() + BINARY_OPERATION_TIMEOUT) - }, - ) - .await + match exactly_once.as_mut() { + Some(exactly_once) => { + let mut messages = processed.messages; + exactly_once + .send( + &context.iggy_clients.producer, + &state_storage, + &mut messages, + ) + .await + } + None => { + send_with_failed_tail_retries( + processed.messages, + plugin_id, + &context.iggy_clients.producer, + |messages| producer.send(messages), + |op| { + context + .iggy_clients + .producer + .resolve_operation(op, Instant::now() + BINARY_OPERATION_TIMEOUT) + }, + ) + .await + } + } } else { Err(IggyError::Error) }; @@ -662,9 +723,26 @@ } let mut state_saved = true; - if let Some(state) = produced_messages.state { - let state_save_start = Instant::now(); - match state_storage.save(state).await { + let state_save_start = Instant::now(); + // An exactly-once source saves its producer only with new plugin bytes. Without + // them, a restart produces the batch again with the same producer section. + let saved = match exactly_once.as_mut() { + Some(exactly_once) => { + exactly_once + .commit( + &context.iggy_clients.producer, + &state_storage, + produced_messages.state, + ) + .await + } + None => match produced_messages.state { + Some(state) => Some(state_storage.save(state).await), + None => None, + }, + }; + if let Some(saved) = saved { + match saved { Ok(()) => { debug!("State saved for source connector with ID: {plugin_id}"); let state_save_elapsed = state_save_start.elapsed(); @@ -1216,6 +1294,7 @@ .producer("stream", "topic") .expect("producer builder should initialize") .build(), + exactly_once: None, }; let context = Arc::new(RuntimeContext { sinks: SinkManager::new(vec![]),
diff --git a/core/connectors/sdk/README.md b/core/connectors/sdk/README.md index 822bf47..ad374c1 100644 --- a/core/connectors/sdk/README.md +++ b/core/connectors/sdk/README.md
@@ -31,6 +31,10 @@ An ACK follows Iggy's quorum confirmation. The topic's `durability` policy decides whether that confirmation also waits for stable storage on the quorum. Both policies normally write messages to disk. +A source with `exactly_once = true` removes the duplicates of the second and third rows. Before each send, the runtime stores the producer id, epoch and first sequence of the batch in the source's state. After a NACK or a crash, the plugin produces the batch again from its saved state. + +If that batch has the same message count, the runtime sends it with the stored values. The partition then answers with the offset of the first write and writes nothing. The plugin must produce the same messages from the same state, because the partition keeps the first copy. The [runtime README](../runtime/README.md#exactly-once-sources) describes the configuration and the limits. + Source-side ACK work should be idempotent because process termination can interrupt it. NACK handling must discard staged cursor changes and staged delete or mark operations so polling can redeliver the batch. The SDK retries NACKed batches with capped exponential backoff and stops after five consecutive NACKs by default. A source whose accepted input cannot be re-read after a restart can override `Source::batch_policy()` and call `BatchPolicy::with_max_consecutive_nacks(None)` to keep retrying until shutdown. The SDK warns after 30 seconds without a batch result, but keeps that batch in flight until the runtime replies or the source shuts down.
diff --git a/core/connectors/sources/README.md b/core/connectors/sources/README.md index d7e53a3..09646b7 100644 --- a/core/connectors/sources/README.md +++ b/core/connectors/sources/README.md
@@ -51,6 +51,7 @@ pub plugin_config: Option<serde_json::Value>, pub verbose: bool, // Log message processing at info level instead of debug (default: false) pub benchmark: bool, // Emit per-batch timing events on the `iggy_connectors::benchmark` target (default: false) + pub exactly_once: bool, // Write each batch once, also after a crash of the runtime (default: false) } ``` @@ -77,6 +78,7 @@ plugin_config_format = "toml" verbose = false # Log message processing at info level instead of debug benchmark = false # Emit per-batch timing events on `iggy_connectors::benchmark` target +exactly_once = false # Write each batch once, also after a crash of the runtime # Collection of the streams to which the produced messages are sent [[streams]] @@ -85,6 +87,7 @@ schema = "json" batch_length = 100 linger_time = "5ms" +partition_id = 0 # Optional: the partition of every message, 0 for an exactly-once source when unset # Custom configuration for the source connector, deserialized to type T from `plugin_config` field [plugin_config] @@ -110,6 +113,8 @@ IGGY_CONNECTORS_SOURCE_RANDOM_ENABLED=false ``` +The [runtime README](../runtime/README.md#exactly-once-sources) describes `exactly_once` and its limits. + ## Sample implementation Let's implement the example source connector, which will simply generate the N random messages depending on the count specified in the configuration.
diff --git a/core/integration/tests/cluster/idempotent_producer.rs b/core/integration/tests/cluster/idempotent_producer.rs new file mode 100644 index 0000000..9574bb7 --- /dev/null +++ b/core/integration/tests/cluster/idempotent_producer.rs
@@ -0,0 +1,509 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Idempotent producers: a batch carries its producer id, epoch and +//! base sequence in the batch header, through the prepare, to the segment +//! file and back to polls. The partition writes each batch of a producer +//! once, also when another connection sends it again after a primary change. + +use iggy::prelude::*; +use iggy_binary_protocol::batch::BatchHeader; +use iggy_binary_protocol::codes::GET_PARTITION_ROUTE_CODE; +use iggy_binary_protocol::requests::messages::GetPartitionRouteRequest; +use iggy_binary_protocol::responses::messages::PartitionRouteResponse; +use iggy_binary_protocol::{WireDecode, WireEncode}; +use iggy_common::wire_conversions::identifier_to_wire; +use integration::harness::{TestBinary, TestHarness}; +use integration::iggy_harness; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::time::Duration; +use tokio::time::{Instant, sleep}; + +const STREAM_NAME: &str = "producers"; +const TOPIC_NAME: &str = "orders"; +const PARTITION_ID: u32 = 0; +const RECOVER_TIMEOUT: Duration = Duration::from_secs(60); +const NODES: usize = 3; +const COMMIT_SPREAD: Duration = Duration::from_secs(1); +const POLL_INTERVAL: Duration = Duration::from_millis(200); + +#[iggy_harness(cluster_nodes = 1, server(sharding.cpu_allocation = "0..1"))] +async fn given_a_producer_batch_when_the_node_restarts_should_serve_it_and_keep_the_producer( + harness: &mut TestHarness, +) { + let client = connect(harness).await; + create_topic(&client).await; + let producer = BatchProducer::new(7_000, 2, 0).expect("a producer within the field limits"); + let payloads = ["first", "second", "third"]; + + let sent = send(&client, producer, &payloads).await; + assert_eq!(sent, Some(0), "the first batch lands at offset 0"); + drop(client); + + harness.node_mut(0).stop().expect("stop the node"); + harness.node_mut(0).start().expect("restart the node"); + + let client = reconnect(harness).await; + assert_eq!(poll_payloads(&client).await, payloads); + assert_eq!( + send(&client, producer, &payloads).await, + Some(0), + "the WAL replay must rebuild the producer table" + ); + assert_eq!(poll_payloads(&client).await, payloads); + let stored = stored_batches(&harness.node(0).data_path()); + assert!( + stored + .iter() + .any(|header| header.producer == Some(producer) && header.message_count == 3), + "the segment file must hold the batch with its producer, found {stored:?}" + ); +} + +#[iggy_harness(cluster_nodes = 1, server(sharding.cpu_allocation = "0..1"))] +async fn given_a_producer_when_batches_repeat_skip_or_go_back_an_epoch_should_write_each_once( + harness: &mut TestHarness, +) { + let producer = |epoch, base_sequence| BatchProducer::new(9_000, epoch, base_sequence).unwrap(); + let first = connect(harness).await; + create_topic(&first).await; + assert_eq!( + send(&first, producer(0, 0), &["a", "b", "c"]).await, + Some(0) + ); + // A new connection is a new session, as a gateway that takes over has. + drop(first); + + let second = connect(harness).await; + assert_eq!( + send(&second, producer(0, 0), &["a", "b", "c"]).await, + Some(0), + "a resent batch gets the offset of its first write" + ); + assert_eq!(send(&second, producer(0, 3), &["d"]).await, Some(3)); + assert_eq!( + try_send(&second, producer(0, 9), &["gap"]).await, + Err(IggyError::ProducerSequenceOutOfOrder) + ); + assert_eq!(send(&second, producer(1, 0), &["e"]).await, Some(4)); + assert_eq!( + try_send(&second, producer(0, 4), &["stale"]).await, + Err(IggyError::ProducerEpochFenced) + ); + let unknown = BatchProducer::new(9_001, 0, 5).unwrap(); + assert_eq!( + try_send(&second, unknown, &["late"]).await, + Err(IggyError::ProducerUnknown), + "a producer that the partition does not hold starts at sequence 0" + ); + assert_eq!(poll_payloads(&second).await, ["a", "b", "c", "d", "e"]); +} + +#[iggy_harness(cluster_nodes = 3, server(sharding.cpu_allocation = "0..1"))] +async fn given_a_batch_resent_after_its_primary_stops_when_sent_through_another_node_should_answer_its_first_offset( + harness: &mut TestHarness, +) { + let producer = |base_sequence| BatchProducer::new(11_000, 0, base_sequence).unwrap(); + let other = BatchProducer::new(12_000, 0, 0).unwrap(); + let first = harness + .root_client_for_node(0) + .await + .expect("connect to node 0"); + create_topic(&first).await; + assert_eq!(send(&first, producer(0), &["a", "b", "c"]).await, Some(0)); + // The next prepare carries the commit of the first, and the wait covers a + // commit heartbeat, so the backups record the batch before the stop. + assert_eq!(send(&first, other, &["x"]).await, Some(3)); + sleep(COMMIT_SPREAD).await; + let primary = partition_primary(&first).await; + drop(first); + + harness + .node_mut(primary) + .stop() + .expect("stop the partition primary"); + let survivor = (0..NODES) + .find(|node| *node != primary) + .expect("a node survives"); + let deadline = Instant::now() + RECOVER_TIMEOUT; + let (second, resent) = loop { + if let Ok(client) = harness.root_client_for_node(survivor).await + && let Ok(resent) = try_send(&client, producer(0), &["a", "b", "c"]).await + { + break (client, resent); + } + assert!( + Instant::now() < deadline, + "no node answered the resent batch within {RECOVER_TIMEOUT:?}" + ); + sleep(POLL_INTERVAL).await; + }; + + assert_eq!(resent, Some(0), "the new primary must know the first write"); + assert_eq!(send(&second, producer(3), &["d"]).await, Some(4)); + assert_eq!(poll_payloads(&second).await, ["a", "b", "c", "x", "d"]); +} + +/// A 128 MiB WAL checkpoints after two batches of 33 MiB, as in `partition_dedup`. +#[iggy_harness( + cluster_nodes = 1, + server(partition.wal_bytes_max = "134225920 B", sharding.cpu_allocation = "0..1") +)] +async fn given_a_checkpointed_producer_table_when_the_node_restarts_should_still_know_the_first_write( + harness: &mut TestHarness, +) { + const CHECKPOINT_OP: u64 = 2; + let producer = |base_sequence| BatchProducer::new(13_000, 0, base_sequence).unwrap(); + let client = connect(harness).await; + create_topic(&client).await; + assert_eq!( + try_send_messages(&client, Some(producer(0)), bulky_messages(1)).await, + Ok(Some(0)) + ); + assert_eq!( + try_send_messages(&client, None, bulky_messages(2)).await, + Ok(Some(u64::from(BULKY_MESSAGES))) + ); + await_receipt_checkpoint(harness, CHECKPOINT_OP).await; + drop(client); + + harness.node_mut(0).stop().expect("stop the node"); + harness.node_mut(0).start().expect("restart the node"); + + let client = reconnect(harness).await; + assert_eq!( + try_send_messages(&client, Some(producer(0)), bulky_messages(1)).await, + Ok(Some(0)), + "the checkpoint must hold the first write" + ); + assert_eq!( + send(&client, producer(BULKY_MESSAGES), &["next"]).await, + Some(2 * u64::from(BULKY_MESSAGES)), + "the resent batch must not take offsets" + ); +} + +#[iggy_harness(cluster_nodes = 1, server(sharding.cpu_allocation = "0..1"))] +async fn given_two_clients_and_a_new_sign_in_when_taking_producer_ids_should_never_repeat_one( + harness: &mut TestHarness, +) { + let first = connect(harness).await; + create_topic(&first).await; + let second = connect(harness).await; + + let first_id = first.next_producer_id().await.expect("an id of the first"); + let second_id = second + .next_producer_id() + .await + .expect("an id of the second"); + assert_ne!(first_id >> 20, second_id >> 20, "each session owns a range"); + assert_eq!(first.next_producer_id().await, Ok(first_id + 1)); + + first.logout_user().await.expect("sign out"); + first + .login_user(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD) + .await + .expect("sign in again"); + let renewed = first + .next_producer_id() + .await + .expect("an id of the new session"); + assert!( + renewed >> 20 > (first_id >> 20).max(second_id >> 20), + "a new session starts a range above every earlier one" + ); + + let producer = BatchProducer::new(renewed, 0, 0).expect("an id within the field limits"); + assert_eq!(send(&first, producer, &["a"]).await, Some(0)); + assert_eq!(send(&second, producer, &["a"]).await, Some(0)); +} + +#[iggy_harness(cluster_nodes = 1, server(sharding.cpu_allocation = "0..1"))] +async fn given_an_idempotent_producer_when_its_chunk_comes_again_should_answer_the_first_offset( + harness: &mut TestHarness, +) { + let client = connect(harness).await; + create_topic(&client).await; + let producer = client + .producer(STREAM_NAME, TOPIC_NAME) + .expect("a producer builder") + .partitioning(Partitioning::partition_id(PARTITION_ID)) + .direct(DirectConfig::builder().batch_length(1).build()) + .idempotent() + .build(); + producer.init().await.expect("init the producer"); + let messages = ["a", "b"] + .iter() + .map(|payload| IggyMessage::from_str(payload).expect("message")) + .collect(); + + let sent = producer.send(messages).await.expect("send two chunks"); + + let offsets: Vec<u64> = sent + .confirmations + .iter() + .map(|confirmation| confirmation.base_offset) + .collect(); + assert_eq!(offsets, [0, 1]); + // A batch can show in more than one `.log` file under the data path. + let mut written: Vec<BatchProducer> = Vec::new(); + for producer in stored_batches(&harness.node(0).data_path()) + .iter() + .filter_map(|header| header.producer) + { + if !written.contains(&producer) { + written.push(producer); + } + } + let id = written + .first() + .expect("a batch with a producer") + .producer_id(); + let numbered = |base_sequence| BatchProducer::new(id, 0, base_sequence).unwrap(); + assert_eq!(written, [numbered(0), numbered(1)]); + assert_eq!( + send(&client, numbered(0), &["a"]).await, + Some(0), + "the first chunk, sent again after a lost reply, answers its first offset" + ); + assert_eq!(poll_payloads(&client).await, ["a", "b"]); +} + +pub(crate) async fn connect(harness: &TestHarness) -> IggyClient { + harness + .node(0) + .tcp_client() + .expect("tcp client builder") + .with_root_login() + .connect() + .await + .expect("connect to the node") +} + +/// A client of the restarted node, once it serves the topic again. +async fn reconnect(harness: &TestHarness) -> IggyClient { + let deadline = Instant::now() + RECOVER_TIMEOUT; + loop { + if let Ok(builder) = harness.node(0).tcp_client() + && let Ok(client) = builder.with_root_login().connect().await + && client + .get_topic(&stream_id(), &topic_id()) + .await + .is_ok_and(|topic| topic.is_some()) + { + return client; + } + assert!( + Instant::now() < deadline, + "the node did not recover within {RECOVER_TIMEOUT:?}" + ); + sleep(POLL_INTERVAL).await; + } +} + +/// One persisted partition that writes each commit to its segment. +pub(crate) async fn create_topic(client: &IggyClient) { + client + .create_stream(STREAM_NAME, &StreamCreateOptions::default()) + .await + .expect("create stream"); + client + .create_topic( + &stream_id(), + TOPIC_NAME, + &TopicCreateOptions { + partitions_count: Some(1), + message_expiry: Some(IggyExpiry::NeverExpire), + messages_required_to_save: Some(1), + durability: Durability::Persisted, + ..TopicCreateOptions::default() + }, + ) + .await + .expect("create topic"); +} + +/// The offset of the batch's first message, as the reply confirms it. +pub(crate) async fn send( + client: &IggyClient, + producer: BatchProducer, + payloads: &[&str], +) -> Option<u64> { + try_send(client, producer, payloads) + .await + .expect("send a producer batch") +} + +async fn try_send( + client: &IggyClient, + producer: BatchProducer, + payloads: &[&str], +) -> Result<Option<u64>, IggyError> { + let messages = payloads + .iter() + .map(|payload| IggyMessage::from_str(payload).expect("message")) + .collect(); + try_send_messages(client, Some(producer), messages).await +} + +async fn try_send_messages( + client: &IggyClient, + producer: Option<BatchProducer>, + mut messages: Vec<IggyMessage>, +) -> Result<Option<u64>, IggyError> { + let partitioning = Partitioning::partition_id(PARTITION_ID); + let response = match producer { + Some(producer) => { + client + .send_producer_messages( + &stream_id(), + &topic_id(), + PARTITION_ID, + producer, + &mut messages, + ) + .await? + } + None => { + client + .send_messages(&stream_id(), &topic_id(), &partitioning, &mut messages) + .await? + } + }; + Ok(response + .confirmations + .first() + .map(|confirmation| confirmation.base_offset)) +} + +const BULKY_MESSAGES: u32 = 33; + +/// 33 messages of 1 MiB, a quarter of the 128 MiB WAL. +fn bulky_messages(fill: u8) -> Vec<IggyMessage> { + (0..BULKY_MESSAGES) + .map(|_| { + IggyMessage::builder() + .payload(vec![fill; 1024 * 1024].into()) + .build() + .expect("build a message") + }) + .collect() +} + +/// Wait until the partition published the retry checkpoint of `op`, which also +/// lets it reclaim the WAL below. +async fn await_receipt_checkpoint(harness: &TestHarness, op: u64) { + let name = format!("receipts-{op}.checkpoint"); + let deadline = Instant::now() + RECOVER_TIMEOUT; + loop { + let mut files = Vec::new(); + collect_files(&harness.node(0).data_path(), &mut files, &|path| { + path.file_name().is_some_and(|file| *file == *name) + }); + if !files.is_empty() { + return; + } + assert!( + Instant::now() < deadline, + "the partition never published {name}" + ); + sleep(POLL_INTERVAL).await; + } +} + +/// The node that holds the partition's primary. +async fn partition_primary(client: &IggyClient) -> usize { + let request = GetPartitionRouteRequest { + stream_id: identifier_to_wire(&stream_id()).expect("stream id"), + topic_id: identifier_to_wire(&topic_id()).expect("topic id"), + partition_id: PARTITION_ID, + }; + let route = client + .send_binary_request(GET_PARTITION_ROUTE_CODE, request.to_bytes()) + .await + .expect("route the partition"); + usize::from( + PartitionRouteResponse::decode_from(&route) + .expect("decode the route") + .primary_replica, + ) +} + +pub(crate) async fn poll_payloads(client: &IggyClient) -> Vec<String> { + client + .poll_messages( + &stream_id(), + &topic_id(), + PARTITION_ID, + &Consumer::default(), + &PollingStrategy::offset(0), + PollOptions { + count: 100, + ..PollOptions::default() + }, + ) + .await + .expect("poll the partition") + .messages + .iter() + .map(|message| String::from_utf8_lossy(&message.payload).into_owned()) + .collect() +} + +fn stream_id() -> Identifier { + Identifier::named(STREAM_NAME).expect("stream identifier") +} + +fn topic_id() -> Identifier { + Identifier::named(TOPIC_NAME).expect("topic identifier") +} + +/// The header of every batch in the node's segment files. +fn stored_batches(data_path: &Path) -> Vec<BatchHeader> { + let mut segments = Vec::new(); + collect_files(data_path, &mut segments, &|path| { + path.extension().is_some_and(|extension| extension == "log") + }); + let mut headers = Vec::new(); + for segment in segments { + let bytes = std::fs::read(&segment).expect("read a segment file"); + let mut position = 0; + while let Ok(header) = BatchHeader::decode(&bytes[position..]) { + position += header.total_size(); + headers.push(header); + if position >= bytes.len() { + break; + } + } + } + headers +} + +fn collect_files(directory: &Path, files: &mut Vec<PathBuf>, wanted: &dyn Fn(&Path) -> bool) { + let Ok(entries) = std::fs::read_dir(directory) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + collect_files(&path, files, wanted); + } else if wanted(&path) { + files.push(path); + } + } +}
diff --git a/core/integration/tests/cluster/mod.rs b/core/integration/tests/cluster/mod.rs index 6bc85ee..1ef1dda 100644 --- a/core/integration/tests/cluster/mod.rs +++ b/core/integration/tests/cluster/mod.rs
@@ -24,6 +24,7 @@ mod crash_recovery_corruption; mod failover_client_continuity; mod fast_primary_rejoin; +mod idempotent_producer; mod metadata_changes; mod metadata_checkpoint_restart; mod metadata_state_transfer;
diff --git a/core/integration/tests/connectors/runtime/http_state.rs b/core/integration/tests/connectors/runtime/http_state.rs index af69ac6..801a6e1 100644 --- a/core/integration/tests/connectors/runtime/http_state.rs +++ b/core/integration/tests/connectors/runtime/http_state.rs
@@ -30,10 +30,16 @@ //! * `503` burst mid-stream -> saves Nack, then recover once the store is //! healthy again, //! * API-driven restart -> the source reloads the served state and resumes -//! the ETag chain. +//! the ETag chain, +//! * exactly-once source whose save after a send fails -> the batch that the +//! plugin produces again lands once, +//! * exactly-once source restarted before its first save after a send -> the +//! batch lands once, and new batches use the next epoch. use assert_cmd::prelude::CommandCargoExt; use async_trait::async_trait; +use iggy::prelude::IggyClient; +use iggy_common::{Consumer, Identifier, MessageClient, PollOptions, PollingStrategy}; use iggy_connector_sdk::api::{ConnectorRuntimeStats, ConnectorStatus, SourceInfoResponse}; use integration::harness::config::TestServerConfig; use integration::harness::{TestBinaryError, TestFixture, TestHarness, seeds}; @@ -57,6 +63,20 @@ const POLL_INTERVAL: Duration = Duration::from_millis(50); const IDEMPOTENCY_KEY_HEADER: HeaderName = HeaderName::from_static("idempotency-key"); const STATE_URL_QUERY_SECRET: &str = "state-query-secret-not-for-logs"; +const EXACTLY_ONCE_KEY: &str = "random_exactly_once"; +const EXACTLY_ONCE_RESOURCE_PATH: &str = "/source_random_exactly_once"; +/// Starts the state of an exactly-once source. +const EXACTLY_ONCE_MAGIC: &[u8] = b"\0IGGYEO\x01"; +/// The next sequence, after the magic, producer id, epoch and partition id. +const EXACTLY_ONCE_NEXT_SEQUENCE_AT: usize = 22; +/// The `messages_range` and `max_count` of the exactly-once source. +const EXACTLY_ONCE_BATCH: usize = 3; +const EXACTLY_ONCE_MESSAGES: usize = 60; +/// Both attempts of one save at `max_attempts = 1`. +const PUTS_PER_FAILED_SAVE: u64 = 2; +/// Time for a batch that a source writes twice to reach the topic. +const SETTLE_TIME: Duration = Duration::from_millis(500); +const POLL_BATCH: u32 = 1_000; /// In-memory state server backing the wiremock responder. Enforces the /// conditional-write contract and exposes counters plus injectable failure @@ -68,6 +88,9 @@ committed_requests: StdMutex<HashMap<String, (Vec<u8>, String)>>, conflict_mode: AtomicBool, fail_next_puts: AtomicU64, + /// Fails writes of an exactly-once state past the first batch, which the runtime saves after a + /// send. + fail_next_settled_puts: AtomicU64, get_count: AtomicU64, put_count: AtomicU64, } @@ -121,6 +144,16 @@ } return ResponseTemplate::new(200).insert_header(ETAG, etag.as_str()); } + if stored_next_sequence(&request.body).is_some_and(|sequence| sequence > 0) + && store + .fail_next_settled_puts + .try_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + { + return ResponseTemplate::new(503); + } if store .fail_next_puts .try_update(Ordering::SeqCst, Ordering::SeqCst, |remaining| { @@ -175,23 +208,72 @@ #[async_trait] impl TestFixture for HttpStateStoreFixture { async fn setup() -> Result<Self, TestBinaryError> { - let server = MockServer::start().await; - let store = Arc::new(SharedStore::default()); - Mock::given(path(RESOURCE_PATH)) - .respond_with(StateStoreResponder(store.clone())) - .mount(&server) - .await; + let (server, store) = start_state_server(RESOURCE_PATH, SharedStore::default()).await; Ok(Self { server, store }) } fn connectors_runtime_envs(&self) -> HashMap<String, String> { - HashMap::from([( - "IGGY_CONNECTORS_STATE_HTTP_URL".to_string(), - self.server.uri(), - )]) + state_server_envs(&self.server) } } +/// The state server of the exactly-once source. Its first `FAILED_SETTLED_PUTS` writes of a +/// state past the first batch answer `503`, so the save after a send fails and the runtime Nacks a +/// batch that Iggy holds. +pub struct ExactlyOnceStateStore<const FAILED_SETTLED_PUTS: u64> { + server: MockServer, + store: Arc<SharedStore>, +} + +#[async_trait] +impl<const FAILED_SETTLED_PUTS: u64> TestFixture for ExactlyOnceStateStore<FAILED_SETTLED_PUTS> { + async fn setup() -> Result<Self, TestBinaryError> { + let store = SharedStore { + fail_next_settled_puts: AtomicU64::new(FAILED_SETTLED_PUTS), + ..SharedStore::default() + }; + let (server, store) = start_state_server(EXACTLY_ONCE_RESOURCE_PATH, store).await; + Ok(Self { server, store }) + } + + fn connectors_runtime_envs(&self) -> HashMap<String, String> { + state_server_envs(&self.server) + } +} + +/// The save after the first send fails. +type LostSaveFixture = ExactlyOnceStateStore<PUTS_PER_FAILED_SAVE>; +/// Every save after a send fails until the test clears the count. +type FailingSavesFixture = ExactlyOnceStateStore<{ u64::MAX }>; + +async fn start_state_server( + resource_path: &str, + store: SharedStore, +) -> (MockServer, Arc<SharedStore>) { + let server = MockServer::start().await; + let store = Arc::new(store); + Mock::given(path(resource_path)) + .respond_with(StateStoreResponder(store.clone())) + .mount(&server) + .await; + (server, store) +} + +fn state_server_envs(server: &MockServer) -> HashMap<String, String> { + HashMap::from([("IGGY_CONNECTORS_STATE_HTTP_URL".to_string(), server.uri())]) +} + +/// The first sequence of the next batch that the state of an exactly-once source names. `None` +/// for any other state. +fn stored_next_sequence(body: &[u8]) -> Option<u32> { + if !body.starts_with(EXACTLY_ONCE_MAGIC) { + return None; + } + body.get(EXACTLY_ONCE_NEXT_SEQUENCE_AT..EXACTLY_ONCE_NEXT_SEQUENCE_AT + 4) + .and_then(|bytes| bytes.try_into().ok()) + .map(u32::from_le_bytes) +} + async fn wait_until<F: Fn() -> bool>(what: &str, condition: F) { let deadline = Instant::now() + WAIT_DEADLINE; while !condition() { @@ -202,7 +284,7 @@ } } -async fn fetch_source(harness: &TestHarness) -> SourceInfoResponse { +async fn fetch_source(harness: &TestHarness, key: &str) -> SourceInfoResponse { let api_url = harness .connectors_runtime() .expect("connectors runtime handle should be available") @@ -217,14 +299,14 @@ .expect("Failed to parse sources"); sources .into_iter() - .find(|source| source.key == SOURCE_KEY) + .find(|source| source.key == key) .expect("HTTP-state source should be reported") } -async fn wait_for_status(harness: &TestHarness, expected: ConnectorStatus) { +async fn wait_for_status(harness: &TestHarness, key: &str, expected: ConnectorStatus) { let deadline = Instant::now() + WAIT_DEADLINE; loop { - let source = fetch_source(harness).await; + let source = fetch_source(harness, key).await; if source.status == expected { return; } @@ -238,6 +320,87 @@ } } +async fn wait_for_last_error(harness: &TestHarness, key: &str, expected: &str) { + let deadline = Instant::now() + WAIT_DEADLINE; + loop { + let source = fetch_source(harness, key).await; + if source + .last_error + .as_ref() + .is_some_and(|error| error.message.contains(expected)) + { + return; + } + if Instant::now() >= deadline { + panic!( + "timed out after {WAIT_DEADLINE:?} waiting for source error {expected:?}, last seen {:?}", + source.last_error + ); + } + sleep(POLL_INTERVAL).await; + } +} + +async fn restart_source(harness: &TestHarness, key: &str) { + let api_url = harness + .connectors_runtime() + .expect("connectors runtime handle should be available") + .http_url(); + let response = Client::new() + .post(format!("{api_url}/sources/{key}/restart")) + .send() + .await + .expect("restart request should be sent"); + assert!( + response.status().is_success(), + "restart should succeed, got {}", + response.status() + ); +} + +/// The messages that partition 0 of the seeded topic holds. +async fn stored_messages(client: &IggyClient) -> usize { + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); + let mut stored = 0; + loop { + let polled = client + .poll_messages( + &stream_id, + &topic_id, + 0, + &Consumer::default(), + &PollingStrategy::offset(stored as u64), + PollOptions { + count: POLL_BATCH, + ..PollOptions::default() + }, + ) + .await + .expect("poll the source topic"); + if polled.messages.is_empty() { + return stored; + } + stored += polled.messages.len(); + } +} + +async fn wait_for_messages(client: &IggyClient, expected: usize) { + let deadline = Instant::now() + WAIT_DEADLINE; + loop { + let stored = stored_messages(client).await; + if stored >= expected { + return; + } + if Instant::now() >= deadline { + panic!( + "timed out after {WAIT_DEADLINE:?} waiting for {expected} messages, the topic holds {stored}" + ); + } + sleep(POLL_INTERVAL).await; + } +} + #[tokio::test] async fn given_unavailable_state_store_when_booting_should_fail_startup() { let mut harness = TestHarness::builder() @@ -307,7 +470,7 @@ fixture: HttpStateStoreFixture, ) { crate::connectors::random_source_liveness::assert_produces_messages(harness).await; - wait_for_status(harness, ConnectorStatus::Running).await; + wait_for_status(harness, SOURCE_KEY, ConnectorStatus::Running).await; wait_until("the first checkpoint PUT to reach the state server", || { fixture.store.version.load(Ordering::SeqCst) > 0 }) @@ -332,7 +495,7 @@ .await; fixture.store.conflict_mode.store(true, Ordering::SeqCst); - wait_for_status(harness, ConnectorStatus::Error).await; + wait_for_status(harness, SOURCE_KEY, ConnectorStatus::Error).await; let version_after_conflict = fixture.store.version.load(Ordering::SeqCst); let puts_after_conflict = fixture.store.put_count.load(Ordering::SeqCst); @@ -350,7 +513,7 @@ let deadline = Instant::now() + WAIT_DEADLINE; loop { - let source = fetch_source(harness).await; + let source = fetch_source(harness, SOURCE_KEY).await; if source .last_error .as_ref() @@ -460,3 +623,79 @@ "the plugin should restore the served state on restart" ); } + +#[iggy_harness( + server(connectors_runtime( + config_path = "tests/connectors/runtime/http_state_exactly_once.toml" + )), + seed = seeds::connector_stream +)] +async fn given_a_failed_save_after_a_send_when_the_source_sends_again_should_write_the_batch_once( + harness: &TestHarness, + fixture: LostSaveFixture, +) { + let client = harness.root_client().await.expect("root client"); + + wait_for_messages(&client, EXACTLY_ONCE_MESSAGES).await; + sleep(SETTLE_TIME).await; + + assert_eq!( + fixture.store.fail_next_settled_puts.load(Ordering::SeqCst), + 0, + "the save after the first send must fail" + ); + assert_eq!( + stored_messages(&client).await, + EXACTLY_ONCE_MESSAGES, + "the Nacked batch went out again and must land once" + ); + assert_eq!( + stored_next_sequence(&fixture.store.body.lock().expect("store lock")), + u32::try_from(EXACTLY_ONCE_MESSAGES).ok(), + "the last save must settle the last batch" + ); +} + +#[iggy_harness( + server(connectors_runtime( + config_path = "tests/connectors/runtime/http_state_exactly_once.toml" + )), + seed = seeds::connector_stream +)] +async fn given_an_unsaved_batch_when_the_source_restarts_should_write_the_batch_once( + harness: &TestHarness, + fixture: FailingSavesFixture, +) { + // No save after a send lands, so the stored state names the first batch until the source + // stops at its Nack limit. + wait_for_last_error(harness, EXACTLY_ONCE_KEY, "consecutive NACK limit reached").await; + let client = harness.root_client().await.expect("root client"); + assert_eq!(stored_messages(&client).await, EXACTLY_ONCE_BATCH); + assert_eq!( + stored_next_sequence(&fixture.store.body.lock().expect("store lock")), + Some(0), + "the stored state must name the first batch" + ); + fixture + .store + .fail_next_settled_puts + .store(0, Ordering::SeqCst); + + restart_source(harness, EXACTLY_ONCE_KEY).await; + wait_for_messages(&client, EXACTLY_ONCE_MESSAGES).await; + sleep(SETTLE_TIME).await; + + assert_eq!( + stored_messages(&client).await, + EXACTLY_ONCE_MESSAGES, + "the unsaved batch went out again after the restart and must land once" + ); + let (stdout, stderr) = harness + .connectors_runtime() + .expect("connectors runtime handle should be available") + .collect_logs(); + assert!( + format!("{stdout}\n{stderr}").contains("at epoch 1 to partition 0"), + "the restarted source must write at the next epoch" + ); +}
diff --git a/core/integration/tests/connectors/runtime/http_state_exactly_once.toml b/core/integration/tests/connectors/runtime/http_state_exactly_once.toml new file mode 100644 index 0000000..9adf437 --- /dev/null +++ b/core/integration/tests/connectors/runtime/http_state_exactly_once.toml
@@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[connectors] +config_type = "local" +config_dir = "tests/connectors/runtime/http_state_exactly_once_config" + +# The state server URL is injected per test via IGGY_CONNECTORS_STATE_HTTP_URL. +[state] +storage = "http" + +[state.http] +timeout = "2s" + +[state.http.retry] +enabled = true +max_attempts = 1 +initial_backoff = "10ms" +max_backoff = "50ms" +backoff_multiplier = 2
diff --git a/core/integration/tests/connectors/runtime/http_state_exactly_once_config/random_source.toml b/core/integration/tests/connectors/runtime/http_state_exactly_once_config/random_source.toml new file mode 100644 index 0000000..7176922 --- /dev/null +++ b/core/integration/tests/connectors/runtime/http_state_exactly_once_config/random_source.toml
@@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "source" +key = "random_exactly_once" +enabled = true +version = 0 +name = "Random source with exactly-once delivery" +path = "../../target/debug/libiggy_connector_random_source" +plugin_config_format = "json" +exactly_once = true + +[[streams]] +stream = "test_stream" +topic = "test_topic" +schema = "json" +batch_length = 100 +linger_time = "5ms" +partition_id = 0 + +# Every batch holds 3 messages, so a batch that the source produces again has the count of the +# batch in flight. +[plugin_config] +interval = "100ms" +max_count = 60 +messages_range = [3, 4] +payload_size = 50
diff --git a/core/kafka_wire/src/api.rs b/core/kafka_wire/src/api.rs index 5ae9fa2..75624f0 100644 --- a/core/kafka_wire/src/api.rs +++ b/core/kafka_wire/src/api.rs
@@ -57,8 +57,8 @@ /// Retriable, nothing written. The Produce stub, and a partition refused because the request /// budget ran out. pub const ERROR_NOT_LEADER_OR_FOLLOWER: i16 = ResponseError::NotLeaderOrFollower.code(); -/// Outcome unknown: the write may have landed. Retriable, so a retry can duplicate it. -/// Idempotent produce (#3545) closes that. +/// Outcome unknown: the write may have landed. Retriable. A retry of an idempotent batch writes +/// nothing new, and a retry of any other batch can write it twice. pub const ERROR_REQUEST_TIMED_OUT: i16 = ResponseError::RequestTimedOut.code(); /// `OffsetCommit`: a partition's metadata is over Kafka's `offset.metadata.max.bytes`. Not retried. pub const ERROR_OFFSET_METADATA_TOO_LARGE: i16 = ResponseError::OffsetMetadataTooLarge.code(); @@ -178,6 +178,16 @@ /// The member id does not hold the `group_instance_id` it named, or a static member that /// returned has since replaced it. pub const ERROR_FENCED_INSTANCE_ID: i16 = 82; +/// Produce: the batch does not follow the last batch of its producer. +/// +/// A batch older than the batches that the partition keeps gets it too. The Java producer retries +/// it, and moves to a new epoch when the gap is real. +pub const ERROR_OUT_OF_ORDER_SEQUENCE_NUMBER: i16 = ResponseError::OutOfOrderSequenceNumber.code(); +/// Produce: a newer epoch of the producer wrote to the partition. +pub const ERROR_INVALID_PRODUCER_EPOCH: i16 = ResponseError::InvalidProducerEpoch.code(); +/// Produce: the partition no longer holds the producer, and the batch does not start at +/// sequence 0. The Java producer moves to a new epoch and sends the batch again from 0. +pub const ERROR_UNKNOWN_PRODUCER_ID: i16 = ResponseError::UnknownProducerId.code(); /// Produce: a record or batch this gateway cannot map. /// /// Not `CORRUPT_MESSAGE` (2), whose text fits but which `kafka-protocol`'s table marks
diff --git a/core/kafka_wire/src/records.rs b/core/kafka_wire/src/records.rs index 4d734ba..396fc17 100644 --- a/core/kafka_wire/src/records.rs +++ b/core/kafka_wire/src/records.rs
@@ -899,6 +899,25 @@ (id != NO_PRODUCER_ID).then_some(id) } +/// The idempotent producer of a record batch, as its header names it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProducerFields { + pub producer_id: i64, + pub producer_epoch: i16, + pub base_sequence: i32, +} + +/// The producer of the first batch in `blob`: its id, bytes 43 to 50, its epoch, bytes 51 and +/// 52, and the sequence of its first record, bytes 53 to 56. `None` when it has no producer id. +#[must_use] +pub fn batch_producer(blob: &[u8]) -> Option<ProducerFields> { + Some(ProducerFields { + producer_id: producer_id(blob)?, + producer_epoch: i16::from_be_bytes(blob.get(51..53)?.try_into().ok()?), + base_sequence: i32::from_be_bytes(blob.get(53..57)?.try_into().ok()?), + }) +} + /// The record count the first batch in `blob` declares, bytes 57 to 60. `None` when it has none. #[must_use] pub fn declared_records(blob: &[u8]) -> Option<usize> { @@ -971,8 +990,7 @@ /// - The decoder reserves from `record_count` (`kafka-protocol-0.18.0/src/records.rs:517`), so /// the count must fit the blob, plus the partition cap when compressed. /// - Control and transactional batches are refused: a stored message cannot carry their flags. -/// An idempotent batch passes, its producer id, epoch and sequence ignored, because a stock -/// Java producer is idempotent. `IDEMPOTENCE.md` has why a retry is then not deduplicated. +/// An idempotent batch passes. Produce reads its producer with [`batch_producer`]. fn preflight(buf: &Bytes, budget: &DecompressionBudget, zstd: Zstd) -> Result<()> { let mut rest = buf.clone(); let infos = RecordBatchDecoder::decode_batch_info(&mut rest) @@ -2520,6 +2538,26 @@ } #[test] + fn given_an_idempotent_batch_when_its_producer_is_read_should_return_the_first_sequence() { + let mut records = vec![record_at_offset(0, b"a"), record_at_offset(1, b"b")]; + for (sequence, record) in (17..).zip(records.iter_mut()) { + record.producer_id = 42; + record.producer_epoch = 3; + record.sequence = sequence; + } + assert_eq!( + batch_producer(&encode_with(&records, Compression::None)), + Some(ProducerFields { + producer_id: 42, + producer_epoch: 3, + base_sequence: 17, + }) + ); + let unowned = encode_with(&[record_at_offset(0, b"v")], Compression::None); + assert_eq!(batch_producer(&unowned), None); + } + + #[test] fn given_any_record_shape_when_encoded_should_fit_the_size_bound() { let shapes = [ record_with(None, None, &[]),
diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index 9d7b556..61705d6 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs
@@ -31,6 +31,7 @@ DiskReadPlan, DiskSegment, PartitionDirResolution, PollContext, PollPlan, PollReadResult, PollTier, ResidentTailSnapshot, }; +use crate::producer_state::{ProducerAdmission, ProducerEntry, ProducerTable, StampedBatch}; use crate::segment::Segment; use crate::state_transfer::{ PartitionTransferSession, PartitionTransferUnavailable, PendingTransferRearm, TransferPlanSlot, @@ -56,6 +57,7 @@ replicate_preflight, restamp_prepare_view, send_prepare_ok as send_prepare_ok_common, verify_prepare_integrity, }; +use iggy_binary_protocol::batch::BatchProducer; use iggy_binary_protocol::primitives::consumer::WireConsumer; use iggy_binary_protocol::primitives::partition_history::ConsumerGroupOwner; use iggy_binary_protocol::requests::consumer_offsets::{ @@ -142,6 +144,10 @@ /// group, where preallocating the cap would reserve hundreds of KiB per /// partition before a client connects). dedup: ClientTable, + /// Committed state of idempotent producers. Every replica records a + /// committed batch in op order. The primary checks a producer's batch + /// against it and the uncommitted prepares before the batch gets offsets. + producers: ProducerTable, pub log: SegmentedLog<PartitionJournal<PartitionJournalMemStorage>>, /// Highest durably persisted offset. pub offset: Arc<AtomicU64>, @@ -906,6 +912,7 @@ consensus::PARTITION_DEDUP_CLIENTS_MAX, ClientTableMode::PartitionSlice, ), + producers: ProducerTable::default(), log: SegmentedLog::default(), offset: Arc::new(AtomicU64::new(0)), dirty_offset: AtomicU64::new(0), @@ -1483,6 +1490,21 @@ &self.consensus } + /// The committed producer table, ascending by producer id, as a + /// checkpoint or a transfer offer carries it. + pub(crate) fn producer_entries(&self) -> Vec<(u64, ProducerEntry)> { + self.producers + .entries() + .map(|(producer_id, entry)| (producer_id, *entry)) + .collect() + } + + /// Replace the committed producer table with the one a checkpoint or a + /// transfer carried. + pub(crate) fn restore_producers(&mut self, entries: Vec<(u64, ProducerEntry)>) { + self.producers = ProducerTable::from_entries(entries); + } + /// This group's dedup slice. Read at admission to classify a request, /// written only from the commit path. #[must_use] @@ -3082,6 +3104,7 @@ consumer_group_owners: std::mem::take(&mut self.consumer_group_owners), history_transition: self.history_transition, owner_retirement: self.owner_retirement.take(), + producers: std::mem::take(&mut self.producers), durable_offset: self.offset.load(Ordering::Acquire), write_offset: self.dirty_offset.load(Ordering::Relaxed), offset_space: self.offset_space, @@ -3122,11 +3145,13 @@ consumer_group_owners, history_transition, owner_retirement, + producers, } = state; self.log = log; self.consumer_group_owners = consumer_group_owners; self.history_transition = history_transition; self.owner_retirement = owner_retirement; + self.producers = producers; if let Some(capacity) = retry_capacity { self.dedup .install_watermarks(capacity, retry_protection) @@ -4670,6 +4695,117 @@ prepare } + /// Checks the batch of an idempotent producer right before the request + /// becomes a prepare and its batch gets offsets. Answers a batch that the + /// partition does not write, and returns whether the request goes on. + async fn admit_producer_batch( + &self, + message: &Message<RoutedRequestHeader>, + reply: &mut Option<consensus::Sender<Message<ReplyHeader>>>, + ) -> bool { + let header = message.header(); + let refusal = match self.check_producer_batch(message) { + Ok(ProducerAdmission::Append) => return true, + Ok(ProducerAdmission::Duplicate { base_offset }) => { + // The batch committed, so the answer is a committed result, + // which also settles a send the client holds as uncertain. + let duplicate = build_reply_from_request( + &self.consensus, + header, + send_confirmation_body(header.group, base_offset), + ) + .transmute_header(|mut duplicate, output: &mut ReplyHeader| { + duplicate.reserved[0] = ReplyHeader::COMMITTED_RESULT; + duplicate.partition_incarnation = header.partition_incarnation; + *output = duplicate; + }); + Self::deliver_reply_or_log( + &self.consensus, + header, + CachedReply::from(duplicate), + reply.take(), + "producer duplicate reply send failed", + ) + .await; + return false; + } + // The first copy can still commit, so the outcome of this send + // stays unknown, as for a request that is still pending. + Ok(ProducerAdmission::DuplicateInFlight) => IggyError::TransientNotCommitted, + Ok(ProducerAdmission::OutOfOrder) => IggyError::ProducerSequenceOutOfOrder, + Ok(ProducerAdmission::Unknown) => IggyError::ProducerUnknown, + Ok(ProducerAdmission::StaleEpoch) => IggyError::ProducerEpochFenced, + Err(error) => error, + }; + Self::send_partition_deny_or_log( + &self.consensus, + header, + refusal.as_code(), + "producer batch refusal reply failed", + reply.take(), + ) + .await; + false + } + + fn check_producer_batch( + &self, + message: &Message<RoutedRequestHeader>, + ) -> Result<ProducerAdmission, IggyError> { + if message.header().operation != Operation::SendMessages { + return Ok(ProducerAdmission::Append); + } + let batch = message + .as_slice() + .get(std::mem::size_of::<RoutedRequestHeader>()..) + .and_then(|body| BatchHeader::decode(body).ok()) + .ok_or(IggyError::InvalidCommand)?; + let Some(producer) = batch.producer else { + return Ok(ProducerAdmission::Append); + }; + let in_flight = self.in_flight_producer_batches(producer.producer_id())?; + Ok(self + .producers + .admit(producer, batch.message_count, &in_flight)) + } + + /// The batches of `producer_id` in the uncommitted prepares, in op order. + /// A prepare this replica cannot read leaves the answer unknown, so the + /// sender is told to retry. + fn in_flight_producer_batches(&self, producer_id: u64) -> Result<Vec<StampedBatch>, IggyError> { + let head = self.consensus.sequencer().current_sequence(); + let mut batches = Vec::new(); + for op in self.consensus.commit_min().saturating_add(1)..=head { + let entry = self + .log + .journal() + .inner + .repair_entry(op) + .filter(|entry| !entry.is_empty()) + .ok_or(IggyError::TransientNotAccepted)?; + if peek_operation(&entry) != Operation::SendMessages { + continue; + } + let batch = entry + .as_slice() + .get(std::mem::size_of::<PrepareHeader>()..) + .and_then(|body| BatchHeader::decode(body).ok()) + .ok_or(IggyError::TransientNotAccepted)?; + if let Some(producer) = batch + .producer + .filter(|producer| producer.producer_id() == producer_id) + { + batches.push(StampedBatch { + producer, + message_count: batch.message_count, + base_offset: batch.base_offset, + timestamp: batch.base_timestamp, + }); + } + } + Ok(batches) + } + fn local_request_must_queue(&self) -> bool { self.consensus.pipeline_is_full() || self @@ -8083,6 +8219,9 @@ .await; return; } + if !self.admit_producer_batch(&message, &mut reply).await { + return; + } if let Some((kind, consumer_id, Some(_), _)) = consumer_offset && !self .admit_consumer_offset_key(message.header(), kind, consumer_id, &mut reply) @@ -8356,6 +8495,15 @@ continue; } } + if !self + .admit_producer_batch(&request.message, &mut reply_sender) + .await + { + if Self::record_promotion_denial(&mut consecutive_denials) { + break; + } + continue; + } consecutive_denials = 0; if queued_install.is_some() { self.pending_owner_install.set(queued_install); @@ -11731,6 +11879,14 @@ } fn publish_committed_batch_stats(&mut self, batch_stats: CommittedBatchStats) { + if let Some(producer) = batch_stats.producer { + self.producers.record(&StampedBatch { + producer, + message_count: batch_stats.message_count, + base_offset: batch_stats.base_offset, + timestamp: batch_stats.timestamp, + }); + } self.widest_batch_bytes .set(self.widest_batch_bytes.get().max(batch_stats.size_bytes)); let end_offset = batch_stats.end_offset(); @@ -11805,6 +11961,8 @@ base_offset: batch.header.base_offset, message_count, size_bytes: batch.header.total_size() as u64, + producer: batch.header.producer, + timestamp: batch.header.base_timestamp, })) } @@ -13817,14 +13975,18 @@ /// it reads the legacy server's empty body. That is also why absence must stay /// absent - a placeholder entry would carry a valid stream/topic/partition/ /// offset tuple and be indistinguishable from a real commit at offset 0. -#[allow(clippy::cast_possible_truncation)] fn send_messages_reply_body( namespace: u64, batch_stats: Option<CommittedBatchStats>, ) -> bytes::Bytes { - let Some(stats) = batch_stats else { - return bytes::Bytes::from_static(&[0, 0, 0, 0]); - }; + batch_stats.map_or_else( + || bytes::Bytes::from_static(&[0, 0, 0, 0]), + |stats| send_confirmation_body(namespace, stats.base_offset), + ) +} + +#[allow(clippy::cast_possible_truncation)] +fn send_confirmation_body(namespace: u64, base_offset: u64) -> bytes::Bytes { let namespace = IggyNamespace::from_raw(namespace); SendMessagesResponse { confirmations: vec![SendMessagesConfirmationResponse { @@ -13833,7 +13995,7 @@ stream_id: namespace.stream_id() as u32, topic_id: namespace.topic_id() as u32, partition_id: namespace.partition_id() as u32, - base_offset: stats.base_offset, + base_offset, }], } .to_bytes() @@ -13848,6 +14010,8 @@ base_offset: u64, message_count: u32, size_bytes: u64, + producer: Option<BatchProducer>, + timestamp: u64, } struct JournaledMessages { @@ -15953,6 +16117,7 @@ external_groups: Vec::new(), dedup: Vec::new(), retirement_paid_through: 0, + producers: Vec::new(), }; partition .install_state_transfer(&repair_config(), 1, staged, &offsets.encode()) @@ -16034,6 +16199,7 @@ external_groups: Vec::new(), dedup: Vec::new(), retirement_paid_through: 0, + producers: Vec::new(), }; partition .install_state_transfer(&repair_config(), 3, staged, &offsets.encode()) @@ -16085,6 +16251,7 @@ external_groups: Vec::new(), dedup: Vec::new(), retirement_paid_through: 0, + producers: Vec::new(), }; assert!(matches!( partition @@ -16312,6 +16479,7 @@ history_transition: None, owner_retirement: None, retirement_paid_through: 0, + producers: Vec::new(), }; let result = partition @@ -19574,6 +19742,7 @@ history_transition: None, owner_retirement: None, retirement_paid_through: 0, + producers: Vec::new(), }; let installed = backup .install_state_transfer(&repair_config(), 1, vec![staged], &offsets.encode()) @@ -20487,6 +20656,7 @@ owners: Vec::new(), history_transition: None, retirement_paid_through: 0, + producers: Vec::new(), }; partition .install_state_transfer(&config, 2, vec![staged], &offsets.encode()) @@ -24141,6 +24311,7 @@ external_groups: Vec::new(), dedup: Vec::new(), retirement_paid_through: 0, + producers: Vec::new(), }; let outcome = partition .install_state_transfer(&repair_config(), 12, Vec::new(), &offer.encode()) @@ -25257,6 +25428,7 @@ external_groups: Vec::new(), dedup: Vec::new(), retirement_paid_through: 0, + producers: Vec::new(), }; partition .install_state_transfer(&repair_config(), 12, Vec::new(), &wire.encode()) @@ -25496,6 +25668,8 @@ base_offset: 0, message_count: 1, size_bytes: 0, + producer: None, + timestamp: 0, }), ), ); @@ -27020,6 +27194,7 @@ external_groups: Vec::new(), dedup: Vec::new(), retirement_paid_through: 0, + producers: Vec::new(), }; partition .install_state_transfer(&repair_config(), 1, Vec::new(), &wire.encode()) @@ -30721,6 +30896,7 @@ external_groups: Vec::new(), dedup: Vec::new(), retirement_paid_through: 0, + producers: Vec::new(), }; let installed_commit_operation = 12; partition @@ -30907,6 +31083,7 @@ external_groups: Vec::new(), dedup: Vec::new(), retirement_paid_through: 0, + producers: Vec::new(), }; let offered_commit_operation = 12; let install_error = partition @@ -36397,6 +36574,7 @@ external_groups: Vec::new(), dedup: Vec::new(), retirement_paid_through: 0, + producers: Vec::new(), }; let refused = partition .install_state_transfer(&repair_config(), 12, Vec::new(), &behind.encode()) @@ -36453,6 +36631,7 @@ external_groups: Vec::new(), dedup: Vec::new(), retirement_paid_through: 0, + producers: Vec::new(), }; let refused = partition .install_state_transfer(&repair_config(), 12, Vec::new(), &stale.encode()) @@ -36533,6 +36712,39 @@ assert_eq!(receiver.created_revision(), incarnation); } + /// The offer carries the committed producer table, so a replica that + /// catches up by transfer answers a resent batch as its donor does. + #[compio::test] + async fn given_a_producer_table_when_a_replica_transfers_should_hold_it() { + let (_origin_directory, mut origin, _) = message_less_origin().await; + origin.producers.record(&StampedBatch { + producer: BatchProducer::new(7, 2, 0).unwrap(), + message_count: 3, + base_offset: 0, + timestamp: 1, + }); + + let offer = origin.state_transfer_offer(&repair_config()).await.unwrap(); + let (_receiver_directory, mut receiver) = transfer_receiver(); + receiver + .install_state_transfer( + &repair_config(), + offer.commit_op, + Vec::new(), + &offer.offsets.1, + ) + .await + .unwrap(); + + assert_eq!(receiver.producer_entries(), origin.producer_entries()); + assert_eq!( + receiver + .producers + .admit(BatchProducer::new(7, 2, 0).unwrap(), 3, &[]), + ProducerAdmission::Duplicate { base_offset: 0 } + ); + } + /// A replica that never took a message and lags past every op its peers /// retain cannot repair, so it catches up from the primary's offer. #[compio::test] @@ -36765,6 +36977,8 @@ base_offset, message_count, size_bytes: 128, + producer: None, + timestamp: 0, } } @@ -37671,6 +37885,7 @@ history_transition: None, owner_retirement: None, retirement_paid_through: 0, + producers: Vec::new(), }; partition .install_state_transfer(&repair_config(), INSTALLED_OPS, staged, &offsets.encode())
diff --git a/core/partitions/src/lib.rs b/core/partitions/src/lib.rs index 601a6ee..0d0066c 100644 --- a/core/partitions/src/lib.rs +++ b/core/partitions/src/lib.rs
@@ -33,6 +33,7 @@ mod partition_storage; mod persistence; mod poll_plan; +pub mod producer_state; mod repair_memory; pub use persistence::FileSyncBarrier; pub(crate) use persistence::PersistenceDrain; @@ -159,6 +160,7 @@ pub(crate) history_transition: Option<iggy_partition::InstalledPartitionHistory>, pub(crate) owner_retirement: Option<iggy_binary_protocol::requests::partitions::RetireConsumerGroupOwnersRequest>, + pub(crate) producers: producer_state::ProducerTable, } /// Partition-level data plane operations.
diff --git a/core/partitions/src/producer_state.rs b/core/partitions/src/producer_state.rs new file mode 100644 index 0000000..d7baf5f --- /dev/null +++ b/core/partitions/src/producer_state.rs
@@ -0,0 +1,535 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Idempotent producer state of one partition: the epoch of each producer and +//! its last batches. +//! +//! Offsets are assigned when the primary stamps a prepare, so a batch that a +//! producer sends again must be caught before that. The primary checks it +//! against the committed table and the uncommitted prepares in its journal. +//! Every replica records a committed batch in op order, so the tables agree. + +use iggy_binary_protocol::batch::BatchProducer; +use std::collections::{BTreeMap, BTreeSet}; + +/// Batches each producer keeps. +/// +/// Five is Kafka's `max.in.flight.requests.per.connection`, so a resend of any +/// in-flight batch of a Kafka producer still finds the offset of its first +/// write. Checkpoints and transfers carry the batch count of each producer, so +/// this can change without a new format version. +pub const PRODUCER_BATCHES_RETAINED: usize = 5; + +/// Producers a partition keeps, which bounds its memory. Past this, the one +/// idle the longest goes, and its next batch starts at sequence 0, as the +/// first batch of a new producer does. +pub const PRODUCERS_MAX: usize = 4096; + +/// A producer idle this long goes. A day lets a producer resend after a +/// restart or a long outage. Measured in prepare timestamps, so every replica +/// drops the same producers at the same op. +pub const PRODUCER_IDLE_EXPIRY_MICROS: u64 = 24 * 60 * 60 * 1_000_000; + +/// One batch of a producer, as the partition wrote it. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ProducerBatch { + pub first_sequence: u32, + pub last_sequence: u32, + pub base_offset: u64, +} + +/// The state of one producer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProducerEntry { + pub epoch: u16, + /// Oldest first. Slots from `len` on stay default, so equality holds. + batches: [ProducerBatch; PRODUCER_BATCHES_RETAINED], + len: usize, + /// Prepare timestamp of the newest batch. + pub last_timestamp: u64, +} + +/// What the primary does with a batch of a producer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProducerAdmission { + /// Write the batch. + Append, + /// The partition already holds the batch at `base_offset`. + Duplicate { base_offset: u64 }, + /// An uncommitted prepare holds the batch. The sender retries, and gets + /// [`Self::Duplicate`] once it commits. + DuplicateInFlight, + /// The batch does not follow the last batch of its producer. + OutOfOrder, + /// The partition holds no state for the producer, and the batch does not + /// start at sequence 0. + Unknown, + /// The producer moved to a newer epoch. + StaleEpoch, +} + +/// A stamped batch of an uncommitted prepare. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StampedBatch { + pub producer: BatchProducer, + pub message_count: u32, + pub base_offset: u64, + pub timestamp: u64, +} + +impl ProducerEntry { + /// An entry that a checkpoint or a transfer carried, oldest batch first. + /// `None` for more than [`PRODUCER_BATCHES_RETAINED`] batches. + #[must_use] + pub fn restored(epoch: u16, last_timestamp: u64, batches: &[ProducerBatch]) -> Option<Self> { + let mut entry = Self { + epoch, + batches: [ProducerBatch::default(); PRODUCER_BATCHES_RETAINED], + len: batches.len(), + last_timestamp, + }; + entry + .batches + .get_mut(..batches.len())? + .copy_from_slice(batches); + Some(entry) + } + + fn new(batch: &StampedBatch) -> Self { + let mut entry = Self { + epoch: batch.producer.epoch(), + batches: [ProducerBatch::default(); PRODUCER_BATCHES_RETAINED], + len: 0, + last_timestamp: batch.timestamp, + }; + entry.push(batch); + entry + } + + /// Oldest first, at most [`PRODUCER_BATCHES_RETAINED`]. + #[must_use] + pub fn batches(&self) -> &[ProducerBatch] { + &self.batches[..self.len] + } + + /// The rules in order: the epoch, a duplicate of a retained batch, then + /// the sequence. + #[must_use] + pub fn check(&self, producer: BatchProducer, message_count: u32) -> ProducerAdmission { + if producer.epoch() < self.epoch { + return ProducerAdmission::StaleEpoch; + } + if producer.epoch() > self.epoch { + // A new epoch starts its sequence again. + return if producer.base_sequence() == 0 { + ProducerAdmission::Append + } else { + ProducerAdmission::OutOfOrder + }; + } + let last_sequence = producer.last_sequence(message_count); + if let Some(batch) = self.batches().iter().find(|batch| { + batch.first_sequence == producer.base_sequence() && batch.last_sequence == last_sequence + }) { + return ProducerAdmission::Duplicate { + base_offset: batch.base_offset, + }; + } + let follows = self.batches().last().is_none_or(|newest| { + BatchProducer::advance(newest.last_sequence, 1) == producer.base_sequence() + }); + if follows { + ProducerAdmission::Append + } else { + ProducerAdmission::OutOfOrder + } + } + + fn record(&mut self, batch: &StampedBatch) { + if batch.producer.epoch() < self.epoch { + return; + } + if batch.producer.epoch() > self.epoch { + self.epoch = batch.producer.epoch(); + self.batches = [ProducerBatch::default(); PRODUCER_BATCHES_RETAINED]; + self.len = 0; + } + self.push(batch); + self.last_timestamp = self.last_timestamp.max(batch.timestamp); + } + + fn push(&mut self, batch: &StampedBatch) { + let written = ProducerBatch { + first_sequence: batch.producer.base_sequence(), + last_sequence: batch.producer.last_sequence(batch.message_count), + base_offset: batch.base_offset, + }; + if self.len == PRODUCER_BATCHES_RETAINED { + self.batches.copy_within(1.., 0); + self.batches[PRODUCER_BATCHES_RETAINED - 1] = written; + } else { + self.batches[self.len] = written; + self.len += 1; + } + } +} + +/// The committed producer state of a partition, keyed by producer id. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ProducerTable { + entries: BTreeMap<u64, ProducerEntry>, + /// `(last_timestamp, producer_id)` of every entry, the longest idle first. + by_idle: BTreeSet<(u64, u64)>, +} + +impl ProducerTable { + /// A table of entries that a checkpoint or a transfer carried. + #[must_use] + pub fn from_entries(entries: impl IntoIterator<Item = (u64, ProducerEntry)>) -> Self { + let entries: BTreeMap<u64, ProducerEntry> = entries.into_iter().collect(); + let by_idle = entries + .iter() + .map(|(producer_id, entry)| (entry.last_timestamp, *producer_id)) + .collect(); + Self { entries, by_idle } + } + + /// Every entry, ascending by producer id. + pub fn entries(&self) -> impl Iterator<Item = (u64, &ProducerEntry)> { + self.entries + .iter() + .map(|(producer_id, entry)| (*producer_id, entry)) + } + + /// Record a committed batch. Every replica calls this in op order, so + /// every replica also drops the same idle producers. + pub fn record(&mut self, batch: &StampedBatch) { + let producer_id = batch.producer.producer_id(); + if let Some(entry) = self.entries.get_mut(&producer_id) { + self.by_idle.remove(&(entry.last_timestamp, producer_id)); + entry.record(batch); + self.by_idle.insert((entry.last_timestamp, producer_id)); + } else { + let entry = ProducerEntry::new(batch); + self.by_idle.insert((entry.last_timestamp, producer_id)); + self.entries.insert(producer_id, entry); + } + self.expire(batch.timestamp); + } + + fn expire(&mut self, now: u64) { + while let Some(&(last_timestamp, producer_id)) = self.by_idle.first() { + let idle = last_timestamp.saturating_add(PRODUCER_IDLE_EXPIRY_MICROS) < now; + if !idle && self.entries.len() <= PRODUCERS_MAX { + break; + } + self.by_idle.pop_first(); + self.entries.remove(&producer_id); + } + } + + /// Check a batch against the committed state and then `in_flight`, the + /// uncommitted batches of the same producer in op order. + #[must_use] + pub fn admit( + &self, + producer: BatchProducer, + message_count: u32, + in_flight: &[StampedBatch], + ) -> ProducerAdmission { + let mut entry = self.entries.get(&producer.producer_id()).copied(); + for batch in in_flight { + match &mut entry { + Some(entry) => entry.record(batch), + None => entry = Some(ProducerEntry::new(batch)), + } + } + // A producer starts at sequence 0. A later batch that overtook the + // first one through another session would otherwise land ahead of it. + let Some(entry) = entry else { + return if producer.base_sequence() == 0 { + ProducerAdmission::Append + } else { + ProducerAdmission::Unknown + }; + }; + match entry.check(producer, message_count) { + ProducerAdmission::Duplicate { base_offset } + if in_flight + .iter() + .any(|batch| batch.base_offset == base_offset) => + { + ProducerAdmission::DuplicateInFlight + } + admission => admission, + } + } + + #[cfg(test)] + fn get(&self, producer_id: u64) -> Option<&ProducerEntry> { + self.entries.get(&producer_id) + } + + #[cfg(test)] + fn len(&self) -> usize { + self.entries.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const PRODUCER_ID: u64 = 42; + + fn producer(epoch: u16, base_sequence: u32) -> BatchProducer { + BatchProducer::new(PRODUCER_ID, epoch, base_sequence).unwrap() + } + + fn stamped(epoch: u16, base_sequence: u32, count: u32, base_offset: u64) -> StampedBatch { + StampedBatch { + producer: producer(epoch, base_sequence), + message_count: count, + base_offset, + timestamp: base_offset, + } + } + + fn table(batches: &[StampedBatch]) -> ProducerTable { + let mut table = ProducerTable::default(); + for batch in batches { + table.record(batch); + } + table + } + + #[test] + fn given_an_unknown_producer_when_admitting_should_append_only_from_sequence_zero() { + let table = ProducerTable::default(); + assert_eq!( + table.admit(producer(0, 0), 3, &[]), + ProducerAdmission::Append + ); + assert_eq!( + table.admit(producer(2, 0), 3, &[]), + ProducerAdmission::Append + ); + assert_eq!( + table.admit(producer(0, 17), 3, &[]), + ProducerAdmission::Unknown + ); + } + + #[test] + fn given_a_first_batch_in_flight_when_its_successor_arrives_should_append_it() { + let table = ProducerTable::default(); + let in_flight = [stamped(0, 0, 3, 100)]; + + assert_eq!( + table.admit(producer(0, 3), 2, &in_flight), + ProducerAdmission::Append + ); + assert_eq!( + table.admit(producer(0, 3), 2, &[]), + ProducerAdmission::Unknown + ); + } + + #[test] + fn given_a_committed_batch_when_sent_again_should_answer_its_offset() { + let table = table(&[stamped(0, 0, 3, 100), stamped(0, 3, 2, 103)]); + + assert_eq!( + table.admit(producer(0, 0), 3, &[]), + ProducerAdmission::Duplicate { base_offset: 100 } + ); + assert_eq!( + table.admit(producer(0, 3), 2, &[]), + ProducerAdmission::Duplicate { base_offset: 103 } + ); + assert_eq!( + table.admit(producer(0, 5), 1, &[]), + ProducerAdmission::Append + ); + } + + #[test] + fn given_a_gap_or_a_changed_split_when_admitting_should_refuse_it() { + let table = table(&[stamped(0, 0, 3, 100)]); + + assert_eq!( + table.admit(producer(0, 4), 1, &[]), + ProducerAdmission::OutOfOrder + ); + assert_eq!( + table.admit(producer(0, 0), 2, &[]), + ProducerAdmission::OutOfOrder + ); + } + + #[test] + fn given_epochs_when_admitting_should_fence_the_old_and_restart_the_new_at_zero() { + let table = table(&[stamped(2, 0, 3, 100)]); + + assert_eq!( + table.admit(producer(1, 3), 1, &[]), + ProducerAdmission::StaleEpoch + ); + assert_eq!( + table.admit(producer(3, 0), 1, &[]), + ProducerAdmission::Append + ); + assert_eq!( + table.admit(producer(3, 3), 1, &[]), + ProducerAdmission::OutOfOrder + ); + } + + #[test] + fn given_a_new_epoch_when_recorded_should_forget_the_old_batches() { + let table = table(&[stamped(0, 0, 3, 100), stamped(1, 0, 2, 103)]); + + let entry = table.get(PRODUCER_ID).unwrap(); + assert_eq!(entry.epoch, 1); + assert_eq!(entry.batches().len(), 1); + assert_eq!( + table.admit(producer(1, 2), 1, &[]), + ProducerAdmission::Append + ); + } + + #[test] + fn given_in_flight_batches_when_admitting_should_follow_them_and_hold_their_duplicates() { + let table = table(&[stamped(0, 0, 3, 100)]); + let in_flight = [stamped(0, 3, 2, 103)]; + + assert_eq!( + table.admit(producer(0, 5), 1, &in_flight), + ProducerAdmission::Append + ); + assert_eq!( + table.admit(producer(0, 3), 2, &in_flight), + ProducerAdmission::DuplicateInFlight + ); + assert_eq!( + table.admit(producer(0, 0), 3, &in_flight), + ProducerAdmission::Duplicate { base_offset: 100 } + ); + assert_eq!( + table.admit(producer(0, 3), 1, &in_flight), + ProducerAdmission::OutOfOrder + ); + } + + #[test] + fn given_more_than_five_batches_when_recorded_should_keep_the_newest_five() { + let batches: Vec<StampedBatch> = (0..7u32) + .map(|index| stamped(0, index, 1, 100 + u64::from(index))) + .collect(); + let table = table(&batches); + + assert_eq!( + table.get(PRODUCER_ID).unwrap().batches().len(), + PRODUCER_BATCHES_RETAINED + ); + assert_eq!( + table.admit(producer(0, 1), 1, &[]), + ProducerAdmission::OutOfOrder + ); + assert_eq!( + table.admit(producer(0, 2), 1, &[]), + ProducerAdmission::Duplicate { base_offset: 102 } + ); + } + + #[test] + fn given_a_producer_idle_past_the_expiry_when_another_records_should_drop_it() { + let mut table = table(&[stamped(0, 0, 1, 100)]); + let other = StampedBatch { + producer: BatchProducer::new(PRODUCER_ID + 1, 0, 0).unwrap(), + message_count: 1, + base_offset: 101, + timestamp: 100 + PRODUCER_IDLE_EXPIRY_MICROS, + }; + table.record(&other); + assert!(table.get(PRODUCER_ID).is_some(), "idle exactly the expiry"); + + table.record(&StampedBatch { + timestamp: other.timestamp + 1, + ..other + }); + + assert!(table.get(PRODUCER_ID).is_none()); + assert_eq!(table.len(), 1); + assert_eq!( + table.admit(producer(0, 9), 1, &[]), + ProducerAdmission::Unknown, + "a dropped producer starts again at sequence 0" + ); + assert_eq!( + table.admit(producer(0, 0), 1, &[]), + ProducerAdmission::Append + ); + } + + #[test] + fn given_a_full_table_when_a_new_producer_records_should_drop_the_longest_idle() { + let mut table = ProducerTable::default(); + for producer_id in 0..=PRODUCERS_MAX as u64 { + table.record(&StampedBatch { + producer: BatchProducer::new(producer_id, 0, 0).unwrap(), + message_count: 1, + base_offset: producer_id, + timestamp: producer_id, + }); + } + + assert_eq!(table.len(), PRODUCERS_MAX); + assert!(table.get(0).is_none(), "the longest idle producer goes"); + assert!(table.get(1).is_some()); + } + + #[test] + fn given_entries_when_rebuilt_should_admit_and_expire_as_the_original() { + let original = table(&[stamped(0, 0, 3, 100), stamped(0, 3, 1, 103)]); + let rebuilt = ProducerTable::from_entries( + original + .entries() + .map(|(producer_id, entry)| (producer_id, *entry)), + ); + + assert_eq!(rebuilt, original); + assert_eq!( + rebuilt.admit(producer(0, 3), 1, &[]), + ProducerAdmission::Duplicate { base_offset: 103 } + ); + } + + #[test] + fn given_a_sequence_at_the_wrap_when_admitting_should_follow_with_zero() { + let table = table(&[stamped(0, BatchProducer::SEQUENCE_MAX - 1, 2, 100)]); + + assert_eq!( + table.admit(producer(0, 0), 1, &[]), + ProducerAdmission::Append + ); + assert_eq!( + table.admit(producer(0, 1), 1, &[]), + ProducerAdmission::OutOfOrder + ); + } +}
diff --git a/core/partitions/src/state_transfer.rs b/core/partitions/src/state_transfer.rs index 777baf8..9ba433c 100644 --- a/core/partitions/src/state_transfer.rs +++ b/core/partitions/src/state_transfer.rs
@@ -33,6 +33,9 @@ use crate::journal::RepairTailLease; use crate::offset_storage::{discard_offset_replacement, stage_offset_replacement}; use crate::partition_storage::{read_revision_record, write_revision_record}; +use crate::producer_state::{ + PRODUCER_BATCHES_RETAINED, PRODUCERS_MAX, ProducerBatch, ProducerEntry, +}; use crate::segment_anchor::ANCHOR_SUFFIX; use crate::types::{OFFSETS_DIR, PartitionsConfig}; use compio::io::{AsyncReadAtExt, AsyncWriteAtExt}; @@ -42,6 +45,7 @@ use consensus::{ ArtifactProgress, DedupWatermark, Sequencer as _, StateArtifactHasher, state_artifact_checksum, }; +use iggy_binary_protocol::batch::BatchProducer; use iggy_binary_protocol::requests::partitions::retire_consumer_group_owners::CONSUMER_GROUP_OWNERS_MAX; use iggy_binary_protocol::requests::partitions::{ InstallConsumerGroupOwnerRequest, RetireConsumerGroupOwnersRequest, @@ -78,7 +82,11 @@ /// Version 6 dropped the purge generation that followed the version byte. /// Version 7 carries the installed consumer-group owners and history fence. /// Version 8 carries the op of the delete that emptied the offset table. -pub(crate) const CONSUMER_OFFSETS_VERSION: u8 = 8; +/// Version 9 carries the producer table of idempotent producers. +pub(crate) const CONSUMER_OFFSETS_VERSION: u8 = 9; +/// The oldest version that [`ConsumerOffsetsWire::decode`] reads. Version 8 +/// builds kept no producers, so its artifact reads as an empty producer table. +const CONSUMER_OFFSETS_VERSION_MIN: u8 = 8; pub(crate) const SEGMENT_SWEEP_STACK_BYTES: usize = 2 * 1024 * 1024; const RETRY_CHECKPOINT_MAGIC: [u8; 4] = *b"IRP2"; @@ -93,7 +101,9 @@ + CONSUMER_OFFSETS_ENTRIES_MAX as usize * OWNERSHIP_ENTRY_LEN + RetireConsumerGroupOwnersRequest::PREFIX_SIZE + CONSUMER_GROUP_OWNERS_MAX as usize * size_of::<u64>() - + size_of::<PrepareHeader>(); + + size_of::<PrepareHeader>() + + size_of::<u32>() + + PRODUCERS_MAX * PRODUCER_ENTRY_LEN_MAX; /// Per-section entry ceiling for the consumer-offsets artifact. /// @@ -108,6 +118,14 @@ InstallConsumerGroupOwnerRequest::ENCODED_SIZE + size_of::<u64>(); const PARTITION_RECEIPT_BYTES_MAX: usize = size_of::<ReplyHeader>() + size_of::<u32>() + CONFIRMATION_SIZE; +/// `producer_id u64 | epoch u16 | last_timestamp u64 | batch_count u8`. +const PRODUCER_ENTRY_PREFIX_LEN: usize = + size_of::<u64>() + size_of::<u16>() + size_of::<u64>() + size_of::<u8>(); +/// `first_sequence u32 | last_sequence u32 | base_offset u64`. +const PRODUCER_BATCH_LEN: usize = 2 * size_of::<u32>() + size_of::<u64>(); +/// An entry holds up to `u8::MAX` batches, whatever window its writer kept. +const PRODUCER_ENTRY_LEN_MAX: usize = + PRODUCER_ENTRY_PREFIX_LEN + u8::MAX as usize * PRODUCER_BATCH_LEN; /// One in-flight partition state transfer on the receiving replica. /// @@ -576,9 +594,11 @@ pub history_transition: Option<InstalledPartitionHistory>, pub owner_retirement: Option<iggy_binary_protocol::requests::partitions::RetireConsumerGroupOwnersRequest>, - /// Trailing and optional on decode, so artifacts written before + /// Optional on decode of version 8, so artifacts written before /// retirement debt read as `0`. pub retirement_paid_through: u64, + /// The committed producer table, ascending by producer id. + pub producers: Vec<(u64, ProducerEntry)>, } impl ConsumerOffsetsWire { @@ -592,8 +612,10 @@ /// history_present u8 | [history fence, partition_op u64] | /// retirement_length u32 | retirement bytes | checksum_present u8 | /// prepare_checksum u128 | prepare_length u32 | checkpoint_prepare bytes | - /// retirement_paid_through u64 | XxHash3_64 trailer`. Little-endian - /// throughout. + /// retirement_paid_through u64 | producer_count u32 | {producer_id u64, + /// epoch u16, last_timestamp u64, batch_count u8, {first_sequence u32, + /// last_sequence u32, base_offset u64}xB}xP | XxHash3_64 trailer`. + /// Little-endian throughout. #[must_use] pub fn encode(&self) -> Vec<u8> { let sections = self.offset_sections(); @@ -671,6 +693,23 @@ ); out.extend_from_slice(&self.checkpoint_prepare); out.extend_from_slice(&self.retirement_paid_through.to_le_bytes()); + out.extend_from_slice( + &u32::try_from(self.producers.len()) + .expect("bounded producer count") + .to_le_bytes(), + ); + for (producer_id, entry) in &self.producers { + out.extend_from_slice(&producer_id.to_le_bytes()); + out.extend_from_slice(&entry.epoch.to_le_bytes()); + out.extend_from_slice(&entry.last_timestamp.to_le_bytes()); + #[allow(clippy::cast_possible_truncation)] + out.push(entry.batches().len() as u8); + for batch in entry.batches() { + out.extend_from_slice(&batch.first_sequence.to_le_bytes()); + out.extend_from_slice(&batch.last_sequence.to_le_bytes()); + out.extend_from_slice(&batch.base_offset.to_le_bytes()); + } + } debug_assert_eq!(out.len() + size_of::<u64>(), reserved, "encode reservation"); let trailer = state_artifact_checksum(&out); out.extend_from_slice(&trailer.to_le_bytes()); @@ -703,7 +742,7 @@ if magic != CONSUMER_OFFSETS_MAGIC { return Err(ConsumerOffsetsWireError::BadMagic); } - if version != CONSUMER_OFFSETS_VERSION { + if !(CONSUMER_OFFSETS_VERSION_MIN..=CONSUMER_OFFSETS_VERSION).contains(&version) { return Err(ConsumerOffsetsWireError::UnsupportedVersion { version }); } let next_offset = cursor.u64()?; @@ -762,11 +801,19 @@ return Err(ConsumerOffsetsWireError::InvalidPrepareChecksum); } let checkpoint_prepare = cursor.take(prepare_length)?.to_vec(); - let retirement_paid_through = if cursor.remaining().len() >= size_of::<u64>() { + // Version 8 builds from before retirement debt end at the prepare. + let retirement_paid_through = if version > CONSUMER_OFFSETS_VERSION_MIN + || cursor.remaining().len() >= size_of::<u64>() + { cursor.u64()? } else { 0 }; + let producers = if version == CONSUMER_OFFSETS_VERSION_MIN { + Vec::new() + } else { + Self::decode_producer_section(&mut cursor)? + }; if !cursor.remaining().is_empty() { // Distinct from `Truncated`: extra bytes point at a NEWER // encoder, and telling the operator the artifact is short would @@ -790,9 +837,73 @@ history_transition, owner_retirement, retirement_paid_through, + producers, }) } + /// The producer section, ascending by producer id. Every field stays + /// within its maximum, and each producer has at least one batch. Only the + /// newest [`PRODUCER_BATCHES_RETAINED`] batches are kept, so a writer with + /// another window needs no new version. + fn decode_producer_section( + cursor: &mut LeCursor<'_>, + ) -> Result<Vec<(u64, ProducerEntry)>, ConsumerOffsetsWireError> { + let count = cursor.u32()?; + if count as usize > PRODUCERS_MAX { + return Err(ConsumerOffsetsWireError::TooManyEntries { + section: "producers", + count, + max: u32::try_from(PRODUCERS_MAX).unwrap_or(u32::MAX), + }); + } + if count as usize * PRODUCER_ENTRY_PREFIX_LEN > cursor.remaining().len() { + return Err(ConsumerOffsetsWireError::Truncated); + } + let mut producers = Vec::with_capacity(count as usize); + let mut previous = None; + for _ in 0..count { + let producer_id = cursor.u64()?; + let epoch = u16::from_le_bytes( + cursor + .take(size_of::<u16>())? + .try_into() + .map_err(|_| ConsumerOffsetsWireError::Truncated)?, + ); + let last_timestamp = cursor.u64()?; + let batch_count = usize::from(cursor.u8()?); + if producer_id > BatchProducer::PRODUCER_ID_MAX + || epoch > BatchProducer::EPOCH_MAX + || batch_count == 0 + || previous.is_some_and(|previous| previous >= producer_id) + { + return Err(ConsumerOffsetsWireError::InvalidProducer); + } + previous = Some(producer_id); + let dropped = batch_count.saturating_sub(PRODUCER_BATCHES_RETAINED); + let mut batches = [ProducerBatch::default(); PRODUCER_BATCHES_RETAINED]; + for index in 0..batch_count { + let batch = ProducerBatch { + first_sequence: cursor.u32()?, + last_sequence: cursor.u32()?, + base_offset: cursor.u64()?, + }; + if batch.first_sequence > BatchProducer::SEQUENCE_MAX + || batch.last_sequence > BatchProducer::SEQUENCE_MAX + { + return Err(ConsumerOffsetsWireError::InvalidProducer); + } + if let Some(slot) = index.checked_sub(dropped) { + batches[slot] = batch; + } + } + let entry = + ProducerEntry::restored(epoch, last_timestamp, &batches[..batch_count - dropped]) + .ok_or(ConsumerOffsetsWireError::InvalidProducer)?; + producers.push((producer_id, entry)); + } + Ok(producers) + } + /// The offset section of each kind, in [`ConsumerKind::ALL`] order. pub(crate) fn offset_sections(&self) -> [&[(u32, u64)]; ConsumerKind::COUNT] { [&self.consumers, &self.groups, &self.external_groups] @@ -830,6 +941,14 @@ .owner_retirement .as_ref() .map_or(0, WireEncode::encoded_size) + + size_of::<u32>() + + self + .producers + .iter() + .map(|(_, entry)| { + PRODUCER_ENTRY_PREFIX_LEN + entry.batches().len() * PRODUCER_BATCH_LEN + }) + .sum::<usize>() + size_of::<u64>() + size_of::<u64>() } @@ -1063,6 +1182,9 @@ /// A dedup entry carries client id zero, which is reserved and refused at /// every ingress: a peer encoder bug. ReservedClient, + /// A producer entry is out of order, above a field maximum, or keeps no + /// batch or too many. + InvalidProducer, } impl From<Truncated> for ConsumerOffsetsWireError { @@ -1100,7 +1222,7 @@ Self::UnsupportedVersion { version } => write!( f, "consumer-offsets artifact version {version} is not understood \ - (this build speaks {CONSUMER_OFFSETS_VERSION})" + (this build reads {CONSUMER_OFFSETS_VERSION_MIN} to {CONSUMER_OFFSETS_VERSION})" ), Self::ChecksumMismatch { expected, actual } => write!( f, @@ -1130,6 +1252,9 @@ "consumer-offsets artifact dedup entry carries reserved client 0" ) } + Self::InvalidProducer => { + write!(f, "invalid producer entry in consumer-offsets artifact") + } } } } @@ -1265,9 +1390,118 @@ dedup_entry(usize::MAX as u128 + 5, 9, 91), ], retirement_paid_through: 0, + producers: Vec::new(), } } + /// A producer entry with `(first_sequence, last_sequence, base_offset)` + /// batches, oldest first. + fn producer_entry(producer_id: u64, batches: &[(u32, u32, u64)]) -> (u64, ProducerEntry) { + let batches: Vec<ProducerBatch> = batches + .iter() + .map( + |&(first_sequence, last_sequence, base_offset)| ProducerBatch { + first_sequence, + last_sequence, + base_offset, + }, + ) + .collect(); + ( + producer_id, + ProducerEntry::restored(3, 1_000, &batches).expect("at most the retained batches"), + ) + } + + #[test] + fn given_producers_when_encoded_should_round_trip() { + let wire = ConsumerOffsetsWire { + producers: vec![ + producer_entry(4, &[(0, 2, 10)]), + producer_entry(9, &[(0, 0, 11), (1, 4, 12)]), + ], + ..table() + }; + + assert_eq!(ConsumerOffsetsWire::decode(&wire.encode()), Ok(wire)); + } + + #[test] + fn given_a_malformed_producer_entry_when_decoded_should_reject_it() { + let one = [(0, 0, 1)]; + let mut epoch_over_max = producer_entry(1, &one); + epoch_over_max.1.epoch = BatchProducer::EPOCH_MAX + 1; + let cases = [ + vec![producer_entry(2, &one), producer_entry(2, &one)], + vec![producer_entry(3, &one), producer_entry(2, &one)], + vec![producer_entry(1, &[])], + vec![producer_entry(BatchProducer::PRODUCER_ID_MAX + 1, &one)], + vec![epoch_over_max], + vec![producer_entry( + 1, + &[(BatchProducer::SEQUENCE_MAX + 1, 0, 1)], + )], + ]; + for producers in cases { + let wire = ConsumerOffsetsWire { + producers: producers.clone(), + ..table() + }; + assert_eq!( + ConsumerOffsetsWire::decode(&wire.encode()), + Err(ConsumerOffsetsWireError::InvalidProducer), + "{producers:?}" + ); + } + } + + /// A writer with a wider window stores more batches, and the decoder keeps + /// the newest. + #[test] + fn given_more_batches_than_the_window_when_decoded_should_keep_the_newest() { + let newest: Vec<(u32, u32, u64)> = (1..=PRODUCER_BATCHES_RETAINED) + .map(|batch| { + let sequence = u32::try_from(batch).unwrap(); + (sequence, sequence, u64::from(sequence)) + }) + .collect(); + let kept = ConsumerOffsetsWire { + producers: vec![producer_entry(1, &newest)], + ..table() + }; + let encoded = kept.encode(); + // An entry cannot hold one batch more, so the older batch goes in on the wire. + let content_len = encoded.len() - size_of::<u64>(); + let batches_at = content_len - PRODUCER_BATCHES_RETAINED * PRODUCER_BATCH_LEN; + let mut wider = encoded[..batches_at].to_vec(); + wider[batches_at - 1] += 1; + wider.extend_from_slice(&[0; PRODUCER_BATCH_LEN]); + wider.extend_from_slice(&encoded[batches_at..content_len]); + let trailer = state_artifact_checksum(&wider); + wider.extend_from_slice(&trailer.to_le_bytes()); + + assert_eq!(ConsumerOffsetsWire::decode(&wider), Ok(kept)); + } + + /// Version 8 ends at the paid-through op, and its builds kept no + /// producers. + #[test] + fn given_version_8_offsets_artifact_when_decoded_should_read_no_producers() { + const RELEASED_VERSION: u8 = 8; + let paid = ConsumerOffsetsWire { + retirement_paid_through: 9, + ..table() + }; + let encoded = paid.encode(); + let content_end = encoded.len() - size_of::<u64>() - size_of::<u32>(); + let mut released = encoded[..content_end].to_vec(); + released[CONSUMER_OFFSETS_MAGIC.len()] = RELEASED_VERSION; + let trailer = state_artifact_checksum(&released); + released.extend_from_slice(&trailer.to_le_bytes()); + + assert_eq!(ConsumerOffsetsWire::decode(&released), Ok(paid)); + } + fn dedup_entry(client: u128, watermark: u64, latest_commit: u64) -> DedupWatermark { let header = ReplyHeader { command: iggy_binary_protocol::Command::Reply, @@ -1449,6 +1683,7 @@ external_groups: Vec::new(), dedup: Vec::new(), retirement_paid_through: 0, + producers: Vec::new(), }; let encoded = empty.encode(); assert_eq!( @@ -1516,7 +1751,10 @@ #[test] fn given_unknown_version_when_decoded_should_reject() { - for version in [CONSUMER_OFFSETS_VERSION - 1, CONSUMER_OFFSETS_VERSION + 1] { + for version in [ + CONSUMER_OFFSETS_VERSION_MIN - 1, + CONSUMER_OFFSETS_VERSION + 1, + ] { let mut wrong = table().encode(); wrong[CONSUMER_OFFSETS_MAGIC.len()] = version; let content_len = wrong.len() - size_of::<u64>(); @@ -1535,10 +1773,9 @@ // forgotten, and a peer on another release then reads the artifact out // of place. One entry per section makes each length cover the header, // every count field and every entry stride. Changing a length is the - // reminder to change its version. The trailing `retirement_paid_through` - // is optional on decode, so it kept version 8 readable without a bump. - const WITHOUT_EXTERNAL_LEN: usize = 563; - const WITH_EXTERNAL_LEN: usize = 575; + // reminder to change its version. + const WITHOUT_EXTERNAL_LEN: usize = 602; + const WITH_EXTERNAL_LEN: usize = 614; const INCARNATION: u64 = 1; let with_external = ConsumerOffsetsWire { @@ -1579,6 +1816,7 @@ live_group_ids: vec![1], }), retirement_paid_through: 0, + producers: vec![producer_entry(1, &[(0, 2, 7)])], }; let without_external = ConsumerOffsetsWire { external_groups: Vec::new(), @@ -1597,7 +1835,7 @@ ); } assert_eq!( - CONSUMER_OFFSETS_VERSION, 8, + CONSUMER_OFFSETS_VERSION, 9, "a consumer-offsets version moved; confirm its layout moved with it" ); } @@ -1688,6 +1926,7 @@ external_groups: Vec::new(), dedup: vec![dedup_entry(9, 1, 1), dedup_entry(4, 2, 2)], retirement_paid_through: 0, + producers: Vec::new(), }; assert_eq!( ConsumerOffsetsWire::decode(&unordered.encode()), @@ -1712,6 +1951,7 @@ external_groups: Vec::new(), dedup: vec![dedup_entry(0, 1, 1), dedup_entry(4, 2, 2)], retirement_paid_through: 0, + producers: Vec::new(), }; assert_eq!( ConsumerOffsetsWire::decode(&reserved.encode()), @@ -1850,6 +2090,7 @@ external_groups: Vec::new(), dedup: Vec::new(), retirement_paid_through: 0, + producers: Vec::new(), }; assert_eq!( ConsumerOffsetsWire::decode(&duplicate.encode()), @@ -1873,6 +2114,7 @@ external_groups: Vec::new(), dedup: Vec::new(), retirement_paid_through: 0, + producers: Vec::new(), }; assert_eq!( ConsumerOffsetsWire::decode(&unordered.encode()), @@ -1906,7 +2148,10 @@ }; let encoded = paid.encode(); assert_eq!(ConsumerOffsetsWire::decode(&encoded), Ok(paid)); - let mut legacy = encoded[..encoded.len() - 2 * size_of::<u64>()].to_vec(); + // Version 8 builds from before retirement debt end at the prepare. + let content_end = encoded.len() - 2 * size_of::<u64>() - size_of::<u32>(); + let mut legacy = encoded[..content_end].to_vec(); + legacy[CONSUMER_OFFSETS_MAGIC.len()] = 8; let trailer = state_artifact_checksum(&legacy); legacy.extend_from_slice(&trailer.to_le_bytes()); assert_eq!(ConsumerOffsetsWire::decode(&legacy), Ok(table())); @@ -3655,6 +3900,7 @@ history_transition: self.history_transition, owner_retirement: self.owner_retirement.clone(), retirement_paid_through: self.retirement_paid_through, + producers: self.producer_entries(), }) } @@ -4917,6 +5163,7 @@ if let Some(retry_table) = install.retry_table.take() { *self.dedup_mut() = retry_table; } + self.restore_producers(std::mem::take(&mut install.offsets_wire.producers)); self.required_metadata_frontier = self .required_metadata_frontier .max(install.offsets_wire.required_metadata_frontier); @@ -6020,6 +6267,7 @@ history_transition: self.history_transition, owner_retirement: self.owner_retirement.clone(), retirement_paid_through: self.retirement_paid_through, + producers: self.producer_entries(), ..Default::default() }; Ok(retry_checkpoint_bytes( @@ -6097,6 +6345,7 @@ let table = checked_retry_table(&wire, through_op, self.dedup().capacity()) .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; *self.dedup_mut() = table; + self.restore_producers(wire.producers); self.history_transition = wire.history_transition; self.owner_retirement = wire.owner_retirement; self.consumer_group_polls.clear();
diff --git a/core/sdk/Cargo.toml b/core/sdk/Cargo.toml index 7932033..62acafe 100644 --- a/core/sdk/Cargo.toml +++ b/core/sdk/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "iggy" -version = "0.11.1-edge.7" +version = "0.11.1-edge.8" description = "Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." edition = "2024" rust-version.workspace = true
diff --git a/core/sdk/README.md b/core/sdk/README.md index d5e2fd0..6b2eadd 100644 --- a/core/sdk/README.md +++ b/core/sdk/README.md
@@ -573,6 +573,58 @@ because the timeout is always `max_wait` plus 10 s. Intentional batching waits count in the measured latency. +## Idempotent producers + +An idempotent producer writes each batch once. If it sends a batch again, the partition does not write the batch a second time. This holds after a lost reply, a reconnect or a lost session. If a restarted client stored its producer state, it holds after the restart too. + +For exactly-once delivery, give the topic `persisted` durability. With `replicated`, a quorum of nodes that crash at the same time can lose acknowledged batches. + +The client puts the producer id, the epoch and the sequence of the first message in the batch header. The epoch is a counter that fences an older copy of the same producer. `BatchProducer::new` returns `None` for an id above `i64::MAX`, an epoch above `i16::MAX` or a sequence above `i32::MAX`. These are Kafka's ranges, so the Kafka gateway passes the epoch and the sequence of a Kafka batch through unchanged. A sequence wraps to 0 after `i32::MAX`, and `BatchProducer::next` does the wrap. + +`next_producer_id` returns a producer id that no other client of the cluster gets. It sends no request, because the id holds the session number of the client and a local index. One session hands out 2^20 ids. After that, the call fails with `ProducerIdsExhausted` (91), and a new sign-in starts a new range. An id stays unique after its session ends, so a producer can store its id and use it again after a restart. + +`send_producer_messages` sends one batch with its producer to the partition that you name, because each partition counts the sequence of a producer separately. The partition keeps the last five batches of each producer and answers as follows: + +- If the partition holds the batch, it does not write the batch again. The reply holds the offset of the first write. +- If an uncommitted write holds the batch, the answer is `TransientNotCommitted` (57). The outcome stays unknown, so the client sends the batch again and gets the offset of the first write. +- If the batch does not follow the last batch of its producer, the answer is `ProducerSequenceOutOfOrder` (92). +- If the epoch is older than the newest epoch of the producer, the answer is `ProducerEpochFenced` (93). +- If the partition does not hold the producer and the batch does not start at sequence 0, the answer is `ProducerUnknown` (94). The producer starts the next epoch at sequence 0. If an earlier send of the batch had an unknown outcome, that send can have landed, and a resend at the new epoch writes the batch twice. + +A new epoch starts at sequence 0. To fence an older copy of itself, for example after a restart, a producer moves to the next epoch. A restarted producer can resend its last batch. It sends that batch first, with the old epoch and the old sequence. After that, it moves to the next epoch. + +```rust +use iggy::prelude::*; + +async fn produce( + client: &IggyClient, + stream: &Identifier, + topic: &Identifier, + batches: &mut [Vec<IggyMessage>], +) -> Result<(), IggyError> { + let producer_id = client.next_producer_id().await?; + let mut producer = BatchProducer::new(producer_id, 0, 0).ok_or(IggyError::InvalidCommand)?; + for batch in batches.iter_mut() { + // A retry of this batch sends the same producer and sequence. + client + .send_producer_messages(stream, topic, 0, producer, batch) + .await?; + let count = u32::try_from(batch.len()).map_err(|_| IggyError::InvalidCommand)?; + producer = producer.next(count); + } + Ok(()) +} +``` + +`IggyProducer` has an idempotent mode, which `idempotent()` on its builder turns on. The producer then gets an id from `next_producer_id` for each stream and topic as you spell them, picks the partition of each chunk as a plain send does and numbers the chunks of each partition. +A topic named by number and by name gets two ids, so the two never share a sequence. After an unknown outcome, it sends a chunk again with the same producer section. If that resend gets `ProducerUnknown`, the send fails with `OperationUncertain`. The mode is off by default, because released consumers cannot poll the batches that it writes. + +A partition keeps 4096 producers. Past that limit, it drops the producer that was idle the longest. It also drops a producer after 24 hours without a batch. The producer state stays through restarts, checkpoints and state transfer between replicas. HTTP clients get `FeatureUnavailable` from `next_producer_id` and `send_producer_messages`. A client with an encryptor encrypts the messages of `send_producer_messages` in place, so pass the plaintext messages again to resend a batch. + +Released builds of the Rust SDK refuse a binary poll that returns a batch with a producer. They require zero bytes in that part of the header. The same holds for every client that embeds the Rust SDK: the Python, C++ and PHP SDKs, the CLI, the Kafka gateway and the connectors runtime. Released Swift SDKs refuse the batch too. The Go, Java, C# and Node.js SDKs do not read that part of the header. Upgrade every consumer of a topic before a producer writes such batches to it. + +Upgrade every server node first as well. A node of an older release cannot store a batch with a producer. + ## Metadata changes `get_metadata_changes` reads the node's change log: one record per public result of a committed metadata operation, such as a stream created, a topic renamed, a user's permissions changed or a session closed. A mirror keeps a cursor, asks for the records after it, and stores the returned `next` cursor. The call needs root or the global `manage_servers` permission.
diff --git a/core/sdk/src/binary/mod.rs b/core/sdk/src/binary/mod.rs index 0c28952..29ef377 100644 --- a/core/sdk/src/binary/mod.rs +++ b/core/sdk/src/binary/mod.rs
@@ -284,6 +284,10 @@ fn sdk_version(&self) -> &'static str { crate::SDK_VERSION } + + fn next_producer_id(&self) -> Result<u64, iggy_common::IggyError> { + self.driver.session().next_producer_id() + } } impl iggy_common::BinaryClient for $client {}
diff --git a/core/sdk/src/client_wrappers/binary_message_client.rs b/core/sdk/src/client_wrappers/binary_message_client.rs index 9a38dd6..350b81f 100644 --- a/core/sdk/src/client_wrappers/binary_message_client.rs +++ b/core/sdk/src/client_wrappers/binary_message_client.rs
@@ -18,8 +18,8 @@ use crate::client_wrappers::client_wrapper::ClientWrapper; use async_trait::async_trait; use iggy_common::{ - Consumer, DecodeError, Identifier, IggyError, IggyMessage, MessageClient, Partitioning, - PollOptions, PolledMessages, PollingStrategy, SendMessagesResponse, + BatchProducer, Consumer, DecodeError, Identifier, IggyError, IggyMessage, MessageClient, + Partitioning, PollOptions, PolledMessages, PollingStrategy, SendMessagesResponse, }; #[async_trait] @@ -83,4 +83,29 @@ ) .await } + + async fn send_producer_messages( + &self, + stream_id: &Identifier, + topic_id: &Identifier, + partition_id: u32, + producer: BatchProducer, + messages: &mut [IggyMessage], + ) -> Result<SendMessagesResponse, IggyError> { + super::dispatch_client!( + self, + MessageClient::send_producer_messages( + stream_id, + topic_id, + partition_id, + producer, + messages + ) + ) + .await + } + + async fn next_producer_id(&self) -> Result<u64, IggyError> { + super::dispatch_client!(self, MessageClient::next_producer_id()).await + } }
diff --git a/core/sdk/src/client_wrappers/client_wrapper.rs b/core/sdk/src/client_wrappers/client_wrapper.rs index 24643b4..6520bbf 100644 --- a/core/sdk/src/client_wrappers/client_wrapper.rs +++ b/core/sdk/src/client_wrappers/client_wrapper.rs
@@ -25,8 +25,9 @@ use iggy_common::{ AssignmentVersion, BinaryClient, BinaryOperationLineage, BinarySendPartitioning, BinaryTransport, ClientState, Consumer, DeferredPollOptions, FetchedSection, Identifier, - IggyError, IggyMessage, MessageClient, PollingStrategy, SendMessagesResponse, SystemClient, - fetch_binary_messages, group_cache_key, send_binary_messages, sync_group_assignment, + IggyError, IggyMessage, MessageClient, Partitioning, PollingStrategy, SendMessagesResponse, + SystemClient, fetch_binary_messages, group_cache_key, select_send_partition, + send_binary_messages, sync_group_assignment, }; #[allow(clippy::large_enum_variant)] @@ -67,6 +68,9 @@ )) .await } + Self::Http(_) if partitioning.producer().is_some() => { + Err(IggyError::FeatureUnavailable) + } Self::Http(client) => { client .send_messages(stream, topic, partitioning.original(), messages) @@ -84,6 +88,38 @@ } } + /// The partition that a send with `partitioning` goes to. See + /// [`select_send_partition`]. + pub(crate) async fn select_send_partition( + &self, + stream: &Identifier, + topic: &Identifier, + partitioning: &Partitioning, + fresh: bool, + ) -> Result<u32, IggyError> { + match self { + Self::Iggy(client) => { + Box::pin(client.client.read().await.select_send_partition( + stream, + topic, + partitioning, + fresh, + )) + .await + } + Self::Http(_) => Err(IggyError::FeatureUnavailable), + Self::Tcp(client) => { + select_send_partition(client, stream, topic, partitioning, fresh).await + } + Self::Quic(client) => { + select_send_partition(client, stream, topic, partitioning, fresh).await + } + Self::WebSocket(client) => { + select_send_partition(client, stream, topic, partitioning, fresh).await + } + } + } + pub(crate) async fn keepalive(&self) -> Result<bool, IggyError> { match self { Self::Iggy(client) => Box::pin(client.client.read().await.keepalive()).await,
diff --git a/core/sdk/src/clients/binary_message.rs b/core/sdk/src/clients/binary_message.rs index d1f7c4e..4c6e5eb 100644 --- a/core/sdk/src/clients/binary_message.rs +++ b/core/sdk/src/clients/binary_message.rs
@@ -19,11 +19,11 @@ use async_trait::async_trait; use bytes::Bytes; use iggy_common::locking::IggyRwLockFn; -use iggy_common::{BinarySendPartitioning, MessageClient, with_binary_deadline}; use iggy_common::{ - CommitPolicy, Consumer, DecodeError, Identifier, IggyError, IggyMessage, Partitioning, - PollOptions, PolledMessages, PollingStrategy, SendMessagesResponse, + BatchProducer, CommitPolicy, Consumer, DecodeError, Identifier, IggyError, IggyMessage, + Partitioning, PollOptions, PolledMessages, PollingStrategy, SendMessagesResponse, }; +use iggy_common::{BinarySendPartitioning, MessageClient, with_binary_deadline}; use tokio::time::Instant; #[async_trait] @@ -127,6 +127,28 @@ ) .await } + + async fn send_producer_messages( + &self, + stream_id: &Identifier, + topic_id: &Identifier, + partition_id: u32, + producer: BatchProducer, + messages: &mut [IggyMessage], + ) -> Result<SendMessagesResponse, IggyError> { + self.send_messages_with_partitioning( + stream_id, + topic_id, + &mut BinarySendPartitioning::new(&Partitioning::partition_id(partition_id)) + .with_producer(producer), + messages, + ) + .await + } + + async fn next_producer_id(&self) -> Result<u64, IggyError> { + super::with_client!(self, client => client.next_producer_id().await) + } } impl IggyClient {
diff --git a/core/sdk/src/clients/mod.rs b/core/sdk/src/clients/mod.rs index 74c72ae..6028111 100644 --- a/core/sdk/src/clients/mod.rs +++ b/core/sdk/src/clients/mod.rs
@@ -53,6 +53,7 @@ pub mod producer_config; pub mod producer_dispatcher; pub mod producer_error_callback; +pub(crate) mod producer_idempotence; pub mod producer_sharding; pub mod reader; pub mod reader_builder;
diff --git a/core/sdk/src/clients/producer.rs b/core/sdk/src/clients/producer.rs index b68bf67..4a3252f 100644 --- a/core/sdk/src/clients/producer.rs +++ b/core/sdk/src/clients/producer.rs
@@ -21,17 +21,18 @@ use crate::clients::producer_builder::SendMode; use crate::clients::producer_config::DirectConfig; use crate::clients::producer_dispatcher::ProducerDispatcher; +use crate::clients::producer_idempotence::{Idempotence, RESENDS, TopicProducer}; use crate::driver::operation_deadline; use bytes::Bytes; use iggy_common::locking::{IggyRwLock, IggyRwLockFn}; use iggy_common::{ - BinarySendPartitioning, Client, StreamClient, StreamCreateOptions, TopicClient, - TopicCreateOptions, + BatchProducer, BinarySendPartitioning, Client, MessageClient, StreamClient, + StreamCreateOptions, TopicClient, TopicCreateOptions, }; use iggy_common::{ Durability, EncryptorKind, IdKind, Identifier, IggyError, IggyExpiry, IggyMessage, - IggyTimestamp, MaxTopicSize, Partitioner, Partitioning, SendMessagesConfirmationResponse, - SendMessagesResponse, with_binary_lineage, + IggyTimestamp, MaxTopicSize, Partitioner, Partitioning, PartitioningKind, + SendMessagesConfirmationResponse, SendMessagesResponse, with_binary_lineage, }; use std::collections::HashSet; use std::num::NonZeroU32; @@ -86,6 +87,7 @@ last_sent_at: Arc<AtomicU64>, uncertain_operations: Mutex<HashSet<u64>>, direct_config: Option<DirectConfig>, + idempotence: Option<Idempotence>, } impl ProducerCore { @@ -160,6 +162,11 @@ .await?; } + if let Some(idempotence) = &self.idempotence { + self.ensure_producer_id(&idempotence.topic_producer(&stream_id, &topic_id)) + .await?; + } + let _ = self .initialized .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst); @@ -169,19 +176,175 @@ .await } - async fn try_send_messages( + /// Sends one chunk, in the idempotent mode when the producer has it. + async fn send_chunk( &self, stream: &Identifier, topic: &Identifier, partitioning: &Arc<Partitioning>, messages: &mut [IggyMessage], ) -> Result<SendMessagesResponse, IggyError> { + match &self.idempotence { + // Boxed, so the idempotent path does not grow the future of every send. + Some(idempotence) => { + Box::pin(self.try_send_idempotent( + idempotence, + stream, + topic, + partitioning, + messages, + )) + .await + } + None => { + self.try_send_messages(stream, topic, partitioning, None, messages) + .await + } + } + } + + /// Sends one chunk with the producer section of its partition. See + /// [`producer_idempotence`](crate::clients::producer_idempotence). + async fn try_send_idempotent( + &self, + idempotence: &Idempotence, + stream: &Identifier, + topic: &Identifier, + partitioning: &Partitioning, + messages: &mut [IggyMessage], + ) -> Result<SendMessagesResponse, IggyError> { + let count = u32::try_from(messages.len()).map_err(|_| IggyError::InvalidMessagesCount)?; + let mut partition_id = self + .select_partition(stream, topic, partitioning, false) + .await?; + let mut refreshed = partitioning.kind == PartitioningKind::PartitionId; + loop { + match self + .send_idempotent_to(idempotence, stream, topic, partition_id, count, messages) + .await + { + // As for a plain send, only a missing partition permits one + // fresh selection. + Err(IggyError::PartitionNotFound(..)) if !refreshed => { + refreshed = true; + partition_id = self + .select_partition(stream, topic, partitioning, true) + .await?; + } + result => return result, + } + } + } + + async fn send_idempotent_to( + &self, + idempotence: &Idempotence, + stream: &Identifier, + topic: &Identifier, + partition_id: u32, + count: u32, + messages: &mut [IggyMessage], + ) -> Result<SendMessagesResponse, IggyError> { + let explicit = Partitioning::partition_id(partition_id); + let topic_producer = idempotence.topic_producer(stream, topic); + let lock = topic_producer.sequence(partition_id); + let mut sequence = lock.lock().await; + self.ensure_producer_id(&topic_producer).await?; + // Every resend carries this section, also after a chunk of another + // partition moved the producer to a new epoch. + let mut producer = idempotence.next_producer(&topic_producer, &mut sequence)?; + let mut uncertain = None; + let mut attempts = 0; + loop { + attempts += 1; + match self + .try_send_messages(stream, topic, &explicit, Some(producer), messages) + .await + { + Ok(response) => { + Idempotence::advance(&mut sequence, producer, count); + return Ok(response); + } + // The partition no longer holds the producer, so the chunk + // starts the next epoch at sequence 0. Not after an uncertain + // send: it can have landed before the partition dropped the + // producer, and the new epoch would write it twice. + Err(IggyError::ProducerUnknown) if attempts <= RESENDS && uncertain.is_none() => { + topic_producer.next_epoch(producer); + self.ensure_producer_id(&topic_producer).await?; + producer = idempotence.next_producer(&topic_producer, &mut sequence)?; + } + // The same producer section makes a resend safe, whether or + // not the first send landed. + Err(IggyError::OperationUncertain(op)) if attempts <= RESENDS => { + self.abandon_operation(op).await; + uncertain = Some(op); + } + Err(error) => { + if error == IggyError::ProducerEpochFenced { + idempotence.fence(); + } + // The chunk may have landed. The next chunk starts a new + // epoch, so it never follows a hole. + topic_producer.next_epoch(producer); + // An earlier send of the chunk can still land, so its + // outcome stays unknown. + return Err(match uncertain { + Some(op) if !matches!(error, IggyError::OperationUncertain(_)) => { + IggyError::OperationUncertain(op) + } + _ => error, + }); + } + } + } + } + + /// Asks the client for a producer id, once and again after the last epoch. + async fn ensure_producer_id(&self, topic_producer: &TopicProducer) -> Result<(), IggyError> { + if topic_producer.producer_id().is_none() { + let producer_id = super::with_client!(self, client => client.next_producer_id().await)?; + topic_producer.set_producer_id(producer_id); + } + Ok(()) + } + + /// The partition of a chunk, picked as a plain send picks it. + async fn select_partition( + &self, + stream: &Identifier, + topic: &Identifier, + partitioning: &Partitioning, + fresh: bool, + ) -> Result<u32, IggyError> { + super::with_client!(self, client => client.select_send_partition(stream, topic, partitioning, fresh).await) + } + + /// Frees the lane of an uncertain chunk that this producer sends again. + async fn abandon_operation(&self, op: u64) { + self.uncertain_operations().remove(&op); + if let Err(error) = self.client.read().await.abandon_operation(op).await { + trace!("Uncertain operation {op} was already released: {error}"); + } + } + + async fn try_send_messages( + &self, + stream: &Identifier, + topic: &Identifier, + partitioning: &Partitioning, + producer: Option<BatchProducer>, + messages: &mut [IggyMessage], + ) -> Result<SendMessagesResponse, IggyError> { super::with_client!(self, client => { let deadline = operation_deadline(); timeout_at(deadline, self.prune_uncertain_operations(&client)) .await .map_err(|_| IggyError::TransientNotAccepted)?; let mut partitioning = BinarySendPartitioning::new(partitioning); + if let Some(producer) = producer { + partitioning = partitioning.with_producer(producer); + } let mut result = client .send_messages_with_partitioning(stream, topic, &mut partitioning, messages) .await; @@ -354,7 +517,7 @@ let end = (index + max).min(msgs.len()); let chunk = &mut msgs[index..end]; - match self.try_send_messages(stream, topic, &part, chunk).await { + match self.send_chunk(stream, topic, &part, chunk).await { Ok(response) => confirmations.extend(response.confirmations), Err(err) => { let failed_tail = msgs.split_off(index); @@ -375,7 +538,7 @@ // background send on _ => { let response = self - .try_send_messages(stream, topic, &part, &mut msgs) + .send_chunk(stream, topic, &part, &mut msgs) .await .map_err(|err| { let sent = msgs.len(); @@ -643,6 +806,36 @@ /// automatic retry will be attempted. The default callback logs the failure and drops the context. /// Implement [`ErrorCallback`] when failed sends must be retained. /// +/// # Idempotent sends +/// +/// [`idempotent()`] turns on the idempotent mode. Each chunk then carries this producer's id, an +/// epoch and the sequence of its first message, and the partition writes a chunk once. After +/// [`IggyError::OperationUncertain`], the producer sends the chunk again with the same producer +/// section, up to three times. So a resend after a lost reply or a lost session writes nothing +/// new, and the confirmation names the offset of the first write. If a resend fails, the cause +/// stays `OperationUncertain`, because the first send can still land. +/// +/// The sequence counts per partition, so the producer picks the partition of every chunk itself, +/// as a plain send picks it: round-robin for [`Partitioning::balanced()`], and by the hash of the +/// key for a messages key. It sends the chunks of one partition one at a time, also from a +/// background producer with [`BalancedSharding`]. +/// +/// When a partition no longer holds the producer, for example after a day without a write, the +/// producer moves to its next epoch and sends the chunk again from sequence 0. If an earlier send +/// of the chunk was uncertain, the send fails with `OperationUncertain` instead, because that send +/// can have landed before the partition dropped the producer. When a chunk fails for good, the +/// next chunk also starts a new epoch, so it never follows a hole. +/// [`IggyError::ProducerEpochFenced`] means that another copy of this producer took over, and the +/// producer refuses every later send. +/// +/// Each stream and topic, as you spell them, gets its own producer id, so a topic named by number +/// and by name never shares a sequence. +/// +/// The id comes from [`next_producer_id`], so the mode needs a binary client. Released builds of +/// this SDK refuse a polled batch with a producer section. So do the Python, C++ and PHP SDKs, the +/// CLI, the Kafka gateway and the connectors runtime, which embed it, and the Swift SDK. Upgrade +/// every consumer of the topic before you turn the mode on. +/// /// # Options and defaults /// /// Everything is configured on the [`IggyProducerBuilder`] before [`build()`] and is fixed @@ -658,6 +851,7 @@ /// | [`create_stream_if_not_exists()`] | on | creating the stream during [`init()`](Self::init) | /// | [`create_topic_if_not_exists()`] | on, one partition, server defaults for expiry and max size | creating the topic during [`init()`](Self::init) | /// | [`encryptor()`] | inherited from the client | encrypting payloads and user headers | +/// | [`idempotent()`] | off | writing each chunk once, also when it is sent again | /// /// There are inverse setters as well, such as [`without_partitioning()`], /// [`without_partitioner()`], [`without_encryptor()`], [`do_not_create_stream_if_not_exists()`] and @@ -712,6 +906,8 @@ /// [`do_not_create_stream_if_not_exists()`]: crate::clients::producer_builder::IggyProducerBuilder::do_not_create_stream_if_not_exists /// [`do_not_create_topic_if_not_exists()`]: crate::clients::producer_builder::IggyProducerBuilder::do_not_create_topic_if_not_exists /// [`encryptor()`]: crate::clients::producer_builder::IggyProducerBuilder::encryptor +/// [`idempotent()`]: crate::clients::producer_builder::IggyProducerBuilder::idempotent +/// [`next_producer_id`]: iggy_common::MessageClient::next_producer_id /// [`partitioner()`]: crate::clients::producer_builder::IggyProducerBuilder::partitioner /// [`partitioning()`]: crate::clients::producer_builder::IggyProducerBuilder::partitioning /// [`stream()`]: crate::clients::producer_builder::IggyProducerBuilder::stream @@ -742,6 +938,7 @@ topic_max_size: MaxTopicSize, topic_durability: Durability, mode: SendMode, + idempotent: bool, ) -> Self { let core = Arc::new(ProducerCore { initialized: AtomicBool::new(false), @@ -766,6 +963,7 @@ SendMode::Direct(ref cfg) => Some(cfg.clone()), _ => None, }, + idempotence: idempotent.then(Idempotence::default), }); let dispatcher = match mode { SendMode::Background(cfg) => Some(ProducerDispatcher::new(core.clone(), cfg)), @@ -975,6 +1173,7 @@ }; use crate::tcp::tcp_client::TcpClient; use bytes::Bytes; + use iggy_binary_protocol::batch::BatchHeader; use iggy_binary_protocol::codes::{ CREATE_STREAM_CODE, GET_STREAM_CODE, GET_TOPIC_CODE, LOGIN_REGISTER_CODE, SEND_MESSAGES_CODE, @@ -988,8 +1187,8 @@ use iggy_binary_protocol::{WireDecode, WireEncode, WireName, WireOptions, WirePartitioning}; use iggy_common::locking::{IggyRwLock, IggyRwLockFn}; use iggy_common::{ - Client, HttpClientConfig, Identifier, IggyDuration, IggyError, IggyMessage, Partitioning, - Sizeable, UserClient, + BatchProducer, Client, HttpClientConfig, Identifier, IggyDuration, IggyError, IggyMessage, + Partitioning, Sizeable, UserClient, }; use std::pin::Pin; use std::sync::{Arc, Mutex}; @@ -1422,6 +1621,362 @@ .partitioning } + fn sent_producer(body: &[u8]) -> Option<BatchProducer> { + let (metadata_length, body) = body.split_first_chunk().unwrap(); + BatchHeader::decode(&body[u32::from_le_bytes(*metadata_length) as usize..]) + .unwrap() + .producer + } + + fn sent_producers(cluster: &FakeCluster) -> Vec<BatchProducer> { + cluster + .received(SEND_MESSAGES_CODE) + .iter() + .map(|received| sent_producer(&received.body).expect("a producer section")) + .collect() + } + + fn idempotent_producer( + client: IggyRwLock<ClientWrapper>, + partitioning: Partitioning, + ) -> IggyProducer { + IggyProducerBuilder::new( + client, + Identifier::numeric(1).unwrap(), + "stream".to_owned(), + Identifier::numeric(1).unwrap(), + "topic".to_owned(), + None, + None, + ) + .partitioning(partitioning) + .direct(DirectConfig::builder().batch_length(1).build()) + .idempotent() + .build() + } + + fn producer(producer_id: u64, epoch: u16, base_sequence: u32) -> BatchProducer { + BatchProducer::new(producer_id, epoch, base_sequence).unwrap() + } + + #[tokio::test(start_paused = true)] + async fn given_an_idempotent_producer_when_sending_chunks_should_number_them_in_order() { + let (cluster, plain) = connected_producer().await; + let producer_under_test = + idempotent_producer(plain.core.client.clone(), Partitioning::partition_id(0)); + cluster.script( + &endpoint(0), + SEND_MESSAGES_CODE, + [confirmation(0), confirmation(1), confirmation(2)], + ); + + producer_under_test + .send(vec![message(b"1"), message(b"2")]) + .await + .unwrap(); + producer_under_test.send(vec![message(b"3")]).await.unwrap(); + + let sent = sent_producers(&cluster); + let id = sent[0].producer_id(); + assert!(id >> 20 > 0, "the id holds the session number"); + assert_eq!( + sent, + [producer(id, 0, 0), producer(id, 0, 1), producer(id, 0, 2)] + ); + } + + #[tokio::test(start_paused = true)] + async fn given_a_topic_named_by_number_and_by_name_when_sending_idempotently_should_use_two_producer_ids() + { + let (cluster, plain) = connected_producer().await; + let producer_under_test = + idempotent_producer(plain.core.client.clone(), Partitioning::partition_id(0)); + cluster.script( + &endpoint(0), + SEND_MESSAGES_CODE, + [confirmation(0), confirmation(1)], + ); + + producer_under_test.send(vec![message(b"1")]).await.unwrap(); + producer_under_test + .send_to( + Arc::new(Identifier::named("stream").unwrap()), + Arc::new(Identifier::named("topic").unwrap()), + vec![message(b"2")], + None, + ) + .await + .unwrap(); + + // One id for both would send sequence 0 twice, and the partition would + // answer the second chunk with the offset of the first. + let sent = sent_producers(&cluster); + assert_eq!(sent.len(), 2); + assert_ne!(sent[0].producer_id(), sent[1].producer_id()); + assert_eq!( + sent, + [ + producer(sent[0].producer_id(), 0, 0), + producer(sent[1].producer_id(), 0, 0) + ] + ); + } + + #[tokio::test(start_paused = true)] + async fn given_a_partition_that_lost_the_producer_when_sending_should_start_the_next_epoch() { + let (cluster, plain) = connected_producer().await; + let producer_under_test = + idempotent_producer(plain.core.client.clone(), Partitioning::partition_id(0)); + cluster.script( + &endpoint(0), + SEND_MESSAGES_CODE, + [ + confirmation(0), + Answer::Refuse(IggyError::ProducerUnknown), + confirmation(1), + ], + ); + + producer_under_test + .send(vec![message(b"1"), message(b"2")]) + .await + .unwrap(); + + let sent = sent_producers(&cluster); + let id = sent[0].producer_id(); + assert_eq!( + sent, + [producer(id, 0, 0), producer(id, 0, 1), producer(id, 1, 0)] + ); + } + + #[tokio::test(start_paused = true)] + async fn given_an_uncertain_chunk_when_sending_idempotently_should_resend_the_same_producer() { + let (cluster, plain) = connected_producer().await; + let producer_under_test = + idempotent_producer(plain.core.client.clone(), Partitioning::partition_id(0)); + cluster.script( + &endpoint(0), + SEND_MESSAGES_CODE, + [Answer::Silence, confirmation(0), confirmation(1)], + ); + + let response = producer_under_test + .send(vec![message(b"1"), message(b"2")]) + .await + .unwrap(); + + assert_eq!(response.confirmations.len(), 2); + let sent = sent_producers(&cluster); + let id = sent[0].producer_id(); + let (resends, rest) = sent.split_at(sent.len() - 1); + assert!(resends.len() >= 2, "{sent:?}"); + assert!( + resends.iter().all(|sent| *sent == producer(id, 0, 0)), + "{sent:?}" + ); + assert_eq!(rest, [producer(id, 0, 1)]); + } + + #[tokio::test(start_paused = true)] + async fn given_a_fenced_producer_when_sending_again_should_refuse_without_a_request() { + let (cluster, plain) = connected_producer().await; + let producer_under_test = + idempotent_producer(plain.core.client.clone(), Partitioning::partition_id(0)); + cluster.script( + &endpoint(0), + SEND_MESSAGES_CODE, + [Answer::Refuse(IggyError::ProducerEpochFenced)], + ); + + for _ in 0..2 { + let Err(IggyError::ProducerSendFailed { cause, .. }) = + producer_under_test.send(vec![message(b"1")]).await + else { + panic!("a fenced producer must fail"); + }; + assert_eq!(*cause, IggyError::ProducerEpochFenced); + } + assert_eq!(cluster.count(SEND_MESSAGES_CODE), 1); + } + + #[tokio::test(start_paused = true)] + async fn given_an_uncertain_chunk_when_another_partition_moves_the_epoch_should_resend_the_same_producer() + { + let (cluster, plain) = connected_producer().await; + cluster + .route + .lock() + .unwrap() + .as_mut() + .unwrap() + .partitions_count = 2; + let producer_under_test = Arc::new(idempotent_producer( + plain.core.client.clone(), + Partitioning::partition_id(0), + )); + cluster.script( + &endpoint(0), + SEND_MESSAGES_CODE, + [ + Answer::Silence, + Answer::Refuse(IggyError::ProducerSequenceOutOfOrder), + confirmation(0), + ], + ); + let uncertain = tokio::spawn({ + let producer = Arc::clone(&producer_under_test); + async move { producer.send(vec![message(b"uncertain")]).await } + }); + eventually("the first chunk arrives", || { + cluster.count(SEND_MESSAGES_CODE) == 1 + }) + .await; + + let refused = producer_under_test + .send_with_partitioning( + vec![message(b"refused")], + Some(Arc::new(Partitioning::partition_id(1))), + ) + .await; + + assert!(refused.is_err(), "{refused:?}"); + uncertain.await.unwrap().unwrap(); + let sends = cluster.received(SEND_MESSAGES_CODE); + let id = sent_producer(&sends[0].body).unwrap().producer_id(); + let first_partition: Vec<_> = sends + .iter() + .filter(|sent| sent_partition(&sent.body) == WirePartitioning::PartitionId(0)) + .map(|sent| sent_producer(&sent.body).unwrap()) + .collect(); + assert!(first_partition.len() >= 2, "{first_partition:?}"); + assert!( + first_partition + .iter() + .all(|sent| *sent == producer(id, 0, 0)), + "{first_partition:?}" + ); + } + + #[tokio::test(start_paused = true)] + async fn given_an_uncertain_chunk_when_its_resend_is_refused_should_report_an_unknown_outcome() + { + let (cluster, plain) = connected_producer().await; + let producer_under_test = + idempotent_producer(plain.core.client.clone(), Partitioning::partition_id(0)); + cluster.script( + &endpoint(0), + SEND_MESSAGES_CODE, + [ + Answer::Silence, + Answer::Refuse(IggyError::ProducerEpochFenced), + ], + ); + + let Err(IggyError::ProducerSendFailed { + cause, uncertain, .. + }) = producer_under_test.send(vec![message(b"1")]).await + else { + panic!("the resend must fail"); + }; + + assert!( + matches!(*cause, IggyError::OperationUncertain(_)), + "{cause:?}" + ); + assert_eq!(uncertain, 1); + assert!(producer_under_test.send(vec![message(b"2")]).await.is_err()); + assert_eq!(cluster.count(SEND_MESSAGES_CODE), 2, "the fence holds"); + } + + #[tokio::test(start_paused = true)] + async fn given_an_uncertain_chunk_when_the_partition_lost_the_producer_should_report_an_unknown_outcome() + { + let (cluster, plain) = connected_producer().await; + let producer_under_test = + idempotent_producer(plain.core.client.clone(), Partitioning::partition_id(0)); + cluster.script( + &endpoint(0), + SEND_MESSAGES_CODE, + [ + Answer::Silence, + Answer::Refuse(IggyError::ProducerUnknown), + confirmation(0), + ], + ); + + let Err(IggyError::ProducerSendFailed { cause, .. }) = + producer_under_test.send(vec![message(b"1")]).await + else { + panic!("the resend must fail"); + }; + producer_under_test.send(vec![message(b"2")]).await.unwrap(); + + // The first send can have landed, so the chunk never goes at a new + // epoch. The next chunk starts one, so it never follows a hole. + assert!( + matches!(*cause, IggyError::OperationUncertain(_)), + "{cause:?}" + ); + let sent = sent_producers(&cluster); + let id = sent[0].producer_id(); + assert_eq!( + sent, + [producer(id, 0, 0), producer(id, 0, 0), producer(id, 1, 0)] + ); + } + + #[tokio::test(start_paused = true)] + async fn given_balanced_partitioning_when_sending_idempotently_should_pick_each_partition_first() + { + let (cluster, plain) = connected_producer().await; + cluster + .route + .lock() + .unwrap() + .as_mut() + .unwrap() + .partitions_count = 2; + cluster.script(&endpoint(0), GET_TOPIC_CODE, [Answer::Body(topic_body(2))]); + let producer_under_test = + idempotent_producer(plain.core.client.clone(), Partitioning::balanced()); + cluster.script( + &endpoint(0), + SEND_MESSAGES_CODE, + [ + confirmation_on(0, 0), + confirmation_on(1, 0), + confirmation_on(0, 1), + ], + ); + + producer_under_test + .send(vec![message(b"1"), message(b"2"), message(b"3")]) + .await + .unwrap(); + + let sends = cluster.received(SEND_MESSAGES_CODE); + let partitions: Vec<WirePartitioning> = sends + .iter() + .map(|sent| sent_partition(&sent.body)) + .collect(); + assert_eq!( + partitions, + [ + WirePartitioning::PartitionId(0), + WirePartitioning::PartitionId(1), + WirePartitioning::PartitionId(0), + ] + ); + let sent = sent_producers(&cluster); + let id = sent[0].producer_id(); + assert_eq!( + sent, + [producer(id, 0, 0), producer(id, 0, 0), producer(id, 0, 1)] + ); + assert_eq!(cluster.count(GET_TOPIC_CODE), 1, "the count is read once"); + } + #[tokio::test(start_paused = true)] async fn given_a_balanced_send_blocked_by_its_producer_when_resolving_should_keep_the_chosen_partition() {
diff --git a/core/sdk/src/clients/producer_builder.rs b/core/sdk/src/clients/producer_builder.rs index bf0fe8c..a0b4908 100644 --- a/core/sdk/src/clients/producer_builder.rs +++ b/core/sdk/src/clients/producer_builder.rs
@@ -51,6 +51,7 @@ topic_durability: Durability, partitioning: Option<Partitioning>, mode: SendMode, + idempotent: bool, } impl IggyProducerBuilder { @@ -80,6 +81,7 @@ topic_max_size: MaxTopicSize::ServerDefault, topic_durability: Durability::default(), mode: SendMode::default(), + idempotent: false, } } @@ -191,6 +193,33 @@ } } + /// Turns on the idempotent mode. Each chunk carries this producer's id, an + /// epoch and the sequence of its first message, and a partition writes it + /// once, also when the producer sends it again from a new session. + /// + /// The producer gets its id from [`next_producer_id`] during + /// [`init()`](IggyProducer::init), so it needs a binary client. It picks the + /// partition of a chunk itself, and it sends the chunks of one partition one + /// at a time. Released builds of this SDK, of every client that embeds it + /// and of the Swift SDK cannot poll the batches that it writes, so upgrade + /// every consumer of the topic first. See [`IggyProducer`]. + /// + /// [`next_producer_id`]: iggy_common::MessageClient::next_producer_id + pub fn idempotent(self) -> Self { + Self { + idempotent: true, + ..self + } + } + + /// Turns off the idempotent mode, the default. + pub fn without_idempotence(self) -> Self { + Self { + idempotent: false, + ..self + } + } + /// Sets the producer to use direct message sending. /// This mode sends from the calling task without buffering between calls. /// `DirectConfig::linger_time` can delay a call before its requests are sent. @@ -223,6 +252,7 @@ self.topic_max_size, self.topic_durability, self.mode, + self.idempotent, ) } }
diff --git a/core/sdk/src/clients/producer_idempotence.rs b/core/sdk/src/clients/producer_idempotence.rs new file mode 100644 index 0000000..2b50966 --- /dev/null +++ b/core/sdk/src/clients/producer_idempotence.rs
@@ -0,0 +1,308 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The idempotent mode of a producer. +//! +//! Every chunk carries the producer id, the epoch and the sequence of its +//! first message. A partition writes a chunk of one producer once, so the +//! producer can send a chunk again after an unknown outcome, also from a new +//! session. The sequence counts per partition, so the producer picks the +//! partition of a chunk itself and sends the chunks of one partition one at a +//! time. +//! +//! A partition knows a producer by its id alone, but the client only knows a +//! topic as the caller spells it. So each stream and topic spelling gets its +//! own id, and a topic named by number and by name never shares a sequence. + +use iggy_common::{BatchProducer, Identifier, IggyError}; +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +/// Sends of one chunk with the same producer after an unknown outcome, +/// before the send fails. +pub(crate) const RESENDS: usize = 3; + +type PartitionSequences = HashMap<u32, Arc<tokio::sync::Mutex<Sequence>>>; + +#[derive(Debug, Default)] +pub(crate) struct Idempotence { + /// Set when a newer copy of this producer wrote to a partition. + fenced: AtomicBool, + topics: Mutex<HashMap<Identifier, HashMap<Identifier, Arc<TopicProducer>>>>, +} + +/// The producer of one stream and topic, as the caller spells them. +#[derive(Debug, Default)] +pub(crate) struct TopicProducer { + identity: Mutex<Identity>, + sequences: Mutex<PartitionSequences>, +} + +/// The producer id and its current epoch. +#[derive(Debug, Default, Clone, Copy)] +struct Identity { + producer_id: Option<u64>, + epoch: u16, +} + +/// The next sequence of one partition, under the producer id and the epoch +/// that it counts for. +#[derive(Debug, Default)] +pub(crate) struct Sequence { + producer_id: u64, + epoch: u16, + next: u32, +} + +impl Idempotence { + /// The producer of `topic` in `stream`, as the caller spells them. + pub(crate) fn topic_producer( + &self, + stream: &Identifier, + topic: &Identifier, + ) -> Arc<TopicProducer> { + let mut topics = self.topics.lock().unwrap_or_else(PoisonError::into_inner); + if let Some(producer) = topics.get(stream).and_then(|topics| topics.get(topic)) { + return Arc::clone(producer); + } + Arc::clone( + topics + .entry(stream.clone()) + .or_default() + .entry(topic.clone()) + .or_default(), + ) + } + + /// The producer section of the next chunk of `sequence`. A new producer + /// id or epoch starts the partition again at sequence 0. + pub(crate) fn next_producer( + &self, + topic_producer: &TopicProducer, + sequence: &mut Sequence, + ) -> Result<BatchProducer, IggyError> { + if self.fenced.load(Ordering::SeqCst) { + return Err(IggyError::ProducerEpochFenced); + } + let identity = *topic_producer.identity(); + let producer_id = identity + .producer_id + .ok_or(IggyError::ProducerIdsExhausted)?; + if sequence.producer_id != producer_id || sequence.epoch != identity.epoch { + *sequence = Sequence { + producer_id, + epoch: identity.epoch, + next: 0, + }; + } + BatchProducer::new(producer_id, identity.epoch, sequence.next) + .ok_or(IggyError::InvalidCommand) + } + + /// Records the written chunk of `count` messages that `producer` started. + pub(crate) fn advance(sequence: &mut Sequence, producer: BatchProducer, count: u32) { + sequence.next = producer.next(count).base_sequence(); + } + + /// A newer copy of this producer wrote to a partition, so this one stops. + pub(crate) fn fence(&self) { + self.fenced.store(true, Ordering::SeqCst); + } +} + +impl TopicProducer { + /// The producer id, once a session gave one. + pub(crate) fn producer_id(&self) -> Option<u64> { + self.identity().producer_id + } + + /// Keeps `producer_id` unless a concurrent chunk already set one. + pub(crate) fn set_producer_id(&self, producer_id: u64) { + let mut identity = self.identity(); + if identity.producer_id.is_none() { + identity.producer_id = Some(producer_id); + identity.epoch = 0; + } + } + + /// The lock that orders the chunks of one partition. The holder sends + /// one chunk at a time, so the sequence order is the send order. + pub(crate) fn sequence(&self, partition_id: u32) -> Arc<tokio::sync::Mutex<Sequence>> { + let mut sequences = self + .sequences + .lock() + .unwrap_or_else(PoisonError::into_inner); + Arc::clone(sequences.entry(partition_id).or_default()) + } + + /// Moves past `producer` to the next epoch, unless a chunk of another + /// partition already did. After the last epoch the producer id goes, and + /// the caller asks for a new one. + pub(crate) fn next_epoch(&self, producer: BatchProducer) { + let mut identity = self.identity(); + if identity.producer_id != Some(producer.producer_id()) + || identity.epoch != producer.epoch() + { + return; + } + if producer.epoch() == BatchProducer::EPOCH_MAX { + identity.producer_id = None; + identity.epoch = 0; + } else { + identity.epoch += 1; + } + } + + fn identity(&self) -> MutexGuard<'_, Identity> { + self.identity.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn stream() -> Identifier { + Identifier::numeric(1).unwrap() + } + + fn topic() -> Identifier { + Identifier::numeric(2).unwrap() + } + + fn producer(producer_id: u64, epoch: u16, base_sequence: u32) -> BatchProducer { + BatchProducer::new(producer_id, epoch, base_sequence).unwrap() + } + + #[tokio::test] + async fn given_written_chunks_when_sending_more_should_count_each_partition_on_its_own() { + let idempotence = Idempotence::default(); + let topic_producer = idempotence.topic_producer(&stream(), &topic()); + topic_producer.set_producer_id(7); + let first = topic_producer.sequence(0); + let second = topic_producer.sequence(1); + + let mut sequence = first.lock().await; + let sent = idempotence + .next_producer(&topic_producer, &mut sequence) + .unwrap(); + assert_eq!(sent, producer(7, 0, 0)); + Idempotence::advance(&mut sequence, sent, 3); + assert_eq!( + idempotence + .next_producer(&topic_producer, &mut sequence) + .unwrap(), + producer(7, 0, 3) + ); + drop(sequence); + + let mut other = second.lock().await; + assert_eq!( + idempotence + .next_producer(&topic_producer, &mut other) + .unwrap(), + producer(7, 0, 0) + ); + } + + #[test] + fn given_two_spellings_of_one_topic_when_taking_their_producers_should_keep_two_ids() { + let idempotence = Idempotence::default(); + let by_number = idempotence.topic_producer(&stream(), &topic()); + let by_name = idempotence.topic_producer( + &Identifier::named("stream").unwrap(), + &Identifier::named("topic").unwrap(), + ); + + by_number.set_producer_id(7); + + assert_eq!(by_name.producer_id(), None, "the name asks for its own id"); + assert!(Arc::ptr_eq( + &idempotence.topic_producer(&stream(), &topic()), + &by_number + )); + } + + #[tokio::test] + async fn given_a_new_epoch_when_sending_should_start_every_partition_at_zero() { + let idempotence = Idempotence::default(); + let topic_producer = idempotence.topic_producer(&stream(), &topic()); + topic_producer.set_producer_id(7); + let lock = topic_producer.sequence(0); + let mut sequence = lock.lock().await; + let sent = idempotence + .next_producer(&topic_producer, &mut sequence) + .unwrap(); + Idempotence::advance(&mut sequence, sent, 5); + + topic_producer.next_epoch(sent); + topic_producer.next_epoch(sent); + + assert_eq!( + idempotence + .next_producer(&topic_producer, &mut sequence) + .unwrap(), + producer(7, 1, 0), + "a second move past the same producer is a no-op" + ); + } + + #[tokio::test] + async fn given_the_last_epoch_when_moving_on_should_drop_the_producer_id() { + let idempotence = Idempotence::default(); + let topic_producer = idempotence.topic_producer(&stream(), &topic()); + topic_producer.set_producer_id(7); + let lock = topic_producer.sequence(0); + let mut sequence = lock.lock().await; + for _ in 0..BatchProducer::EPOCH_MAX { + let sent = idempotence + .next_producer(&topic_producer, &mut sequence) + .unwrap(); + topic_producer.next_epoch(sent); + } + let last = idempotence + .next_producer(&topic_producer, &mut sequence) + .unwrap(); + assert_eq!(last.epoch(), BatchProducer::EPOCH_MAX); + + topic_producer.next_epoch(last); + + assert_eq!(topic_producer.producer_id(), None); + topic_producer.set_producer_id(8); + assert_eq!( + idempotence + .next_producer(&topic_producer, &mut sequence) + .unwrap(), + producer(8, 0, 0) + ); + } + + #[tokio::test] + async fn given_a_fenced_producer_when_sending_should_refuse() { + let idempotence = Idempotence::default(); + let topic_producer = idempotence.topic_producer(&stream(), &topic()); + topic_producer.set_producer_id(7); + idempotence.fence(); + let lock = topic_producer.sequence(0); + + assert_eq!( + idempotence.next_producer(&topic_producer, &mut *lock.lock().await), + Err(IggyError::ProducerEpochFenced) + ); + } +}
diff --git a/core/sdk/src/prelude.rs b/core/sdk/src/prelude.rs index 5f28759..e252507 100644 --- a/core/sdk/src/prelude.rs +++ b/core/sdk/src/prelude.rs
@@ -50,15 +50,15 @@ pub use crate::tcp::tcp_client::TcpClient; pub use crate::websocket::websocket_client::WebSocketClient; pub use iggy_common::{ - Aes256GcmEncryptor, Args, ArgsOptional, AutoCommit, AutoLogin, CacheMetrics, CacheMetricsKey, - Checkpoint, ClientError, ClientInfoDetails, ClusterMetadata, ClusterNode, ClusterNodeRole, - ClusterNodeStatus, CommitAfter, CommitEvery, CommitPolicy, CompressionAlgorithm, Consumer, - ConsumerGroup, ConsumerGroupAssignment, ConsumerGroupDetails, ConsumerGroupMember, - ConsumerKind, Credentials, Durability, EncryptorKind, GlobalPermissions, HeaderField, - HeaderKey, HeaderKind, HeaderValue, HttpClientConfig, HttpClientConfigBuilder, HttpMethod, - IdKind, Identifier, IdentityInfo, IggyByteSize, IggyDuration, IggyError, IggyExpiry, - IggyIndexView, IggyMessage, IggyMessageHeader, IggyMessageHeaderView, IggyMessageView, - IggyMessageViewIterator, IggyTimestamp, MaxTopicSize, NonZeroDurationError, + Aes256GcmEncryptor, Args, ArgsOptional, AutoCommit, AutoLogin, BatchProducer, CacheMetrics, + CacheMetricsKey, Checkpoint, ClientError, ClientInfoDetails, ClusterMetadata, ClusterNode, + ClusterNodeRole, ClusterNodeStatus, CommitAfter, CommitEvery, CommitPolicy, + CompressionAlgorithm, Consumer, ConsumerGroup, ConsumerGroupAssignment, ConsumerGroupDetails, + ConsumerGroupMember, ConsumerKind, Credentials, Durability, EncryptorKind, GlobalPermissions, + HeaderField, HeaderKey, HeaderKind, HeaderValue, HttpClientConfig, HttpClientConfigBuilder, + HttpMethod, IdKind, Identifier, IdentityInfo, IggyByteSize, IggyDuration, IggyError, + IggyExpiry, IggyIndexView, IggyMessage, IggyMessageHeader, IggyMessageHeaderView, + IggyMessageView, IggyMessageViewIterator, IggyTimestamp, MaxTopicSize, NonZeroDurationError, NonZeroIggyDuration, OptionKey, OptionSpec, OptionValue, OptionsScope, Partition, PartitionContext, PartitionResizePolicy, Partitioner, Partitioning, PendingOperationsConfig, Permissions, PersonalAccessTokenExpiry, PollMessages, PollOptions, PolledMessages, PollingKind,
diff --git a/core/sdk/src/session_owner.rs b/core/sdk/src/session_owner.rs index 276c156..faa31f9 100644 --- a/core/sdk/src/session_owner.rs +++ b/core/sdk/src/session_owner.rs
@@ -25,6 +25,7 @@ }; use lanes::{Lanes, ProvedOutcomes}; use pending::PendingOperations; +use producer_ids::ProducerIds; use secrecy::{ExposeSecret, SecretString}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; @@ -32,16 +33,18 @@ mod lanes; mod pending; +mod producer_ids; pub(crate) use lanes::{LaneKey, LaneTicket, RetainedOperation}; /// The session state that every link of one client shares: the consensus -/// session, its epoch, the lanes, the metadata watermark and the sign-in. +/// session, its epoch, the lanes, the metadata watermark, the sign-in and the +/// producer ids. /// /// Lock order is session, lanes, then either the proved outcomes or the -/// pending-operation usage. The sign-in also comes after the session. The -/// epoch is read and written under session or lanes, never the other way -/// round. No lock is held across an await. +/// pending-operation usage. The sign-in and the producer ids also come after +/// the session. The epoch is read and written under session or lanes, never +/// the other way round. No lock is held across an await. #[derive(Debug)] pub(crate) struct SessionOwner { /// The initial client id identifies this owner even after session replacement. @@ -61,6 +64,7 @@ metadata_watermark: AtomicU64, sign_in: Mutex<SignIn>, auto_login: AutoLogin, + producer_ids: Mutex<ProducerIds>, } /// The fence of the current session, whether the session is bound, and the @@ -126,6 +130,7 @@ metadata_watermark: AtomicU64::new(0), sign_in: Mutex::default(), auto_login, + producer_ids: Mutex::default(), } } @@ -133,6 +138,18 @@ Arc::clone(&self.consumer_group_state) } + /// The next producer id of the current session. See [`ProducerIds`]. + pub(crate) fn next_producer_id(&self) -> Result<u64, IggyError> { + // Hold the session until the id is taken. A reset in between would + // restart the range of the old session and hand out its ids again. + let session = self.session()?; + let number = session.session().ok_or(IggyError::Unauthenticated)?; + self.producer_ids + .lock() + .map_err(|_| IggyError::InvalidConfiguration)? + .next(number) + } + /// The caller must not replace the session through the guard, because /// only [`Self::reset`] raises the fence. pub(crate) fn session(&self) -> Result<MutexGuard<'_, ConsensusSession>, IggyError> {
diff --git a/core/sdk/src/session_owner/producer_ids.rs b/core/sdk/src/session_owner/producer_ids.rs new file mode 100644 index 0000000..43b4fe1 --- /dev/null +++ b/core/sdk/src/session_owner/producer_ids.rs
@@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Producer ids built from the session number. +//! +//! The session number is the op of the client's `Register` in the metadata +//! log, so no other session of the cluster ever gets it. An id puts it above +//! a local index: +//! +//! ```text +//! 63 62 20 19 0 +//! +---+----------------------------------------+---------------+ +//! | 0 | session number, 43 bits | index, 20 bits | +//! +---+----------------------------------------+---------------+ +//! ``` + +use iggy_common::IggyError; + +const INDEX_BITS: u32 = 20; + +/// Ids one session hands out. +pub(crate) const IDS_PER_SESSION: u64 = 1 << INDEX_BITS; + +/// The first session number that no longer fits beside the index in a +/// non-negative 64-bit id. +pub(crate) const SESSION_LIMIT: u64 = 1 << (i64::BITS - 1 - INDEX_BITS); + +/// The id range of the current session and the next index in it. +#[derive(Debug, Default)] +pub(crate) struct ProducerIds { + session: u64, + next_index: u64, +} + +impl ProducerIds { + /// The next id of `session`, the current session number of the client. + /// + /// A new session starts its own range, and the ids of an earlier one stay + /// valid because no other client gets its number. + pub(crate) fn next(&mut self, session: u64) -> Result<u64, IggyError> { + if session == 0 { + return Err(IggyError::Unauthenticated); + } + if session >= SESSION_LIMIT { + return Err(IggyError::ProducerIdsExhausted); + } + if session != self.session { + self.session = session; + self.next_index = 0; + } + if self.next_index == IDS_PER_SESSION { + return Err(IggyError::ProducerIdsExhausted); + } + let id = session << INDEX_BITS | self.next_index; + self.next_index += 1; + Ok(id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use iggy_binary_protocol::batch::BatchProducer; + + #[test] + fn given_a_session_when_handing_out_ids_should_count_from_its_range() { + let mut ids = ProducerIds::default(); + + assert_eq!(ids.next(4711), Ok(4711 << 20)); + assert_eq!(ids.next(4711), Ok(4711 << 20 | 1)); + assert_eq!(ids.next(4711), Ok(4_939_841_538)); + } + + #[test] + fn given_two_clients_when_handing_out_ids_should_never_share_one() { + let mut first = ProducerIds::default(); + let mut second = ProducerIds::default(); + let first_ids: Vec<u64> = (0..1_000).map(|_| first.next(10).unwrap()).collect(); + let second_ids: Vec<u64> = (0..1_000).map(|_| second.next(11).unwrap()).collect(); + + assert!(first_ids.iter().all(|id| !second_ids.contains(id))); + assert!(first_ids.iter().all(|id| id >> 20 == 10)); + assert!(second_ids.iter().all(|id| id >> 20 == 11)); + } + + #[test] + fn given_a_new_session_when_handing_out_ids_should_start_its_own_range() { + let mut ids = ProducerIds::default(); + let before = ids.next(10).unwrap(); + + let after = ids.next(25).unwrap(); + + assert_eq!(after, 25 << 20); + assert!(after > before, "an earlier session's ids stay below"); + } + + #[test] + fn given_a_spent_session_when_handing_out_another_id_should_fail_until_a_new_session() { + let mut ids = ProducerIds { + session: 10, + next_index: IDS_PER_SESSION - 1, + }; + + assert_eq!(ids.next(10), Ok(10 << 20 | (IDS_PER_SESSION - 1))); + assert_eq!(ids.next(10), Err(IggyError::ProducerIdsExhausted)); + assert_eq!(ids.next(11), Ok(11 << 20)); + } + + #[test] + fn given_a_session_number_at_the_limit_when_handing_out_an_id_should_fail() { + let mut ids = ProducerIds::default(); + + assert_eq!( + ids.next(SESSION_LIMIT), + Err(IggyError::ProducerIdsExhausted) + ); + let last = ids.next(SESSION_LIMIT - 1).unwrap(); + assert!(BatchProducer::new(last, 0, 0).is_some()); + assert_eq!(ids.next(0), Err(IggyError::Unauthenticated)); + } +}
diff --git a/core/server/src/dispatch/partition.rs b/core/server/src/dispatch/partition.rs index 889b3ef..4631878 100644 --- a/core/server/src/dispatch/partition.rs +++ b/core/server/src/dispatch/partition.rs
@@ -1740,6 +1740,7 @@ use consensus::PartitionsHandle; #[cfg(target_os = "linux")] use consensus::Sequencer; + use iggy_binary_protocol::batch::{BATCH_HEADER_SIZE, BATCH_PRODUCER_OFFSET}; use iggy_binary_protocol::codes::PING_CODE; use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment; use iggy_binary_protocol::requests::consumer_groups::CreateConsumerGroupRequest; @@ -2714,6 +2715,24 @@ ), IggyError::InvalidIdentifier, )); + for partitioning in [ + WirePartitioning::Balanced, + WirePartitioning::MessagesKey(b"key".to_vec()), + ] { + let mut body = send_body( + WireIdentifier::Numeric(0), + WireIdentifier::Numeric(0), + partitioning, + ); + let mut batch = [0; BATCH_HEADER_SIZE]; + batch[BATCH_PRODUCER_OFFSET] = 1; + body.extend_from_slice(&batch); + cases.push(( + Operation::SendMessages, + body, + IggyError::ProducerPartitionRequired, + )); + } cases.push(( Operation::DeleteSegments, DeleteSegmentsRequest {
diff --git a/core/server/src/namespace.rs b/core/server/src/namespace.rs index 08b2fb9..21b59bb 100644 --- a/core/server/src/namespace.rs +++ b/core/server/src/namespace.rs
@@ -26,6 +26,7 @@ use crate::shell::{ShellBus, ShellShard}; use consensus::MetadataHandle; use iggy_binary_protocol::PrepareHeader; +use iggy_binary_protocol::batch::BATCH_PRODUCER_OFFSET; use iggy_binary_protocol::primitives::consumer::WireConsumer; use iggy_binary_protocol::requests::consumer_offsets::{ DeleteConsumerOffsetRequest, StoreConsumerOffsetRequest, @@ -123,6 +124,15 @@ } let header = SendMessagesHeader::decode_from(&body[4..4 + metadata_length]) .map_err(|_| IggyError::InvalidCommand)?; + // A producer counts its sequence per partition, so its batch must + // name one. Refused before `Balanced` moves its cursor. + if !matches!(header.partitioning, WirePartitioning::PartitionId(_)) + && body + .get(4 + metadata_length + BATCH_PRODUCER_OFFSET) + .is_some_and(|&flags| flags != 0) + { + return Err(IggyError::ProducerPartitionRequired); + } resolve_send_messages_namespace(shard, &header)? } Operation::StoreConsumerOffset => {
diff --git a/core/server/src/partition_helpers.rs b/core/server/src/partition_helpers.rs index 13a1e27..8aa95d5 100644 --- a/core/server/src/partition_helpers.rs +++ b/core/server/src/partition_helpers.rs
@@ -1447,7 +1447,7 @@ const REPLICA: u8 = 1; const REPLICAS: u8 = 3; const OFFSETS_MAGIC: &[u8; 4] = b"ICO1"; - const OFFSETS_VERSION: u8 = 8; + const OFFSETS_VERSION: u8 = 9; const RETRY_CHECKPOINT_MAGIC: &[u8; 4] = b"IRP2"; #[compio::test] @@ -2483,6 +2483,8 @@ protection.push(0); protection.extend_from_slice(&0_u128.to_le_bytes()); protection.extend_from_slice(&0_u32.to_le_bytes()); + protection.extend_from_slice(&0_u64.to_le_bytes()); + protection.extend_from_slice(&0_u32.to_le_bytes()); protection .extend_from_slice(&consensus::state_artifact_checksum(&protection).to_le_bytes()); let mut checkpoint = RETRY_CHECKPOINT_MAGIC.to_vec();
diff --git a/core/simulator/src/client.rs b/core/simulator/src/client.rs index 88236e2..531e0cd 100644 --- a/core/simulator/src/client.rs +++ b/core/simulator/src/client.rs
@@ -16,6 +16,7 @@ // under the License. use bytes::{Bytes, BytesMut}; +use iggy_binary_protocol::batch::BatchProducer; use iggy_binary_protocol::codes::{ CANCEL_DEFERRED_REQUEST_CODE, FETCH_MESSAGES_CODE, GET_STREAM_CODE, POLL_MESSAGES_CODE, POLL_MESSAGES_DEFERRED_CODE, @@ -784,6 +785,28 @@ group: IggyNamespace, messages: &[Bytes], ) -> Message<RoutedRequestHeader> { + self.send_batch(group, None, messages) + } + + /// [`Self::send_messages`] as one batch of an idempotent producer. + /// + /// # Panics + /// Panics if a group id exceeds `u32` or the request buffer is invalid. + pub fn send_producer_messages( + &self, + group: IggyNamespace, + producer: BatchProducer, + messages: &[Bytes], + ) -> Message<RoutedRequestHeader> { + self.send_batch(group, Some(producer), messages) + } + + fn send_batch( + &self, + group: IggyNamespace, + producer: Option<BatchProducer>, + messages: &[Bytes], + ) -> Message<RoutedRequestHeader> { let to_u32 = |v: usize| u32::try_from(v).expect("group id fits u32"); let stream_id = WireIdentifier::Numeric(to_u32(group.stream_id())); let topic_id = WireIdentifier::Numeric(to_u32(group.topic_id())); @@ -806,8 +829,15 @@ let size = SendMessagesEncoder::encoded_size(&stream_id, &topic_id, &partitioning, &raw); let mut buf = BytesMut::with_capacity(size); - SendMessagesEncoder::encode(&mut buf, &stream_id, &topic_id, &partitioning, &raw) - .expect("simulator send batch encodes"); + SendMessagesEncoder::encode_with_producer( + &mut buf, + &stream_id, + &topic_id, + &partitioning, + producer, + &raw, + ) + .expect("simulator send batch encodes"); self.build_request_with_namespace(Operation::SendMessages, &buf, group) }
diff --git a/core/simulator/src/idempotent_producer_tests.rs b/core/simulator/src/idempotent_producer_tests.rs new file mode 100644 index 0000000..32bafe0 --- /dev/null +++ b/core/simulator/src/idempotent_producer_tests.rs
@@ -0,0 +1,231 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A batch of an idempotent producer lands once. Its primary is cut off +//! before it commits the batch, the next view commits it, and another client +//! sends the batch again to the new primary. + +use bytes::Bytes; +use consensus::Status; +use iggy_binary_protocol::batch::BatchProducer; +use iggy_binary_protocol::responses::messages::SendMessagesResponse; +use iggy_binary_protocol::{ReplyHeader, RoutedRequestHeader, WireDecode}; +use iggy_common::IggyError; +use server_common::Message; +use server_common::sharding::IggyNamespace; + +use crate::Simulator; +use crate::client::SimClient; +use crate::packet::{PacketSimulatorOptions, ProcessId}; +use crate::partition_repair_driver_tests::group_state; +use crate::stale_prepare_commit_tests::{isolate, polled, primary_of}; + +const SEED: u64 = 0x5052_4F44; +const REPLICAS: u8 = 3; +const OLD_PRIMARY: u8 = 0; +const FIRST_CLIENT: u128 = 0x1D3_0001; +const SECOND_CLIENT: u128 = 0x1D3_0002; +const STEPS_PER_SEND: usize = 12; +const REPLY_STEPS: usize = 100; +/// Well past `NORMAL_HEARTBEAT_TICKS` (500), so the majority elects. +const ELECTION_STEPS: usize = 5_000; +const RESEND_ATTEMPTS: usize = 50; + +#[test] +fn given_a_producer_batch_cut_off_before_its_commit_when_another_client_resends_it_to_the_next_primary_should_land_once() + { + let (mut sim, first, second) = two_client_cluster(SEED); + let namespace = IggyNamespace::new(1, 1, 0); + sim.init_partition(namespace); + sim.register_client_with_primary(&first); + sim.register_client_with_primary(&second); + let producer = BatchProducer::new(21_000, 0, 0).expect("a producer within the field limits"); + + let warmup = first.send_messages(namespace, &[Bytes::from_static(b"warmup")]); + let reply = submit_and_wait(&mut sim, &first, OLD_PRIMARY, warmup); + assert_eq!(reply.header().status, 0, "the warmup send must commit"); + let (_, _, warm_commit, _) = group_state(&sim, OLD_PRIMARY, namespace); + + // The backups journal the batch, but their acks never reach the primary, + // so it never commits there. + set_acks(&mut sim, false); + let batch = first.send_producer_messages(namespace, producer, &[Bytes::from_static(b"once")]); + sim.submit_request(first.client_id(), OLD_PRIMARY, batch.into_generic()); + for _ in 0..STEPS_PER_SEND { + sim.step(); + } + let (_, _, cut_commit, _) = group_state(&sim, OLD_PRIMARY, namespace); + assert_eq!( + cut_commit, warm_commit, + "the old primary must not commit the batch" + ); + + isolate(&mut sim, OLD_PRIMARY, true); + let new_view = (0..ELECTION_STEPS) + .find_map(|_| { + sim.step(); + let (status, view, _, _) = group_state(&sim, 1, namespace); + (status == Status::Normal && view > 0).then_some(view) + }) + .expect("replicas 1 and 2 never elected a new view"); + let new_primary = primary_of(&sim, 1, namespace, new_view); + assert_ne!(new_primary, OLD_PRIMARY); + + let base_offset = resend_until_answered(&mut sim, &second, new_primary, namespace, producer); + assert_eq!( + base_offset, 1, + "the resent batch must get the offset the view committed" + ); + let payloads: Vec<String> = polled(&mut sim, new_primary, namespace) + .into_iter() + .map(|(_, payload)| payload) + .collect(); + assert_eq!(payloads, ["warmup", "once"], "the batch must land once"); +} + +#[test] +fn given_a_producer_batch_in_flight_when_another_client_resends_it_to_the_same_primary_should_hold_it_until_the_first_commits() + { + let (mut sim, first, second) = two_client_cluster(SEED); + let namespace = IggyNamespace::new(1, 1, 0); + sim.init_partition(namespace); + sim.register_client_with_primary(&first); + sim.register_client_with_primary(&second); + let producer = BatchProducer::new(22_000, 0, 0).expect("a producer within the field limits"); + + let warmup = first.send_messages(namespace, &[Bytes::from_static(b"warmup")]); + let reply = submit_and_wait(&mut sim, &first, OLD_PRIMARY, warmup); + assert_eq!(reply.header().status, 0, "the warmup send must commit"); + + set_acks(&mut sim, false); + let batch = first.send_producer_messages(namespace, producer, &[Bytes::from_static(b"once")]); + sim.submit_request(first.client_id(), OLD_PRIMARY, batch.into_generic()); + for _ in 0..STEPS_PER_SEND { + sim.step(); + } + let resend = second.send_producer_messages(namespace, producer, &[Bytes::from_static(b"once")]); + let reply = submit_and_wait(&mut sim, &second, OLD_PRIMARY, resend); + assert_eq!( + reply.header().status, + IggyError::TransientNotCommitted.as_code(), + "a resend of an uncommitted batch must wait for it" + ); + + set_acks(&mut sim, true); + let base_offset = resend_until_answered(&mut sim, &second, OLD_PRIMARY, namespace, producer); + assert_eq!(base_offset, 1); + let payloads: Vec<String> = polled(&mut sim, OLD_PRIMARY, namespace) + .into_iter() + .map(|(_, payload)| payload) + .collect(); + assert_eq!(payloads, ["warmup", "once"], "the batch must land once"); +} + +/// Cut or restore the links from the backups to the old primary, which carry +/// their acks. +fn set_acks(sim: &mut Simulator, delivered: bool) { + for backup in 1..REPLICAS { + sim.network.set_link_filter( + ProcessId::Replica(backup), + ProcessId::Replica(OLD_PRIMARY), + delivered, + ); + } +} + +/// Send the batch again until the new primary answers it. It answers +/// `TransientNotCommitted` while the batch is still in flight there, and +/// `TransientNotAccepted` while it cannot take the request yet. +fn resend_until_answered( + sim: &mut Simulator, + client: &SimClient, + target: u8, + namespace: IggyNamespace, + producer: BatchProducer, +) -> u64 { + for _ in 0..RESEND_ATTEMPTS { + let resend = + client.send_producer_messages(namespace, producer, &[Bytes::from_static(b"once")]); + let reply = submit_and_wait(sim, client, target, resend); + match reply.header().status { + 0 => { + let response = SendMessagesResponse::decode_from(reply.body()) + .expect("a send reply names its offset"); + return response.confirmations[0].base_offset; + } + status + if status == IggyError::TransientNotCommitted.as_code() + || status == IggyError::TransientNotAccepted.as_code() => + { + for _ in 0..STEPS_PER_SEND { + sim.step(); + } + } + status => panic!("the resent batch was refused with {status}"), + } + } + panic!("the new primary never answered the resent batch"); +} + +fn submit_and_wait( + sim: &mut Simulator, + client: &SimClient, + target: u8, + request: Message<RoutedRequestHeader>, +) -> Message<ReplyHeader> { + let request_id = request.header().request; + sim.submit_request(client.client_id(), target, request.into_generic()); + for _ in 0..REPLY_STEPS { + if let Some(reply) = sim.step().into_iter().find(|reply| { + reply.header().client == client.client_id() && reply.header().request == request_id + }) { + return reply; + } + } + panic!( + "request {request_id} of client {} got no reply", + client.client_id() + ); +} + +/// The partition cluster of `partition_repair_driver_tests::cluster`, with a +/// second client: the gateway that takes over a producer. +fn two_client_cluster(seed: u64) -> (Simulator, SimClient, SimClient) { + server_common::MemoryPool::init_pool(&server_common::MemoryPoolSettings { + enabled: false, + size: iggy_common::IggyByteSize::from(0u64), + bucket_capacity: 1, + }); + let clients = [FIRST_CLIENT, SECOND_CLIENT]; + let network_options = PacketSimulatorOptions { + node_count: REPLICAS, + client_count: u8::try_from(clients.len()).expect("two clients"), + seed, + ..PacketSimulatorOptions::default() + }; + let sim = Simulator::with_shards( + usize::from(REPLICAS), + 1, + clients.into_iter(), + network_options, + ); + ( + sim, + SimClient::new(FIRST_CLIENT), + SimClient::new(SECOND_CLIENT), + ) +}
diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs index 0729baa..650ceec 100644 --- a/core/simulator/src/lib.rs +++ b/core/simulator/src/lib.rs
@@ -2244,6 +2244,10 @@ #[cfg(test)] mod deferred_commit_tests; + +#[cfg(test)] +mod idempotent_producer_tests; + #[cfg(test)] mod slow_replica_tests;
diff --git a/core/simulator/src/stale_prepare_commit_tests.rs b/core/simulator/src/stale_prepare_commit_tests.rs index dd8302d..cb62c41 100644 --- a/core/simulator/src/stale_prepare_commit_tests.rs +++ b/core/simulator/src/stale_prepare_commit_tests.rs
@@ -52,7 +52,7 @@ } } -fn isolate(sim: &mut Simulator, replica: u8, isolated: bool) { +pub fn isolate(sim: &mut Simulator, replica: u8, isolated: bool) { for peer in (0..REPLICAS).filter(|peer| *peer != replica) { sim.network.set_link_filter( ProcessId::Replica(replica), @@ -69,7 +69,7 @@ /// `(offset, payload)` for every message a consumer can poll from `replica`, /// read through the ordinary client poll path. -fn polled(sim: &mut Simulator, replica: u8, namespace: IggyNamespace) -> Vec<(u64, String)> { +pub fn polled(sim: &mut Simulator, replica: u8, namespace: IggyNamespace) -> Vec<(u64, String)> { let mut out = Vec::new(); loop { let next = out.last().map_or(0, |(offset, _)| offset + 1); @@ -505,7 +505,7 @@ .repair } -fn primary_of(sim: &Simulator, replica: u8, namespace: IggyNamespace, view: u32) -> u8 { +pub fn primary_of(sim: &Simulator, replica: u8, namespace: IggyNamespace, view: u32) -> u8 { sim.replicas[usize::from(replica)] .partition_shard(namespace) .plane
diff --git a/examples/python/uv.lock b/examples/python/uv.lock index b259564..f1c66f9 100644 --- a/examples/python/uv.lock +++ b/examples/python/uv.lock
@@ -8,7 +8,7 @@ [[package]] name = "apache-iggy" -version = "0.9.1.dev8" +version = "0.9.1.dev9" source = { directory = "../../foreign/python" } [package.metadata]
diff --git a/foreign/python/Cargo.toml b/foreign/python/Cargo.toml index 7c3adc5..d737d65 100644 --- a/foreign/python/Cargo.toml +++ b/foreign/python/Cargo.toml
@@ -17,7 +17,7 @@ [package] name = "apache-iggy" -version = "0.9.1-dev8" +version = "0.9.1-dev9" edition = "2024" authors = ["Iggy Committers <dev@iggy.apache.org>"] license = "Apache-2.0" @@ -37,7 +37,7 @@ [dependencies] bytes = "1.12.1" futures = "0.3.34" -iggy = { path = "../../core/sdk", version = "0.11.1-edge.7" } +iggy = { path = "../../core/sdk", version = "0.11.1-edge.8" } paste = "1" pyo3 = { version = "0.29.3", features = ["experimental-async"] } pyo3-async-runtimes = { version = "0.29.0", features = [
diff --git a/foreign/python/pyproject.toml b/foreign/python/pyproject.toml index 9a5e496..3091074 100644 --- a/foreign/python/pyproject.toml +++ b/foreign/python/pyproject.toml
@@ -22,7 +22,7 @@ [project] name = "apache-iggy" requires-python = ">=3.11" -version = "0.9.1.dev8" +version = "0.9.1.dev9" description = "Apache Iggy is the persistent message streaming platform written in Rust, supporting QUIC, TCP and HTTP transport protocols, capable of processing millions of messages per second." readme = "README.md" license = { file = "LICENSE" }
diff --git a/foreign/python/uv.lock b/foreign/python/uv.lock index a1ce1e6..0ab7ee1 100644 --- a/foreign/python/uv.lock +++ b/foreign/python/uv.lock
@@ -11,7 +11,7 @@ [[package]] name = "apache-iggy" -version = "0.9.1.dev8" +version = "0.9.1.dev9" source = { editable = "." } [package.optional-dependencies]
diff --git a/foreign/swift/Sources/Iggy/Errors/IggyErrorCode.swift b/foreign/swift/Sources/Iggy/Errors/IggyErrorCode.swift index 18930a3..ea152ad 100644 --- a/foreign/swift/Sources/Iggy/Errors/IggyErrorCode.swift +++ b/foreign/swift/Sources/Iggy/Errors/IggyErrorCode.swift
@@ -97,6 +97,11 @@ case lifecycleBusy = 88 case notPrimary = 89 case deferredPollCapacityExceeded = 90 + case producerIdsExhausted = 91 + case producerSequenceOutOfOrder = 92 + case producerEpochFenced = 93 + case producerUnknown = 94 + case producerPartitionRequired = 95 case clientNotFound = 100 case invalidClientId = 101 case connectionClosed = 206 @@ -365,6 +370,11 @@ case .lifecycleBusy: "lifecycle_busy" case .notPrimary: "not_primary" case .deferredPollCapacityExceeded: "deferred_poll_capacity_exceeded" + case .producerIdsExhausted: "producer_ids_exhausted" + case .producerSequenceOutOfOrder: "producer_sequence_out_of_order" + case .producerEpochFenced: "producer_epoch_fenced" + case .producerUnknown: "producer_unknown" + case .producerPartitionRequired: "producer_partition_required" case .clientNotFound: "client_not_found" case .invalidClientId: "invalid_client_id" case .connectionClosed: "connection_closed"
diff --git a/foreign/swift/Sources/Iggy/Iggy.swift b/foreign/swift/Sources/Iggy/Iggy.swift index ddcbde6..b0de655 100644 --- a/foreign/swift/Sources/Iggy/Iggy.swift +++ b/foreign/swift/Sources/Iggy/Iggy.swift
@@ -21,5 +21,5 @@ public static let sdkName = "swift-sdk" /// Version of this SDK as reported to the server. Provisional until the /// release wiring lands; the release tooling reads this constant. - public static let sdkVersion = "0.1.7" + public static let sdkVersion = "0.1.8" }
diff --git a/foreign/swift/Sources/Iggy/Protocol/Batch.swift b/foreign/swift/Sources/Iggy/Protocol/Batch.swift index 2d47bd9..9e59314 100644 --- a/foreign/swift/Sources/Iggy/Protocol/Batch.swift +++ b/foreign/swift/Sources/Iggy/Protocol/Batch.swift
@@ -28,11 +28,21 @@ static let messageHeaderSize = 48 static let checksumOffset = 40 static let messageCountOffset = 48 - static let reservedOffset = 52 + /// A flags byte, then the producer fields. See `BATCH_PRODUCER_OFFSET`. + static let producerOffset = 52 + static let reservedOffset = 72 /// A message's timestamp delta is a `u32` of microseconds, so one batch /// spans at most about 71 minutes of producer clock. static let maxTimestampDelta = UInt64(UInt32.max) + /// The idempotent producer that wrote a batch. Mirrors + /// `BatchProducer` in `core/binary_protocol/src/batch.rs`. + struct Producer: Equatable { + var producerID: UInt64 + var epoch: UInt16 + var baseSequence: UInt32 + } + struct Header: Equatable { var partitionID: UInt64 var baseOffset: UInt64 @@ -42,6 +52,8 @@ var batchLength: UInt64 var batchChecksum: UInt64 var messageCount: UInt32 + /// Set only for a batch of an idempotent producer. + var producer: Producer? = nil static func decode(_ bytes: ArraySlice<UInt8>) throws -> Header { guard bytes.count >= Batch.headerSize else { @@ -64,7 +76,35 @@ originTimestamp: UInt64(littleEndianBytes: bytes[start + 24..<start + 32]), batchLength: batchLength, batchChecksum: UInt64(littleEndianBytes: bytes[start + Batch.checksumOffset..<start + Batch.checksumOffset + 8]), - messageCount: UInt32(littleEndianBytes: bytes[start + Batch.messageCountOffset..<start + Batch.messageCountOffset + 4])) + messageCount: UInt32(littleEndianBytes: bytes[start + Batch.messageCountOffset..<start + Batch.messageCountOffset + 4]), + producer: try decodeProducer(bytes)) + } + + /// The producer section of a header that `decode` already bounds-checked. + private static func decodeProducer(_ bytes: ArraySlice<UInt8>) throws -> Producer? { + let start = bytes.startIndex + Batch.producerOffset + let section = bytes[start..<bytes.startIndex + Batch.reservedOffset] + let malformed = WireError.validation("batch header producer section is malformed") + switch bytes[start] { + case 0: + guard section.allSatisfy({ $0 == 0 }) else { throw malformed } + return nil + case 1: + guard bytes[start + 1..<start + 4].allSatisfy({ $0 == 0 }), + bytes[start + 14..<start + 16].allSatisfy({ $0 == 0 }) + else { throw malformed } + let producer = Producer( + producerID: UInt64(littleEndianBytes: bytes[start + 4..<start + 12]), + epoch: UInt16(littleEndianBytes: bytes[start + 12..<start + 14]), + baseSequence: UInt32(littleEndianBytes: bytes[start + 16..<start + 20])) + guard producer.producerID <= UInt64(Int64.max), + producer.epoch <= UInt16(Int16.max), + producer.baseSequence <= UInt32(Int32.max) + else { throw malformed } + return producer + default: + throw malformed + } } func encode() -> [UInt8] { @@ -76,6 +116,16 @@ writer.write(batchLength) writer.write(batchChecksum) writer.write(messageCount) + if let producer { + writer.write(UInt8(1)) + writer.writeZeros(3) + writer.write(producer.producerID) + writer.write(producer.epoch) + writer.writeZeros(2) + writer.write(producer.baseSequence) + } else { + writer.writeZeros(Batch.reservedOffset - Batch.producerOffset) + } writer.writeZeros(Batch.headerSize - Batch.reservedOffset) return writer.bytes } @@ -89,6 +139,13 @@ writer.write(originTimestamp) writer.write(batchLength) writer.write(messageCount) + // Hashed only when present, as the Rust checksum does. + if let producer { + writer.write(UInt8(1)) + writer.write(producer.producerID) + writer.write(producer.epoch) + writer.write(producer.baseSequence) + } return writer.bytes } }
diff --git a/foreign/swift/Tests/IggyTests/Fixtures/golden.json b/foreign/swift/Tests/IggyTests/Fixtures/golden.json index b753c4c..ec2fb87 100644 --- a/foreign/swift/Tests/IggyTests/Fixtures/golden.json +++ b/foreign/swift/Tests/IggyTests/Fixtures/golden.json
@@ -717,6 +717,11 @@ "88": "lifecycle_busy", "89": "not_primary", "90": "deferred_poll_capacity_exceeded", + "91": "producer_ids_exhausted", + "92": "producer_sequence_out_of_order", + "93": "producer_epoch_fenced", + "94": "producer_unknown", + "95": "producer_partition_required", "100": "client_not_found", "101": "invalid_client_id", "206": "connection_closed", @@ -930,7 +935,7 @@ "polling.next": "050000000000000000", "polling.offset.100": "016400000000000000", "polling.timestamp.1700000000000": "020068e5cf8b010000", - "request.bind_session": "023000000973776966742d73646b05302e312e3701000000000000000000000000000000640000000000000078000000000000005a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a", + "request.bind_session": "023000000973776966742d73646b05302e312e3801000000000000000000000000000000640000000000000078000000000000005a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a", "request.change_password": "010401000000086f6c642d706173730c6e65772d706173732d313233", "request.create_consumer_group": "01040100000001040200000003677270", "request.create_partitions": "01040100000001040200000005000000", @@ -971,8 +976,8 @@ "request.get_users": "", "request.join_consumer_group": "010401000000010402000000010403000000", "request.leave_consumer_group": "010401000000010402000000020167", - "request.login_register": "023000000973776966742d73646b05302e312e375a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a0469676779046967677900000000", - "request.login_register_with_pat": "023000000973776966742d73646b05302e312e375a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a107061742d61626331323364656634353600000000", + "request.login_register": "023000000973776966742d73646b05302e312e385a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a0469676779046967677900000000", + "request.login_register_with_pat": "023000000973776966742d73646b05302e312e385a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a5a107061742d61626331323364656634353600000000", "request.logout_user": "", "request.ping": "", "request.poll_messages": "0101040100000001040a00000001041400000001050000000164000000000000003200000001", @@ -1014,6 +1019,7 @@ "response.partition_route": "07000000000000000a00000000000000030000000400000002", "response.poll_messages": "07000000660000000000000003000000000000000000000000000000000000000000000000000000070000000000000064000000000000008813000000000000e8030000000000006e01000000000000f576a2492e1328b302000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000fa7fe16d19356ec20b00000000000000000000000000000000000000000000000000000001000000000000000000000061b7f159184e9d8ac20c000000000000000000000000000000010000000a0000000c0000000100000000000000000000006202010000006b020100000076070000000000000066000000000000007017000000000000d00700000000000031010000000000006739773bc7d38608010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000009377fb7d5180f51e0d00000000000000000000000000000000000000000000000000000001000000000000000000000063", "response.poll_messages.empty": "00000000000000000000000000000000000000000000000000000000000000000000000000000000", + "response.poll_messages.producer": "07000000c800000000000000010000000000000000000000000000000000000000000000000000000700000000000000c800000000000000581b000000000000b80b00000000000031010000000000007440970f0aa5fe160100000001000000a00f00000000000003000000110000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000057452afd38db4424150000000000000000000000000000000000000000000000000000000100000000000000000000006b", "response.raw_personal_access_token": "167261772d7365637265742d746f6b656e2d76616c7565", "response.send_messages": "0200000001000000020000000300000004000000000000000100000002000000000000002a00000000000000", "response.send_messages.empty": "00000000", @@ -1026,7 +1032,7 @@ "response.user_details.no_permissions": "0100000000e090663c130600010561646d696e000000000000000000000000", "stream_key.fingerprint": "8441e38538cbfa57", "user_headers.sample": "0207000000617474656d70740b0400000003000000020800000074726163652d69640203000000616263", - "version_info": "023000000973776966742d73646b05302e312e37", + "version_info": "023000000973776966742d73646b05302e312e38", "vsr.eviction.incompatible_protocol": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab89674523010230000002300000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000e", "vsr.eviction.stale_client": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001000000000000000000000d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab89674523010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000d", "vsr.reply.committed": "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c0100000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001032547698badcfeefcdab896745230100000000000000000c0000000000000000000000000000000700000000000000800000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000",
diff --git a/foreign/swift/Tests/IggyTests/ResponseVectorTests.swift b/foreign/swift/Tests/IggyTests/ResponseVectorTests.swift index 18b62f2..d4b64bb 100644 --- a/foreign/swift/Tests/IggyTests/ResponseVectorTests.swift +++ b/foreign/swift/Tests/IggyTests/ResponseVectorTests.swift
@@ -418,6 +418,13 @@ let empty = try Responses.polledMessages(body("response.poll_messages.empty")) #expect(empty.messages.isEmpty) + // A batch of an idempotent producer carries it in the header, under the checksum. + let produced = try Responses.polledMessages(body("response.poll_messages.producer"), verifyChecksums: true) + #expect(produced.messages.map(\.id) == [21]) + #expect(produced.messages.map(\.payloadString) == ["k"]) + let producedHeader = try Batch.Header.decode(golden.bytes("response.poll_messages.producer")[40...]) + #expect(producedHeader.producer == Batch.Producer(producerID: 4_000, epoch: 3, baseSequence: 17)) + var corrupted = golden.bytes("response.poll_messages") corrupted[corrupted.count - 1] ^= 0xFF #expect(throws: IggyError.self) {
diff --git a/foreign/swift/Tools/golden-vectors/src/main.rs b/foreign/swift/Tools/golden-vectors/src/main.rs index 1390b68..c66939e 100644 --- a/foreign/swift/Tools/golden-vectors/src/main.rs +++ b/foreign/swift/Tools/golden-vectors/src/main.rs
@@ -28,7 +28,8 @@ use bytes::{BufMut, Bytes, BytesMut}; use iggy_binary_protocol::batch::{ - BATCH_HEADER_SIZE, BATCH_MESSAGE_HEADER_SIZE, BatchHeader, calculate_batch_checksum, + BATCH_HEADER_SIZE, BATCH_MESSAGE_HEADER_SIZE, BatchHeader, BatchProducer, + calculate_batch_checksum, }; use iggy_binary_protocol::codec::WireEncode; use iggy_binary_protocol::codes::*; @@ -1811,6 +1812,23 @@ } .encode(&mut empty); vectors.insert("response.poll_messages.empty".to_owned(), hex(&empty)); + + // A batch of an idempotent producer. + let mut record = batch_record(7, 200, 7_000, 3_000, &[frame(21, 0, 0, b"k", &[])]); + let mut header = BatchHeader::decode(&record).expect("a batch header"); + header.producer = Some(BatchProducer::new(4_000, 3, 17).expect("a producer")); + header.batch_checksum = calculate_batch_checksum(&header, &record[BATCH_HEADER_SIZE..]); + header.encode_into(&mut record[..BATCH_HEADER_SIZE]); + let mut produced = BytesMut::new(); + PollMessagesResponseHeader { + context: Default::default(), + partition_id: 7, + current_offset: 200, + messages_count: 1, + } + .encode(&mut produced); + produced.put_slice(&record); + vectors.insert("response.poll_messages.producer".to_owned(), hex(&produced)); } add( &mut vectors,
diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index cf9fb82..35f2b59 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md
@@ -28,7 +28,7 @@ > they answer `GROUP_AUTHORIZATION_FAILED` (30). Iggy's `metadata.partitions_max` defaults to 0 > (unlimited), so set it before exposing the gateway. See [docs/SCOPE.md](docs/SCOPE.md). > -> InitProducerId does real work too, with or without the bridge: it allocates a producer id, so a stock idempotent producer starts instead of failing at startup. +> InitProducerId does real work too. With the bridge, Iggy builds the producer id. With `IGGY_KAFKA_IDEMPOTENT_PRODUCERS=true`, each partition also writes each batch of an idempotent producer once. Without the bridge, the gateway allocates the id, so a stock idempotent producer still starts. > > Consumer group coordination is not a stub either: `FindCoordinator`, `JoinGroup`, `Heartbeat`, > `LeaveGroup`, `SyncGroup`, `DescribeGroups` and `ListGroups` are real, with real membership, @@ -73,7 +73,7 @@ | `IGGY_KAFKA_READ_TIMEOUT_SECS` | `15` | Seconds allowed to read a frame body once its length prefix arrives | | `IGGY_KAFKA_WRITE_TIMEOUT_SECS` | `10` | Seconds allowed to write a response frame | | `IGGY_KAFKA_SHUTDOWN_DRAIN_TIMEOUT_SECS` | `25` | Seconds graceful shutdown waits for in-flight connections before abandoning them. The bridge then signs its Iggy clients out, 5 s at most | -| `IGGY_KAFKA_INSTANCE_ID` | `IGGY_KAFKA_NODE_ID`, else `0` | This gateway's number among the gateways fronting one Iggy cluster. It is the high half of every producer id `InitProducerId` hands out, and Kafka requires those to be cluster-unique, so give every gateway its own value. Unset, it follows `IGGY_KAFKA_NODE_ID`. A single gateway can leave both unset. | +| `IGGY_KAFKA_INSTANCE_ID` | `IGGY_KAFKA_NODE_ID`, else `0` | This gateway's number among the gateways fronting one Iggy cluster. Without the bridge, it is the high half of every producer id `InitProducerId` hands out, so give every gateway its own value. With the bridge, Iggy builds the ids, and this value does not change them. Unset, it follows `IGGY_KAFKA_NODE_ID`. A single gateway can leave both unset. | | `IGGY_KAFKA_NODE_ID` | `1` | This gateway's broker id in Metadata, 0 to 65535. Beside each Iggy node, use that node's replica id. Required with `IGGY_KAFKA_BROKERS` | | `IGGY_KAFKA_BROKERS` | unset | Every gateway in front of the Iggy cluster, this one included, as `id@host:port` entries separated by commas. With several listeners, one `id@PROTOCOL://host:port` entry per gateway and listener. Give every gateway the same list. See [Several gateways](#several-gateways-4515) | | `IGGY_KAFKA_COORDINATOR_LOG` | `true` with `IGGY_KAFKA_BROKERS` and the bridge, else `false` | Share the consumer groups through the coordinator log in Iggy (`true` or `false`). Needs the bridge, and `manage_servers` on the bridge's Iggy user. Setup tries again for up to 60 s while Iggy does not answer. See [Several gateways](#several-gateways-4515) | @@ -262,15 +262,23 @@ ### Delivery guarantees -Delivery through this gateway is **at-least-once**, and stays at-least-once across a gateway -restart. Transactions are not supported, and will not be. An idempotent Kafka producer is given -a producer id so that it starts, but its retries are not deduplicated: Produce uses the producer -id only to pick the producer's Iggy session, and ignores the epoch and sequence, so a retry after a -network timeout writes the record twice, and both copies reach the stream with their own offsets. +With the bridge and `IGGY_KAFKA_IDEMPOTENT_PRODUCERS=true`, an idempotent Kafka producer writes +each batch once. The Java producer is idempotent by default. Each batch carries the producer id, the +epoch and the sequence of its first record. The Iggy partition writes a batch of one producer once. +A resend answers the offset of the first write. That holds through any gateway, and across a +gateway restart. -Iggy deduplicates writes on its own partition plane, and that does not close this gap, because it -guards the hop from the gateway to Iggy rather than the hop from the producer to the gateway. -[docs/IDEMPOTENCE.md](docs/IDEMPOTENCE.md) has the detail and what closing it needs. +The switch is off by default. Released builds of the gateway, of the Rust SDK and of every client +that embeds it cannot read a batch that carries a producer, and older Iggy nodes cannot store it. +Upgrade every Iggy node, every gateway and every consumer of the topics first, then turn it on. +While it is off, an idempotent producer gets at-least-once delivery. So does a producer that keeps +an id from an older gateway, until it restarts. + +A producer without idempotence gets at-least-once delivery. A retry that follows a network timeout +writes the record twice. The partition keeps the last 5 batches of each producer, and at most 4096 +producers. It forgets a producer that wrote nothing for a day. Past those limits a resend can +write a batch twice, as in Kafka. +[docs/IDEMPOTENCE.md](docs/IDEMPOTENCE.md) has the detail. Transactions are **not supported**, and will not be. AddPartitionsToTxn (24), AddOffsetsToTxn (25), EndTxn (26) and TxnOffsetCommit (28) are never advertised, so a conforming client never @@ -388,7 +396,8 @@ nodes. The bridge itself needs that address at startup. To start while that node is down, point it at a load balancer in front of the cluster. A DNS name with one record per node can bind a client's first link to the wrong node (not tested). -- An idempotent producer can still write a batch twice when it moves to another gateway. +- With `IGGY_KAFKA_IDEMPOTENT_PRODUCERS=true` on every gateway, an idempotent producer that moves + to another gateway still writes each batch once. - With TLS to Iggy, the SDK dials each node at the address the roster advertises, so each node's certificate must name that address, or set `IGGY_KAFKA_IGGY_TLS_DOMAIN`. @@ -572,14 +581,15 @@ | ----- | ------- | | Partition | Index as sent. Both count from 0. | | Base offset | From the send confirmation. `-1` if none. | -| `log_start_offset` | Always `-1`. | +| `log_start_offset` | `-1`, but `0` with 59. While it is `-1`, a Java producer retries a 59 unchanged. | | `acks` | `0`, `1`, `-1` write the same. Other values: 21, before any per-partition check. | | `acks=0` | Writes, answers nothing. Any failed partition closes the connection. | | Topics | Creates none. Missing: 3. Bad name: 17. | | `timeout_ms` | Honored, max 20 s. The Iggy send stops by then too. Past it: 7. A send that never went out: 6. | | Compression | gzip, snappy, lz4. zstd from v7, else 76. | -| Producer id | Picks the producer's Iggy session. See [Concurrency ceiling](#concurrency-ceiling). | -| Producer epoch, sequence | Ignored, so a retry writes twice. | +| Producer id | Picks the producer's Iggy session. See [Concurrency ceiling](#concurrency-ceiling). Goes to Iggy with the batch only with `IGGY_KAFKA_IDEMPOTENT_PRODUCERS=true`, and only an id that a gateway got from Iggy. | +| Producer epoch, sequence | With `IGGY_KAFKA_IDEMPOTENT_PRODUCERS=true`, go to Iggy with the batch. A resend answers the first offset. A gap: 45. An older epoch: 47. A producer that the partition does not hold, above sequence 0: 59. Off (the default): not checked, so a resend can write twice. | +| Producer id with a negative epoch or sequence | 87. | | Timestamps over about 71 min apart | Clamped into one send. `kafka.ts` keeps the real one. | | Several batches in one partition | 87, as Kafka. | | Bytes after the batch | 87. | @@ -693,6 +703,7 @@ | `IGGY_KAFKA_IGGY_MAX_MESSAGE_SIZE` | `64MiB` | Iggy's `message_bus.max_message_size`. Set both together. Larger partitions answer 10 | | `IGGY_KAFKA_PRODUCE_SESSIONS` | `256` | Producers that get an Iggy client of their own. The rest share one. `0`: all share one | | `IGGY_KAFKA_PRODUCE_SESSION_IDLE_SECS` | `600` | An unused Produce client closes after this long | +| `IGGY_KAFKA_IDEMPOTENT_PRODUCERS` | `false` | `true` passes the producer of an idempotent batch to Iggy, so each batch lands once. Turn it on after every Iggy node, gateway and consumer of the topics is upgraded. See [docs/IDEMPOTENCE.md](docs/IDEMPOTENCE.md) | The initial connect retries a fixed, bounded number of times (`RECONNECTION_RETRIES = 3`, not the Iggy SDK client's own default of unlimited retries, one dial per second, forever), and the whole @@ -837,9 +848,9 @@ - Each producer sends on an `IggyClient` of its own, one request at a time, so its batches land in order. Different producers send at once. -- A producer is its producer id, if this gateway gave that id out since it started, plus its user - when SASL is on. Every other producer is its Kafka connection: no id, an id from before a - gateway restart, or an id from another gateway. +- A producer is its producer id, if this gateway gave that id out lately, plus its user when SASL + is on. Every other producer is its Kafka connection: no id, an id from before a gateway + restart, or an id from another gateway. - Up to `IGGY_KAFKA_PRODUCE_SESSIONS` (256) of these clients are open. A full pool closes its least recently used free client for a new producer, if that client sat unused for 30 s. Otherwise the producer shares 1 more client, one request at a time, and moves to its own at a @@ -871,8 +882,11 @@ `Admin.createPartitions` report a `TimeoutException`, and the caller must retry. - OffsetCommit and OffsetFetch cost one Iggy call per partition, spread over the offset slots. In a cluster, an offset commit client can also open one connection per node for stores. -- Order: set `max.in.flight.requests.per.connection=1`, or `retries=0`. Otherwise a retried batch - can land after later ones. +- Order: with `IGGY_KAFKA_IDEMPOTENT_PRODUCERS=true`, an idempotent producer keeps its order. The + partition refuses a batch that does not follow with 45, and the producer sends it again. With + the switch off (the default), every producer, the idempotent Java default too, needs + `max.in.flight.requests.per.connection=1` or `retries=0`. Otherwise a retried batch can land + after later ones. - Order after a 7: the send can still run. A producer with a producer id waits behind it on any connection. A producer that is its connection does not once it reconnects, so its retry or its next batch can land first. To keep that order use an idempotent producer, the Java default, or
diff --git a/gateways/kafka/docs/BRIDGE_MAPPING.md b/gateways/kafka/docs/BRIDGE_MAPPING.md index 147b801..b3d1b89 100644 --- a/gateways/kafka/docs/BRIDGE_MAPPING.md +++ b/gateways/kafka/docs/BRIDGE_MAPPING.md
@@ -203,22 +203,21 @@ ## Batch-level fields -Per-record storage drops what the Kafka record batch header carries: producer id, producer -epoch, base sequence, the transactional flag, compression and the batch CRC. Fetch synthesizes -a batch with producer id `-1`, epoch `-1`, base sequence `-1`, no compression, `CreateTime` +Per-record storage drops the transactional flag, the compression and the batch CRC of the Kafka +record batch header. With `IGGY_KAFKA_IDEMPOTENT_PRODUCERS=true`, the producer id, the epoch and +the base sequence of an idempotent batch go to the Iggy batch header, and the partition keeps them +as producer state. Otherwise the gateway drops them. Fetch synthesizes a +batch with producer id `-1`, epoch `-1`, base sequence `-1`, no compression, `CreateTime` timestamps, and a recomputed CRC32C. Two consequences worth stating before they surprise someone: -- Idempotent-producer deduplication cannot be reconstructed from stored data later. If - [#3545](https://github.com/apache/iggy/issues/3545) ever grows past a stub, producer id, - epoch and sequence need their own tracking. +- The partition writes each batch of an idempotent producer once, from the producer state that + it keeps. See [`IDEMPOTENCE.md`](IDEMPOTENCE.md). - The bytes a consumer receives are not the bytes the producer sent, so anything comparing batches byte for byte across the gateway will differ. -Whether producer id `-1` is what Fetch actually sends depends on the InitProducerId decision in -[`IDEMPOTENCE.md`](IDEMPOTENCE.md). Allocating producer ids does not change what is stored, only -what Produce accepts, so this section holds under either answer. +Fetch sends producer id `-1` whatever producer wrote the batch. A consumer does not need it. Produce refuses a control batch and a transactional batch. A stored message carries neither flag. A control record admitted here therefore reaches consumers as ordinary application data, and a
diff --git a/gateways/kafka/docs/IDEMPOTENCE.md b/gateways/kafka/docs/IDEMPOTENCE.md index 543a230..738d89b 100644 --- a/gateways/kafka/docs/IDEMPOTENCE.md +++ b/gateways/kafka/docs/IDEMPOTENCE.md
@@ -1,6 +1,7 @@ # InitProducerId and idempotent producers -Status: implemented, [#3545](https://github.com/apache/iggy/issues/3545). Gates the Phase 1 +Status: implemented, [#3545](https://github.com/apache/iggy/issues/3545) for InitProducerId and +[#4513](https://github.com/apache/iggy/issues/4513) for writing each batch once. Gates the Phase 1 end-to-end test ([#3539](https://github.com/apache/iggy/issues/3539)), which drives `kafka-console-producer.sh`. @@ -18,93 +19,31 @@ The gateway's stated purpose is that a Kafka user swaps the broker and changes no application code. A broker that the default producer cannot start against does not meet it. -## What Iggy already deduplicates +## How a resend is written once -Iggy deduplicates on the partition plane, and did so before this gateway existed. The key is -`(client_id, user_id, request)`. `client_id` is a `u128` the client generates, and `request` is a -monotonic counter on the client's session. Each partition group holds a per-client watermark with -a 128-bit `committed_window` under it. A request below the watermark with its bit clear is a -reordered arrival still to execute. A request below the window is refused with `RequestTooOld`, -because its outcome is unknown. -`partition.dedup_clients_max` caps the distinct clients one partition tracks, at 4096 by default -and 65536 at the ceiling. +An Iggy partition keeps the state of each idempotent producer that writes to it, as a Kafka log +does. The core SDK README has the rules, under "Idempotent producers". In short: -The shape is closer to Kafka's than it looks. A producer id with a per-partition sequence maps -onto a session id with a request id. Under that mapping a request id is `base_sequence + 1` (see -below), so it advances by each batch's record count and the window is 128 records deep, not 128 -batches. Four batches of 32 records fill it, one fewer than the five a Kafka producer keeps in -flight. +- Each batch carries the producer id, the epoch and the sequence of its first record. +- A partition keeps the last 5 batches of each producer, and at most 4096 producers. +- A partition drops a producer that wrote nothing for a day. +- Before the partition primary assigns offsets, it compares each batch with that state and with + its uncommitted writes. -What breaks the mapping is the hop each side protects. Iggy's covers gateway to Iggy. Kafka's -covers producer to gateway. A retrying producer sends a fresh Produce request, and the gateway -turns it into a fresh Iggy send carrying a fresh request id. Nothing recognises the replay. +With `IGGY_KAFKA_IDEMPOTENT_PRODUCERS=true`, Produce passes the three fields of a Kafka batch to +Iggy, the id without the gateway's mark. So every resend finds the first write: through any +gateway, on any Iggy session, and across a gateway restart. -## Closing that gap +The switch is off by default. Released gateways, and released consumers built on the Rust SDK, +refuse to read a batch that carries a producer, and older Iggy nodes cannot store it. Upgrade every +Iggy node, every gateway and every consumer of the topics, then turn the switch on for every +gateway. While it is off, Produce drops the three fields, and an idempotent producer gets +at-least-once delivery. -Both halves of Iggy's key have to come from the Kafka request rather than from the SDK. - -The client id is not a per-request field. `dispatch_partition_request` takes it from the -transport's bound session, and the header transmute at -`core/server/src/dispatch/partition.rs:247` overwrites whatever the SDK sent. Deriving it from a -Kafka producer id therefore needs one bound session per producer. - -The request id is different. The transmute copies the header and replaces only `group`, `client`, -`session` and `user_id`, so a caller-supplied request id survives to the `ClientTable`. - -One session per producer is a connection pool keyed by producer id. The gateway has that pool, for -throughput: see the README's "Concurrency ceiling". Retries are still not deduplicated, because two -SDK seams are missing: - -- a caller-chosen client id at build time. `ConsensusSession::with_client_id` is public, but both - construction sites build with `ConsensusSession::new()` - (`core/sdk/src/tcp/tcp_client.rs:378` and `:583`) -- a caller-supplied request id. `send_raw_with_response` already takes - `preencoded: Option<RequestHeader>` for transient replay, and it is private - -None of this lands in this phase. Produce and Fetch do not need it, delivery is at-least-once -before and after, and the additions belong to whoever owns that SDK surface. - -### Restart - -A session registers with a fresh random client id per gateway process. A restarted gateway -therefore cannot collide with a watermark that outlived it, and a retry that spans a restart is -not deduplicated. That is still at-least-once, which the README states plainly. - -The alternative is a stable client id derived from the producer id. It deduplicates across a -restart, and it is unsafe without also persisting the last request id per producer. Request ids -restarting at 1 under a live watermark land below it. Inside the window, an id whose bit is set is -absorbed as a duplicate and its fresh write is discarded, while an id whose bit is clear executes. -An id below the window is refused with `RequestTooOld`. Silent loss is worse than duplicate -delivery, so the fresh random id wins. - -### Confirmed and not - -The hop mismatch, one session per producer, and the restart choice are agreed in the maintainer -thread on [#3545](https://github.com/apache/iggy/issues/3545) and in Discord. - -Two further readings are ours and are not confirmed yet. One session per producer is enough, -rather than one per producer and partition, because the `ClientTable` is per partition group. The -same request number on two partitions is two entries, so `request_id = base_sequence + 1` stays -monotonic inside each. And the gateway has to preserve per-partition ordering per producer, or -records land out of order. A lower sequence arriving after a higher one lands below the watermark: -inside the window, with its bit clear, it executes after its successor, and below the window it is -refused with `RequestTooOld`. - -## Options - -| Option | Cost | What a stock producer does | -| -------- | ------ | ---------------------------- | -| Stub with `UNSUPPORTED_VERSION` | none | fails at startup unless the user sets `enable.idempotence=false` | -| Allocate only | about a day | works untouched, at-least-once delivery | -| Allocate and pool | weeks, blocked on the SDK | works untouched, retries absorbed on both hops | - -## Decision - -Allocate only, and defer deduplication. - -Rejecting the stock producer to avoid that work trades away the one requirement the maintainers -named. It buys a guarantee nobody is asking for yet. Allocating costs about a day, leaves -delivery where it already is, and blocks nothing deduplication later needs. +Iggy's client table does not do this job. It guards the hop from the gateway to Iggy. It keys on a +request number that the Iggy client picks. A Kafka retry is a new Produce request. The gateway +sends it as a new Iggy request, with a new number. The producer state keys on the numbers that +the Kafka producer picks, so every resend carries the same ones. ## Behavior @@ -113,43 +52,31 @@ the schemas, request v0 to v5 and response v0 to v6, flexible from v2; the gateway serves v0 to v5. The handler is `src/protocol/handlers/init_producer_id.rs`. -InitProducerId with no `transactional_id`: +InitProducerId with no `transactional_id`, with the bridge: -- allocate the next producer id, return it with epoch 0 and error code 0 -- build the id from an instance number in the high 16 bits and a counter in the low 47 bits, - leaving bit 63 clear. `producer_id` is an `i64` and `-1` means no producer id, so the value has - to stay non-negative. That leaves room for 65536 instances holding 140 trillion ids each +- ask Iggy for the next producer id, and return it with epoch 0 and error code 0. +- Iggy builds the id from the session number of the bridge's id client and an index. No other + client of the cluster gets that id, on any gateway and across restarts. +- the gateway marks the id: it sets bit 46 and moves the bits from 46 up one place. Ids from older + gateways hold a clock in milliseconds in their low 47 bits, so bit 46 stays clear there until + the year 4199. +- a request that names its current id and epoch (KIP-360, v3 and later) also gets a new id. Kafka + does the same for a producer without a transactional id. +- if Iggy cannot give an id, answer `COORDINATOR_LOAD_IN_PROGRESS` (14), which the Java producer + retries. Not 15 or 16: those send it to look for a transaction coordinator, which this gateway + has none of. +- one Iggy session hands out 2^20 ids. After that, the id client signs in again and gets a new + range. -The instance number comes from configuration (`IGGY_KAFKA_INSTANCE_ID`, else -`IGGY_KAFKA_NODE_ID`, else 0), not from a -draw at startup. A random 16-bit number collides with even odds at around 300 instances. That is -a birthday collision, not a remote one. +For a gap or a lost producer, an idempotent Java producer moves to the next epoch by itself. A new +epoch starts its sequences at 0. The producer asks for a new id only at epoch 32767. -The id is a pool key, not a dedup identity. Under the design above, the dedup identity is the -session's own random client id, minted at register. The producer id only decides which connection -serves a producer. Kafka still requires it to be unique across the cluster, which is what the -instance number buys. It also has to stay unique across a restart once dedup lands or -Produce persists, because `producer_epoch` is always 0, so a replayed id is a replayed -`(producer_id, producer_epoch)` pair a live producer may still hold. The counter therefore starts -at the wall clock in milliseconds rather than at 0. A restarted gateway starts above every id its -previous run handed out unless that run averaged more than one allocation per millisecond of its -uptime, or the clock stepped back across the restart. A clock that reads before the Unix epoch -seeds 0 and replays from the bottom. 2^47 milliseconds leaves the counter space thousands of years -from running out. - -Nothing is persisted for this, so the clock seed narrows the replay window without closing it. -Produce reads the id as a pool key, and only an id this run handed out: any other id keys by -connection. A replayed pair therefore makes two producers of one user share a session and send one -at a time, and nothing more. It stops being harmless once deduplication reads the id. That work -must first either persist a high-water mark and seed the counter above it, or bump -`producer_epoch` on every gateway start. Either one makes a replayed -`(producer_id, producer_epoch)` pair impossible rather than unlikely. - -`IGGY_KAFKA_INSTANCE_ID` follows `IGGY_KAFKA_NODE_ID` when only that one is set, and is 0 -otherwise, so a single gateway needs no configuration. Two gateways -left on the default draw from the same 47-bit space and can hand out identical ids, on the first -allocation if they start in the same millisecond. Nothing in the cluster detects that, so a gateway started without the -variable logs a warning, and the running value is in the startup log line. +Without the bridge, a local allocator gives the id. It puts an instance number in the high 16 bits +and a counter in the low 47 bits. Bit 63 stays clear, because `producer_id` is an `i64`, and `-1` +means no producer. The instance number comes from configuration (`IGGY_KAFKA_INSTANCE_ID`, else +`IGGY_KAFKA_NODE_ID`, else 0). The counter starts at the wall clock in milliseconds. So a restarted +gateway starts above the ids of its previous run. Without the bridge nothing is written, so these +ids only let a stock producer start. An empty `transactional_id` reads as absent. A wire null decodes to `None`, but `kafka-protocol`'s own `Default` is `Some("")`, and a producer that is idempotent-only names no @@ -180,19 +107,30 @@ Produce: -- accept `producer_id`, `producer_epoch` and `base_sequence` on the request. The id picks the - producer's session when this run handed it out. The epoch and the sequence are ignored -- never answer `OUT_OF_ORDER_SEQUENCE_NUMBER` (45) or `DUPLICATE_SEQUENCE_NUMBER` (46) +- pass `producer_id`, `producer_epoch` and `base_sequence` of an idempotent batch to Iggy, when + `IGGY_KAFKA_IDEMPOTENT_PRODUCERS` is `true`. The id goes without its mark. A batch with a + producer id but a negative epoch or sequence gets `INVALID_RECORD` (87). +- write a batch whose id has no mark as a plain batch. A Java producer keeps its id across + reconnects, and older gateways on the same instance number gave two producers the same id. +- a resend of a batch that the partition holds answers the offset of its first write. +- answer `OUT_OF_ORDER_SEQUENCE_NUMBER` (45) for a gap, as Kafka does. The Java producer retries + a 45. For a real gap, it moves to the next epoch. +- answer 45 also for a resend older than the 5 batches that the partition keeps. +- answer `INVALID_PRODUCER_EPOCH` (47) for a batch of an older epoch. +- answer `UNKNOWN_PRODUCER_ID` (59) for a batch above sequence 0 from a producer that the partition + does not hold. That partition response carries `log_start_offset` 0. With a log start of `-1`, + the Java producer retries a 59 unchanged up to its delivery timeout. +- a duplicate that is still uncommitted gets Iggy's `TransientNotCommitted`, because the first + copy can still commit. The SDK retries it up to the deadline. Past it, the client gets a + retriable 7. +- never answer `DUPLICATE_SEQUENCE_NUMBER` (46). A Kafka broker sends 45 for an old duplicate too. +- an id that this gateway gave out lately also picks the producer's Iggy session. - reject a non-empty `transactional_id` (v3+) with `UNSUPPORTED_VERSION` (35), at the partition level, keeping the connection open. Under `acks=0` there is no response to carry 35, so the gateway closes the connection instead, as a Kafka broker does on any `acks=0` produce error. - The check runs before the `acks=0` branch, so no write path can see a transactional batch + The check runs before the `acks=0` branch, so no write path can see a transactional batch. -Those first two codes stay unsent even once dedup lands. The watermark accepts any request -above it without noticing a gap, so a gap cannot be told apart from ordinary traffic. Sending -either code claims a detection the gateway does not have. - -The third is where `maybeTransitionToErrorState` is exact. `Sender.completeBatch` -> +The transactional refusal is where `maybeTransitionToErrorState` is exact. `Sender.completeBatch` -> `canRetry` false -> `failBatch` -> `handleFailedBatch` -> `maybeTransitionToErrorState`, whose explicit fatal set holds ClusterAuthorization, TransactionalIdAuthorization, ProducerFenced, UnsupportedVersion and InvalidPidMapping. `INVALID_TXN_STATE` (48) is explicitly rewritten to @@ -233,29 +171,35 @@ ## What this does not give you -A producer that holds an id believes its retries are deduplicated. They are not. A retry after a -network timeout writes the record twice, and both copies reach the stream with their own offsets. - -Iggy's own deduplication does not help, because it guards the other hop. Delivery through the -gateway is at-least-once until dedup lands, and at-least-once across a gateway restart after that. - -The README states that limitation under "Delivery guarantees", so a user does not have to infer -it from the presence of key 22. +- A producer without idempotence still gets at-least-once delivery. A retry that follows a + network timeout writes the record twice. +- A producer that keeps an id from an older gateway gets at-least-once delivery until it restarts. +- A partition forgets a producer that wrote nothing for a day. Past 4096 producers, it also + forgets the one idle the longest. A resend of an old batch of a forgotten producer can write it + twice, as in Kafka. +- A resend older than the 5 batches that the partition keeps gets 45, as in Kafka. +- Released builds refuse a polled batch with a producer section: the gateway, the Rust SDK and + every client that embeds it (the Python, C++ and PHP SDKs, the CLI and the connectors runtime), + and the Swift SDK. Before you set `IGGY_KAFKA_IDEMPOTENT_PRODUCERS=true`, upgrade every Iggy + node, every gateway and every consumer of the topics. +- Exactly-once delivery needs topics with `persisted` durability. With `replicated`, a quorum of + nodes that crash at the same time can lose acknowledged batches. +- Transactions are not supported. ## Resolution -The open question was allocate only, as above, against stub and document -`enable.idempotence=false`. Allocate only was taken, which this document named as the default -outcome, and is what shipped. +InitProducerId first only allocated ids, and delivery stayed at-least-once. The producer state in +Iggy partitions ([#4513](https://github.com/apache/iggy/issues/4513)) now writes each batch of an +idempotent producer once. ## References - Record mapping: [`BRIDGE_MAPPING.md`](BRIDGE_MAPPING.md), batch-level fields - Scope and phases: [`SCOPE.md`](SCOPE.md) - Version firewall: `src/protocol/api.rs`, `SUPPORTED_RANGES` -- Dedup key and window: `core/consensus/src/client_table.rs` -- Session identity: `core/sdk/src/session.rs`, `core/sdk/src/tcp/tcp_client.rs` -- Header rewrite: `core/server/src/dispatch/partition.rs` +- Producer state of a partition: `core/partitions/src/producer_state.rs` +- Producer ids from the session number: `core/sdk/src/session_owner/producer_ids.rs` +- The gateway's id client: `src/bridge/iggy_bridge/producer_ids.rs` - Produce fatal path: `TransactionManager.maybeTransitionToErrorState`, apache/kafka trunk - InitProducerId fatal path: `TransactionManager.InitProducerIdHandler.handleResponse`, same file - librdkafka's fatal set: `rd_kafka_idemp_check_error`, `src/rdkafka_idempotence.c`
diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md index ea193b5..092cdc9 100644 --- a/gateways/kafka/docs/SCOPE.md +++ b/gateways/kafka/docs/SCOPE.md
@@ -67,7 +67,7 @@ | 14 | SyncGroup | 0 | 5 | 0, 1, 2, 3, 4, 5 | Relays the leader's assignment blobs; flexible encoding at v4+ | | 15 | DescribeGroups | 0 | 6 | 0, 1, 2, 3, 4, 5, 6 | Members, assignment and state of groups on this coordinator. A group that is not there but holds offsets in Iggy is `Empty`. Any other missing group is `Dead`, and v6 also returns `GROUP_ID_NOT_FOUND` (69). Flexible encoding at v5+ | | 16 | ListGroups | 0 | 5 | 0, 1, 2, 3, 4, 5 | Groups on this coordinator, plus the groups that hold offsets in Iggy as `Empty` (see [`CONSUMER_GROUPS.md`](CONSUMER_GROUPS.md)). v4+ state filter and v5 type filter. Flexible encoding at v3+ | -| 22 | InitProducerId | 0 | 5 | 0, 1, 2, 3, 4, 5 | Allocate a producer id (epoch 0); a `transactional_id` gets `UNSUPPORTED_VERSION` (35); flexible encoding at v2+ | +| 22 | InitProducerId | 0 | 5 | 0, 1, 2, 3, 4, 5 | A producer id at epoch 0, from Iggy with a bridge. A `transactional_id` gets `UNSUPPORTED_VERSION` (35). Flexible encoding at v2+ | | 32 | DescribeConfigs | 1 | 4 | 1, 2, 3, 4 | Topic resources only. Returns `retention.ms` and leaves out `cleanup.policy`. Flexible encoding at v4. See [`CONFIGS.md`](CONFIGS.md) | | 33 | AlterConfigs | 0 | 2 | 0, 1, 2 | Persists `retention.ms` only. Flexible encoding at v2. See [`CONFIGS.md`](CONFIGS.md) | | 42 | DeleteGroups | 0 | 2 | 0, 1, 2 | With a bridge: deletes a group without members, with its offsets in Iggy. While it has members: `NON_EMPTY_GROUP` (68). When it does not exist: `GROUP_ID_NOT_FOUND` (69). Flexible encoding at v2. v3 adds an error message that `kafka-protocol` cannot encode yet | @@ -529,7 +529,8 @@ - [x] InitProducerId (22) allocates a producer id so a stock idempotent producer starts; a transactional request is refused -- [ ] Deduplicate an idempotent producer's retries. Delivery stays at-least-once until then +- [x] Write each batch of an idempotent producer once, with `IGGY_KAFKA_IDEMPOTENT_PRODUCERS=true` + ([#4513](https://github.com/apache/iggy/issues/4513)). Iggy partitions keep the producer state Authentication design ([#3549](https://github.com/apache/iggy/issues/3549)): [`AUTHENTICATION.md`](AUTHENTICATION.md).
diff --git a/gateways/kafka/src/bridge/config.rs b/gateways/kafka/src/bridge/config.rs index 7334c0f..221bb45 100644 --- a/gateways/kafka/src/bridge/config.rs +++ b/gateways/kafka/src/bridge/config.rs
@@ -133,6 +133,10 @@ /// Transport security of every client the bridge opens. pub tls: IggyTls, pub produce_sessions: ProduceSessionLimits, + /// Pass the producer of an idempotent Kafka batch to Iggy, so the partition writes the batch + /// once. Off by default, because released consumers cannot read such batches. When off, a + /// retry of an idempotent producer can write its records twice. + pub idempotent_producers: bool, /// The nodes of the last roster the bridge read, which each new client dials after `address`. /// No variable sets it: the bridge fills it, and every clone of this config shares it. pub nodes: KnownNodes, @@ -159,6 +163,7 @@ "IGGY_KAFKA_IGGY_TLS_CA_FILE", "IGGY_KAFKA_PRODUCE_SESSIONS", "IGGY_KAFKA_PRODUCE_SESSION_IDLE_SECS", + "IGGY_KAFKA_IDEMPOTENT_PRODUCERS", ]; /// Builds config from `IGGY_KAFKA_*` env vars, defaulting to the Iggy server's own @@ -185,7 +190,8 @@ /// `IGGY_KAFKA_IGGY_USERNAME` is set but empty, or if `IGGY_KAFKA_TOPIC_MAP_PATH` is set but /// the file is missing, fails to parse, or fails validation (e.g. a malformed mapping key or /// an empty `default_stream`), or if `IGGY_KAFKA_IGGY_MAX_MESSAGE_SIZE`, - /// `IGGY_KAFKA_PRODUCE_SESSIONS` or `IGGY_KAFKA_PRODUCE_SESSION_IDLE_SECS` does not parse. + /// `IGGY_KAFKA_PRODUCE_SESSIONS`, `IGGY_KAFKA_PRODUCE_SESSION_IDLE_SECS` or + /// `IGGY_KAFKA_IDEMPOTENT_PRODUCERS` does not parse. pub fn from_env() -> Result<Self, BridgeError> { let address = std::env::var("IGGY_KAFKA_IGGY_ADDR").unwrap_or_else(|_| DEFAULT_IGGY_ADDR.to_string()); @@ -274,6 +280,12 @@ })?; } + let idempotent_producers = match std::env::var("IGGY_KAFKA_IDEMPOTENT_PRODUCERS") { + Ok(raw) => parse_bool("IGGY_KAFKA_IDEMPOTENT_PRODUCERS", &raw) + .map_err(BridgeError::InvalidConfig)?, + Err(_) => false, + }; + Ok(Self { address, username, @@ -282,6 +294,7 @@ max_message_size, tls, produce_sessions, + idempotent_producers, nodes: KnownNodes::default(), }) } @@ -318,6 +331,7 @@ max_message_size: DEFAULT_MAX_MESSAGE_SIZE, tls: IggyTls::default(), produce_sessions: ProduceSessionLimits::default(), + idempotent_producers: false, nodes: KnownNodes::default(), } } @@ -445,6 +459,33 @@ #[test] #[serial] + fn from_env_reads_the_idempotent_producers_switch() { + // Safety: process-wide, not per-variable - see the note on + // from_env_uses_documented_defaults_when_only_password_is_set above. + unsafe { + std::env::set_var("IGGY_KAFKA_IGGY_PASSWORD", "iggy"); + } + let unset = IggyBridgeConfig::from_env(); + unsafe { + std::env::set_var("IGGY_KAFKA_IDEMPOTENT_PRODUCERS", "true"); + } + let on = IggyBridgeConfig::from_env(); + unsafe { + std::env::set_var("IGGY_KAFKA_IDEMPOTENT_PRODUCERS", "yes"); + } + let garbage = IggyBridgeConfig::from_env(); + unsafe { + std::env::remove_var("IGGY_KAFKA_IGGY_PASSWORD"); + std::env::remove_var("IGGY_KAFKA_IDEMPOTENT_PRODUCERS"); + } + + assert!(!unset.unwrap().idempotent_producers, "off by default"); + assert!(on.unwrap().idempotent_producers); + assert!(matches!(garbage, Err(BridgeError::InvalidConfig(_)))); + } + + #[test] + #[serial] fn from_env_rejects_missing_password() { // Safety: process-wide, not per-variable - see the note on // from_env_uses_documented_defaults_when_only_password_is_set above.
diff --git a/gateways/kafka/src/bridge/error.rs b/gateways/kafka/src/bridge/error.rs index 8c312cb..1a6fa5a 100644 --- a/gateways/kafka/src/bridge/error.rs +++ b/gateways/kafka/src/bridge/error.rs
@@ -21,9 +21,10 @@ use crate::bridge::iggy_bridge::{ClusterIdCall, CoordinatorLogCall}; use crate::cluster_id::InvalidClusterId; use crate::protocol::api::{ - ERROR_COORDINATOR_LOAD_IN_PROGRESS, ERROR_INVALID_PARTITIONS, ERROR_INVALID_REQUEST, - ERROR_INVALID_TOPIC_EXCEPTION, ERROR_NOT_LEADER_OR_FOLLOWER, ERROR_POLICY_VIOLATION, - ERROR_REQUEST_TIMED_OUT, ERROR_TOPIC_ALREADY_EXISTS, ERROR_TOPIC_AUTHORIZATION_FAILED, + ERROR_COORDINATOR_LOAD_IN_PROGRESS, ERROR_INVALID_PARTITIONS, ERROR_INVALID_PRODUCER_EPOCH, + ERROR_INVALID_REQUEST, ERROR_INVALID_TOPIC_EXCEPTION, ERROR_NOT_LEADER_OR_FOLLOWER, + ERROR_OUT_OF_ORDER_SEQUENCE_NUMBER, ERROR_POLICY_VIOLATION, ERROR_REQUEST_TIMED_OUT, + ERROR_TOPIC_ALREADY_EXISTS, ERROR_TOPIC_AUTHORIZATION_FAILED, ERROR_UNKNOWN_PRODUCER_ID, ERROR_UNKNOWN_SERVER_ERROR, ERROR_UNKNOWN_TOPIC_OR_PARTITION, }; @@ -320,6 +321,10 @@ // The outcome was forgotten; a retry after client eviction could duplicate the write. // UNKNOWN_SERVER_ERROR is non-retryable in the Kafka protocol. IggyError::RequestTooOld => ERROR_UNKNOWN_SERVER_ERROR, + // The partition refused an idempotent batch and wrote nothing, as Kafka's log does. + IggyError::ProducerSequenceOutOfOrder => ERROR_OUT_OF_ORDER_SEQUENCE_NUMBER, + IggyError::ProducerEpochFenced => ERROR_INVALID_PRODUCER_EPOCH, + IggyError::ProducerUnknown => ERROR_UNKNOWN_PRODUCER_ID, // Not `ERROR_INVALID_PARTITIONS` (37): that code's own text, per `kafka-protocol`'s // table, is "Number of partitions is below 1" - the opposite condition from "too many" // (Iggy's server-side cap, above 1000). Reusing 37 for both directions would return a @@ -718,6 +723,15 @@ ), (ERROR_INVALID_PARTITIONS, ResponseError::InvalidPartitions), (ERROR_INVALID_REQUEST, ResponseError::InvalidRequest), + ( + ERROR_OUT_OF_ORDER_SEQUENCE_NUMBER, + ResponseError::OutOfOrderSequenceNumber, + ), + ( + ERROR_INVALID_PRODUCER_EPOCH, + ResponseError::InvalidProducerEpoch, + ), + (ERROR_UNKNOWN_PRODUCER_ID, ResponseError::UnknownProducerId), ] { assert_eq!(ours, theirs.code()); }
diff --git a/gateways/kafka/src/bridge/iggy_bridge/mod.rs b/gateways/kafka/src/bridge/iggy_bridge/mod.rs index 757f7b3..5995993 100644 --- a/gateways/kafka/src/bridge/iggy_bridge/mod.rs +++ b/gateways/kafka/src/bridge/iggy_bridge/mod.rs
@@ -44,6 +44,7 @@ mod group_offsets; mod offsets; mod produce; +mod producer_ids; mod topics; pub use change_log::{DeletedTopic, PermissionBoard, TopicShape, TopicShapes, UserAccess}; @@ -61,7 +62,8 @@ pub use group_offsets::{OFFSET_GROUP_PREFIX, OffsetCalls}; pub(crate) use offsets::{LocatedRecord, RecordLookup}; use produce::ProducePool; -pub use produce::{ProduceTurn, Producer}; +pub use produce::{PartitionBatch, ProduceTurn, Producer}; +pub use producer_ids::{iggy_producer_id, kafka_producer_id}; pub use topics::{ DEFAULT_TOPIC_PARTITIONS, KafkaTopicMetadata, MAX_TOPIC_PARTITIONS, StreamTopicCache, TopicCreationOutcome, TopicLoad, @@ -195,6 +197,10 @@ permissions: Arc<PermissionBoard>, /// Gets each topic that the change log shows deleted. topic_deletions: broadcast::Sender<DeletedTopic>, + /// The client whose session hands out producer ids. See `next_producer_id`. + id_client: LazyClient, + /// The producer ids this gateway gave out, which Produce may key a session on. + issued_producer_ids: producer_ids::IssuedProducerIds, } /// The task that follows the change log. Dropping it stops the task. @@ -280,6 +286,8 @@ topic_shapes: Arc::new(watch::Sender::new(Arc::default())), permissions: Arc::default(), topic_deletions: broadcast::Sender::new(TOPIC_DELETIONS_CAPACITY), + id_client: LazyClient::default(), + issued_producer_ids: producer_ids::IssuedProducerIds::default(), }; bridge.follow_change_log().await; Ok(bridge) @@ -468,6 +476,7 @@ offset_clients, scan_client, change_client, + id_client, shared_client, ) = tokio::join!( self.produce_pool.close(), @@ -478,6 +487,7 @@ self.offset_pool.close(), self.scan_client.close(), change_client, + self.id_client.close(), sign_out_and_shut_down(&self.client), ); produce_clients @@ -488,6 +498,7 @@ .and(offset_clients) .and(scan_client) .and(change_client) + .and(id_client) .and(shared_client) } @@ -562,6 +573,7 @@ max_message_size: crate::bridge::DEFAULT_MAX_MESSAGE_SIZE, produce_sessions: crate::bridge::ProduceSessionLimits::default(), tls: crate::bridge::IggyTls::default(), + idempotent_producers: false, nodes: KnownNodes::default(), }; let client = Arc::new(client); @@ -584,6 +596,8 @@ topic_shapes: Arc::new(watch::Sender::new(Arc::default())), permissions: Arc::default(), topic_deletions: broadcast::Sender::new(TOPIC_DELETIONS_CAPACITY), + id_client: LazyClient::default(), + issued_producer_ids: producer_ids::IssuedProducerIds::default(), } } }
diff --git a/gateways/kafka/src/bridge/iggy_bridge/produce.rs b/gateways/kafka/src/bridge/iggy_bridge/produce.rs index 1966924..2fa6f22 100644 --- a/gateways/kafka/src/bridge/iggy_bridge/produce.rs +++ b/gateways/kafka/src/bridge/iggy_bridge/produce.rs
@@ -26,7 +26,9 @@ use std::time::Duration; use futures::future::join_all; -use iggy::prelude::{IggyClient, IggyError, IggyMessage, MessageClient, Partitioning}; +use iggy::prelude::{ + BatchProducer, IggyClient, IggyError, IggyMessage, MessageClient, Partitioning, +}; use iggy_common::{binary_operation_deadline, with_binary_deadline}; use tokio::runtime::Handle; use tokio::sync::{OwnedSemaphorePermit, Semaphore, oneshot}; @@ -70,6 +72,16 @@ Connection(u64), } +/// The messages of one partition, sent to Iggy in one request. +#[derive(Debug)] +pub struct PartitionBatch { + /// Passes through unchanged: Kafka and Iggy both count partitions from 0. + pub partition: u32, + /// The id, epoch and first sequence of an idempotent Kafka producer. + pub producer: Option<BatchProducer>, + pub messages: Vec<IggyMessage>, +} + /// One Iggy client. One request holds its turn at a time, so the sends on it reach Iggy in order. struct ProduceSession { client: LazyClient, @@ -351,6 +363,13 @@ self.config.max_message_size.saturating_sub(SEND_FRAME_ROOM) } + /// Whether a batch of an idempotent Kafka producer carries its producer to Iggy. See + /// [`IggyBridgeConfig::idempotent_producers`](crate::bridge::IggyBridgeConfig::idempotent_producers). + #[must_use] + pub const fn writes_producers(&self) -> bool { + self.config.idempotent_producers + } + /// The turn of `producer` on its session. `None` if it does not free by `deadline`: an /// earlier request of the producer, or a send that outlived one, still holds it. /// @@ -370,11 +389,13 @@ self.produce_pool.forget(&Producer::Connection(connection)); } - /// Appends `messages` to `partition` of `target` on the session of `turn`, and returns the + /// Appends `batch` to its partition of `target` on the session of `turn`, and returns the /// base offset. Hands `turn` and `kept` back with it. /// - /// `partition` passes through unchanged: both systems count from 0. The server rejects a - /// partition the topic lacks, so no round trip checks first. + /// With a producer, the batch carries the Kafka producer's id, epoch and first sequence, and + /// the partition writes it once. A resend of a batch it holds answers the first offset. + /// + /// The server rejects a partition the topic lacks, so no round trip checks first. /// /// `Ok(None)`: committed, but the server named no offset (a duplicate request, or a journal /// entry that is gone). Do not retry it. @@ -384,9 +405,9 @@ /// /// A send still running at `deadline` keeps `turn` and `kept`, and hands back `None` for /// them. So the producer's later requests answer 7 meanwhile and cannot overtake it, and what - /// `kept` reserves for `messages` stays reserved. Other producers send on. + /// `kept` reserves for the messages stays reserved. Other producers send on. /// - /// Such a send drops `messages` and `kept` when it ends, at `SLOT_LIMIT` at the latest. Cut + /// Such a send drops the messages and `kept` when it ends, at `SLOT_LIMIT` at the latest. Cut /// off there, it keeps `turn` until its connection is closed. /// /// # Errors @@ -401,10 +422,14 @@ turn: ProduceTurn, kept: K, target: &TopicTarget, - partition: u32, - mut messages: Vec<IggyMessage>, + batch: PartitionBatch, deadline: Instant, ) -> (Result<Option<u64>, BridgeError>, Option<(ProduceTurn, K)>) { + let PartitionBatch { + partition, + producer, + mut messages, + } = batch; let session = Arc::clone(&turn.seat.session); let connect = self.produce_pool.client_of(&session, &self.config); let client = match timeout_at(deadline, connect).await { @@ -431,10 +456,30 @@ async move { let partitioning = Partitioning::partition_id(partition); loop { - match client - .send_messages(&stream_id, &topic_id, &partitioning, &mut messages) - .await - { + let sent = match producer { + Some(producer) => { + client + .send_producer_messages( + &stream_id, + &topic_id, + partition, + producer, + &mut messages, + ) + .await + } + None => { + client + .send_messages( + &stream_id, + &topic_id, + &partitioning, + &mut messages, + ) + .await + } + }; + match sent { Err(IggyError::LaneBlocked(operation)) => { resolve_blocker(&client, operation, sdk_deadline.into_std()) .await?; @@ -595,8 +640,11 @@ .expect("a free turn"), 7, &target, - 0, - Vec::new(), + PartitionBatch { + partition: 0, + producer: None, + messages: Vec::new(), + }, Instant::now(), ) .await;
diff --git a/gateways/kafka/src/bridge/iggy_bridge/producer_ids.rs b/gateways/kafka/src/bridge/iggy_bridge/producer_ids.rs new file mode 100644 index 0000000..525941d --- /dev/null +++ b/gateways/kafka/src/bridge/iggy_bridge/producer_ids.rs
@@ -0,0 +1,170 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Producer ids for `InitProducerId`. +//! +//! Iggy builds a producer id from the session number of the client that asks, so no two clients +//! of the cluster get the same id, also across gateways and restarts. The ids come from a client +//! of their own: one session hands out 2^20 ids, and a new sign-in starts a new range without +//! touching the other clients. +//! +//! A Kafka producer gets the Iggy id with a mark: bit 46 set, and the bits from 46 up one place +//! higher. Older gateways gave ids with a clock in milliseconds in the low 47 bits, which keeps bit +//! 46 clear until the year 4199. Produce passes only a marked id to Iggy, so producers that older +//! gateways gave the same id never share one producer in Iggy. + +use std::collections::{HashSet, VecDeque}; +use std::sync::{Mutex, PoisonError}; + +use iggy::prelude::{IggyError, MessageClient}; + +use super::IggyBridge; +use crate::bridge::error::BridgeError; + +/// Ids that [`IssuedProducerIds`] remembers. An older id still writes once, but Produce keys its +/// session on the connection instead. +const ISSUED_IDS_KEPT: usize = 65_536; + +/// The bit that marks a Kafka producer id built from an Iggy id. +const IGGY_ID_MARK: u32 = 46; +const BELOW_MARK: u64 = (1 << IGGY_ID_MARK) - 1; + +/// The Kafka producer id for `iggy_id`. `None` from 2^62 on, where the mark leaves no room. +#[must_use] +pub fn kafka_producer_id(iggy_id: u64) -> Option<i64> { + if iggy_id >> (u64::BITS - 2) != 0 { + return None; + } + i64::try_from(((iggy_id & !BELOW_MARK) << 1) | (1 << IGGY_ID_MARK) | (iggy_id & BELOW_MARK)) + .ok() +} + +/// The Iggy producer id in a marked Kafka producer id. `None` for an id without the mark, such as +/// one from an older gateway. +#[must_use] +pub fn iggy_producer_id(kafka_id: u64) -> Option<u64> { + ((kafka_id >> IGGY_ID_MARK) & 1 == 1) + .then_some(((kafka_id >> (IGGY_ID_MARK + 1)) << IGGY_ID_MARK) | (kafka_id & BELOW_MARK)) +} + +impl IggyBridge { + /// A producer id that no other Iggy client gets, with the mark. + /// + /// # Errors + /// + /// [`BridgeError::Iggy`] if the id client cannot connect or sign in, or Iggy refuses the id. + pub async fn next_producer_id(&self) -> Result<i64, BridgeError> { + let mut renewed = false; + loop { + let client = self.id_client.connected(&self.config).await?; + match client.next_producer_id().await { + Ok(id) => { + let id = kafka_producer_id(id) + .ok_or(BridgeError::Iggy(IggyError::ProducerIdsExhausted))?; + self.issued_producer_ids.remember(id); + return Ok(id); + } + // The session handed out its whole range. A new session starts a new one. + Err(IggyError::ProducerIdsExhausted) if !renewed => { + renewed = true; + self.id_client.close().await?; + } + Err(error) => return Err(BridgeError::Iggy(error)), + } + } + } + + /// Whether this gateway gave out `producer_id` lately. An id that a client made up, or one + /// from another gateway, is not. + pub fn issued_producer_id(&self, producer_id: i64) -> bool { + self.issued_producer_ids.contains(producer_id) + } +} + +/// The last [`ISSUED_IDS_KEPT`] producer ids this gateway gave out. +#[derive(Debug, Default)] +pub(super) struct IssuedProducerIds(Mutex<Issued>); + +#[derive(Debug, Default)] +struct Issued { + ids: HashSet<i64>, + order: VecDeque<i64>, +} + +impl IssuedProducerIds { + fn remember(&self, producer_id: i64) { + let mut issued = self.0.lock().unwrap_or_else(PoisonError::into_inner); + if !issued.ids.insert(producer_id) { + return; + } + issued.order.push_back(producer_id); + if issued.order.len() > ISSUED_IDS_KEPT + && let Some(oldest) = issued.order.pop_front() + { + issued.ids.remove(&oldest); + } + } + + fn contains(&self, producer_id: i64) -> bool { + self.0 + .lock() + .unwrap_or_else(PoisonError::into_inner) + .ids + .contains(&producer_id) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::protocol::handlers::init_producer_id::ProducerIdAllocator; + + #[test] + fn given_an_iggy_id_when_marked_should_read_back_unchanged() { + for iggy_id in [0, 4711 << 20 | 3, BELOW_MARK, BELOW_MARK + 1, (1 << 62) - 1] { + let kafka_id = kafka_producer_id(iggy_id).unwrap(); + + assert_eq!( + iggy_producer_id(u64::try_from(kafka_id).unwrap()), + Some(iggy_id) + ); + } + assert_eq!(kafka_producer_id(1 << 62), None); + } + + #[test] + fn given_an_id_from_an_older_gateway_when_read_should_have_no_mark() { + for instance_id in [0, 1, 0xBEEF, u16::MAX] { + let id = ProducerIdAllocator::new(instance_id).allocate().unwrap(); + + assert_eq!(iggy_producer_id(u64::try_from(id).unwrap()), None); + } + } + + #[test] + fn given_more_ids_than_kept_when_remembering_should_forget_the_oldest() { + let issued = IssuedProducerIds::default(); + for id in 0..=i64::try_from(ISSUED_IDS_KEPT).unwrap() { + issued.remember(id); + } + + assert!(!issued.contains(0)); + assert!(issued.contains(1)); + assert!(issued.contains(i64::try_from(ISSUED_IDS_KEPT).unwrap())); + assert!(!issued.contains(-1)); + } +}
diff --git a/gateways/kafka/src/bridge/mod.rs b/gateways/kafka/src/bridge/mod.rs index 0a622a8..bd4c249 100644 --- a/gateways/kafka/src/bridge/mod.rs +++ b/gateways/kafka/src/bridge/mod.rs
@@ -34,9 +34,9 @@ CLUSTER_ID_TOPIC, COORDINATOR_LEASE_GROUP, COORDINATOR_LOG_TOPIC, COORDINATOR_OWNER_TOPIC, ClusterIdCall, CoordinatorLink, DEFAULT_TOPIC_PARTITIONS, DeletedTopic, IggyBridge, KafkaTopicMetadata, LoggedRecord, MAX_TOPIC_PARTITIONS, OFFSET_GROUP_PREFIX, OffsetCalls, - OffsetGroupSource, OffsetGroups, PermissionBoard, ProduceTurn, Producer, ScannedGroups, - StreamTopicCache, TopicCreationOutcome, TopicLoad, TopicShape, TopicShapes, TopicTarget, - UserAccess, cluster_id_stream, + OffsetGroupSource, OffsetGroups, PartitionBatch, PermissionBoard, ProduceTurn, Producer, + ScannedGroups, StreamTopicCache, TopicCreationOutcome, TopicLoad, TopicShape, TopicShapes, + TopicTarget, UserAccess, cluster_id_stream, iggy_producer_id, kafka_producer_id, }; pub use nodes::KnownNodes; pub use topic_map::{TopicMapping, TopicOverride};
diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index d658081..a05a691 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs
@@ -325,6 +325,16 @@ const PRODUCE_CAPS: usize = PRODUCE_LARGE + 1; impl GatewayState { + /// Whether this gateway gave out `producer_id` lately, so Produce may key an Iggy session on + /// it. With a bridge the ids come from Iggy. Without one, from the local allocator. + #[must_use] + pub(crate) fn issued_producer_id(&self, producer_id: i64) -> bool { + self.bridge.as_ref().map_or_else( + || self.producer_ids.issued(producer_id), + |bridge| bridge.issued_producer_id(producer_id), + ) + } + #[must_use] pub fn new( bridge: Option<Arc<IggyBridge>>,
diff --git a/gateways/kafka/src/protocol/handlers/init_producer_id.rs b/gateways/kafka/src/protocol/handlers/init_producer_id.rs index 4f62d75..582d13a 100644 --- a/gateways/kafka/src/protocol/handlers/init_producer_id.rs +++ b/gateways/kafka/src/protocol/handlers/init_producer_id.rs
@@ -19,8 +19,11 @@ //! //! A Java producer sets `enable.idempotence=true` without being asked (KIP-679, default since //! Kafka 3.0) and sends this before its first record, so answering it is what lets a stock -//! producer start against this gateway at all. Produce uses the id only to pick the producer's -//! Iggy session: delivery stays at-least-once, and no retry is deduplicated. See +//! producer start against this gateway at all. +//! +//! With a bridge, the id comes from Iggy, which makes it unique across the cluster, and the +//! partitions write each batch of the producer once. Without one, the gateway's own allocator +//! gives the id, and Produce uses it only to pick the producer's Iggy session. See //! `docs/IDEMPOTENCE.md`. use std::sync::atomic::{AtomicU64, Ordering}; @@ -31,12 +34,12 @@ use crate::error::Result; use crate::protocol::api::{ - API_KEY_INIT_PRODUCER_ID, ApiVersionRange, ERROR_NONE, ERROR_UNKNOWN_SERVER_ERROR, - ERROR_UNSUPPORTED_VERSION, GatewayState, HandleOutcome, + API_KEY_INIT_PRODUCER_ID, ApiVersionRange, ERROR_COORDINATOR_LOAD_IN_PROGRESS, ERROR_NONE, + ERROR_UNKNOWN_SERVER_ERROR, ERROR_UNSUPPORTED_VERSION, GatewayState, HandleOutcome, }; use crate::protocol::bounds_guard::validate_init_producer_id_shape; use crate::protocol::handlers::{ - decode_guarded, encode_message, handle_versioned_request, is_transactional, + decode_guarded, decode_request, encode_message, is_transactional, respond_or_close, }; pub const RANGE: ApiVersionRange = ApiVersionRange { @@ -51,8 +54,8 @@ const COUNTER_BITS: u32 = 47; const MAX_COUNTER: u64 = (1 << COUNTER_BITS) - 1; -/// The epoch every allocated id carries. Epochs only advance when a producer is fenced, which -/// needs the transactional state this gateway does not keep. +/// The epoch every new id starts at. An idempotent Java producer moves to the next epoch by +/// itself, and a new epoch starts its sequences at 0, which the partitions accept. const PRODUCER_EPOCH: i16 = 0; /// Hands out producer ids that are unique across gateway instances sharing one Iggy cluster. @@ -67,9 +70,9 @@ /// allocation per millisecond of its uptime, the clock stepped back across the restart, or the /// clock reads before the Unix epoch and seeds 0. Nothing is persisted. /// -/// Produce keys an Iggy session on the id, and only on an id of this run (`issued`). A -/// replayed pair then makes two producers share a session, and nothing more. Before deduplication -/// reads the id, the allocator has to persist a high-water mark or bump the epoch on every start. +/// Only a gateway without a bridge gives out these ids, and it writes nothing. Older gateways +/// gave them out with a bridge too, so Produce writes an id without the Iggy mark as a plain +/// batch (see `bridge::iggy_producer_id`). #[derive(Debug)] pub struct ProducerIdAllocator { instance_id: u16, @@ -109,20 +112,37 @@ } } -#[expect( - clippy::unused_async, - reason = "the shared handler signature, kept until a handler awaits the bridge" -)] +/// Answers with a new producer id at epoch 0. +/// +/// A request that names its current id and epoch (KIP-360) also gets a new id: Kafka does the +/// same for a producer without a transactional id. pub async fn handle(state: &GatewayState, api_version: i16, body: Bytes) -> HandleOutcome { - handle_versioned_request( + let req = match decode_request( API_KEY_INIT_PRODUCER_ID, api_version, body, |v, b| decode_guarded::<InitProducerIdRequest>(v, b, validate_init_producer_id_shape), - |v, req| encode_response(v, req, &state.producer_ids), encode_error_response, "InitProducerId", - ) + ) { + Ok(req) => req, + Err(outcome) => return outcome, + }; + let encoded = match &state.bridge { + Some(bridge) if !is_transactional(req.transactional_id.as_ref()) => { + match bridge.next_producer_id().await { + Ok(producer_id) => encode_inner(api_version, ERROR_NONE, producer_id), + // Retriable, so the producer asks again. Not 15 or 16: those send a Java + // producer to look for a transaction coordinator, which this gateway has none of. + Err(error) => { + tracing::warn!(%error, "could not get a producer id from Iggy"); + encode_error_response(api_version, ERROR_COORDINATOR_LOAD_IN_PROGRESS) + } + } + } + _ => encode_response(api_version, &req, &state.producer_ids), + }; + respond_or_close(encoded, "InitProducerId") } /// `InitProducerId` response carrying `error_code` and no usable producer id.
diff --git a/gateways/kafka/src/protocol/handlers/produce.rs b/gateways/kafka/src/protocol/handlers/produce.rs index f720f1e..a6d0775 100644 --- a/gateways/kafka/src/protocol/handlers/produce.rs +++ b/gateways/kafka/src/protocol/handlers/produce.rs
@@ -21,7 +21,7 @@ use std::time::Duration; use bytes::Bytes; -use iggy::prelude::{IggyError, IggyMessage, Sizeable}; +use iggy::prelude::{BatchProducer, IggyError, Sizeable}; use kafka_protocol::messages::produce_request::PartitionProduceData; use kafka_protocol::messages::produce_response::{PartitionProduceResponse, TopicProduceResponse}; use kafka_protocol::messages::{ProduceRequest, ProduceResponse}; @@ -29,14 +29,17 @@ use tokio::sync::{AcquireError, OwnedSemaphorePermit}; use crate::auth::AuthenticatedPrincipal; -use crate::bridge::{BridgeError, IggyBridge, ProduceTurn, Producer, TopicTarget}; +use crate::bridge::{ + BridgeError, IggyBridge, PartitionBatch, ProduceTurn, Producer, TopicTarget, iggy_producer_id, +}; use crate::error::Result; use crate::protocol::api::{ API_KEY_PRODUCE, ApiVersionRange, ConnectionState, ERROR_INVALID_RECORD, ERROR_INVALID_REQUIRED_ACKS, ERROR_MESSAGE_TOO_LARGE, ERROR_NONE, ERROR_NOT_LEADER_OR_FOLLOWER, - ERROR_REQUEST_TIMED_OUT, ERROR_UNKNOWN_SERVER_ERROR, ERROR_UNKNOWN_TOPIC_OR_PARTITION, - ERROR_UNSUPPORTED_COMPRESSION_TYPE, ERROR_UNSUPPORTED_VERSION, GatewayState, HandleOutcome, - REQUEST_DEADLINE, UNKNOWN_OFFSET, supported_max_version, + ERROR_REQUEST_TIMED_OUT, ERROR_UNKNOWN_PRODUCER_ID, ERROR_UNKNOWN_SERVER_ERROR, + ERROR_UNKNOWN_TOPIC_OR_PARTITION, ERROR_UNSUPPORTED_COMPRESSION_TYPE, + ERROR_UNSUPPORTED_VERSION, GatewayState, HandleOutcome, REQUEST_DEADLINE, UNKNOWN_OFFSET, + supported_max_version, }; use crate::protocol::bounds_guard::validate_produce_shape; use crate::protocol::handlers::{ @@ -45,8 +48,8 @@ }; use crate::protocol::probe_board::ProbeBoard; use crate::records::{ - Allowance, DecompressionBudget, RecordCodecError, TimestampWindow, Zstd, decode_batch, - is_compressed, producer_id, record_slots, to_iggy, + Allowance, DecompressionBudget, RecordCodecError, TimestampWindow, Zstd, batch_producer, + decode_batch, is_compressed, producer_id, record_slots, to_iggy, }; pub const RANGE: ApiVersionRange = ApiVersionRange { @@ -300,7 +303,7 @@ .flat_map(|topic| &topic.partition_data) .find_map(|partition| partition.records.as_deref()) .and_then(producer_id) - .filter(|id| state.producer_ids.issued(*id)) + .filter(|id| state.issued_producer_id(*id)) .map_or_else( || Producer::Connection(connection.id()), |id| Producer::Id { @@ -374,6 +377,7 @@ deadline: Deadline, ) -> Vec<TopicProduceResponse> { let max_send = bridge.max_send_bytes(); + let writes_producers = bridge.writes_producers(); let mut held = Some(held); let mut topics = Vec::with_capacity(request.topic_data.len()); for topic in &request.topic_data { @@ -385,9 +389,12 @@ let (outcome, kept) = match (&target, held.take()) { (Ok(target), Some(kept)) if !deadline.passed() => { match plan(&budget, zstd, name, partition, max_send) { - Ok((id, messages)) => { + Ok(mut batch) => { + if !writes_producers { + batch.producer = None; + } let (sent, kept) = - send_by(bridge, kept, target, name, id, messages, deadline).await; + send_by(bridge, kept, target, name, batch, deadline).await; (sent.map_err(Refusal::from), kept) } Err(refusal) => (Err(refusal), Some(kept)), @@ -424,7 +431,7 @@ kafka_topic: &str, partition: &PartitionProduceData, max_send: u64, -) -> std::result::Result<(u32, Vec<IggyMessage>), Refusal> { +) -> std::result::Result<PartitionBatch, Refusal> { let run = || plan_partition(budget, zstd, kafka_topic, partition, max_send); // A spent budget refuses before it reads a byte, so there is nothing to move off the worker. let heavy = !budget.is_spent() @@ -448,7 +455,15 @@ }), ), Err(Refusal { code, message }) => { - partition_response(index, code).with_error_message(message.map(StrBytes::from_string)) + let response = partition_response(index, code) + .with_error_message(message.map(StrBytes::from_string)); + // The Java producer retries a 59 unchanged while the log start is unknown, until the + // delivery timeout. With a known start, it moves to a new epoch and sends again. + if code == ERROR_UNKNOWN_PRODUCER_ID { + response.with_log_start_offset(0) + } else { + response + } } } } @@ -461,7 +476,7 @@ kafka_topic: &str, partition: &PartitionProduceData, max_send: u64, -) -> std::result::Result<(u32, Vec<IggyMessage>), Refusal> { +) -> std::result::Result<PartitionBatch, Refusal> { // The records are not what is wrong with a negative index, so not `INVALID_RECORD`. let Ok(partition_id) = u32::try_from(partition.index) else { return Err(ERROR_UNKNOWN_TOPIC_OR_PARTITION.into()); @@ -475,6 +490,23 @@ Refusal::from(error) }; + // An idempotent batch carries its producer to Iggy, which writes it once. A batch with a + // producer id but a negative epoch or sequence is one no Kafka producer sends. An id without + // the Iggy mark, as older gateways gave out, writes as a plain batch. + let producer = match batch_producer(blob) { + None => None, + Some(fields) => { + let (Ok(id), Ok(epoch), Ok(sequence)) = ( + u64::try_from(fields.producer_id), + u16::try_from(fields.producer_epoch), + u32::try_from(fields.base_sequence), + ) else { + return Err(ERROR_INVALID_RECORD.into()); + }; + iggy_producer_id(id).and_then(|id| BatchProducer::new(id, epoch, sequence)) + } + }; + // The clone is a refcount, and `decode_batch` drains what it is given. let records = decode_batch(&mut blob.clone(), budget, zstd).map_err(refuse)?; // An empty blob decodes to no batch rather than to an error, so this rules on it. @@ -501,7 +533,11 @@ ); return Err(ERROR_MESSAGE_TOO_LARGE.into()); } - Ok((partition_id, messages)) + Ok(PartitionBatch { + partition: partition_id, + producer, + messages, + }) } /// The shorter of the client's `timeout_ms` and this gateway's ceiling. @@ -527,12 +563,12 @@ (turn, admission): Held, target: &TopicTarget, kafka_topic: &str, - partition_id: u32, - messages: Vec<IggyMessage>, + batch: PartitionBatch, deadline: Deadline, ) -> (std::result::Result<Option<u64>, i16>, Option<Held>) { + let partition_id = batch.partition; let (sent, held) = bridge - .send_records(turn, admission, target, partition_id, messages, deadline.at) + .send_records(turn, admission, target, batch, deadline.at) .await; let sent = match sent { Ok(base_offset) => Ok(base_offset), @@ -692,6 +728,7 @@ mod tests { use super::*; use bytes::BytesMut; + use iggy::prelude::IggyMessage; use kafka_protocol::indexmap::IndexMap; use kafka_protocol::messages::TopicName; use kafka_protocol::messages::produce_request::TopicProduceData; @@ -702,6 +739,7 @@ }; use tokio_util::sync::CancellationToken; + use crate::bridge::kafka_producer_id; use crate::group::{GroupCoordinator, GroupCoordinatorConfig}; use crate::protocol::acl::PrincipalPermissions; use crate::records::encode_batch; @@ -754,7 +792,86 @@ zstd: Zstd, entry: &PartitionProduceData, ) -> std::result::Result<(u32, Vec<IggyMessage>), i16> { - plan_partition(budget, zstd, TOPIC, entry, u64::MAX).map_err(|refusal| refusal.code) + plan_partition(budget, zstd, TOPIC, entry, u64::MAX) + .map(|batch| (batch.partition, batch.messages)) + .map_err(|refusal| refusal.code) + } + + /// Two records of Iggy producer 7 at epoch 1, the first at `base_sequence`. + fn idempotent_batch(base_sequence: i32) -> Bytes { + batch_of_producer(kafka_producer_id(7).unwrap(), base_sequence) + } + + fn batch_of_producer(producer_id: i64, base_sequence: i32) -> Bytes { + let records: Vec<Record> = (0..2) + .map(|offset| Record { + producer_id, + producer_epoch: 1, + sequence: base_sequence.wrapping_add(i32::try_from(offset).unwrap()), + ..record(offset, b"v") + }) + .collect(); + compressed_batch(&records, Compression::None) + } + + #[test] + fn given_an_idempotent_batch_when_planned_should_carry_its_producer() { + let planned = plan_partition( + &partition_budget(TEST_BUDGET, usize::MAX), + Zstd::Allowed, + TOPIC, + &entry(0, Some(idempotent_batch(5))), + u64::MAX, + ) + .unwrap_or_else(|refusal| panic!("refused with {}", refusal.code)); + + assert_eq!(planned.producer, Some(BatchProducer::new(7, 1, 5).unwrap())); + assert_eq!(planned.messages.len(), 2); + let plain = plan_partition( + &partition_budget(TEST_BUDGET, usize::MAX), + Zstd::Allowed, + TOPIC, + &entry( + 0, + Some(compressed_batch(&[record(0, b"v")], Compression::None)), + ), + u64::MAX, + ) + .unwrap_or_else(|refusal| panic!("refused with {}", refusal.code)); + assert_eq!(plain.producer, None); + } + + #[test] + fn given_a_producer_id_without_the_iggy_mark_when_planned_should_write_a_plain_batch() { + let planned = plan_partition( + &partition_budget(TEST_BUDGET, usize::MAX), + Zstd::Allowed, + TOPIC, + &entry(0, Some(batch_of_producer(7, 5))), + u64::MAX, + ) + .unwrap_or_else(|refusal| panic!("refused with {}", refusal.code)); + + assert_eq!(planned.producer, None); + assert_eq!(planned.messages.len(), 2); + } + + #[test] + fn given_a_producer_id_without_a_sequence_when_planned_should_refuse_the_batch() { + assert_eq!( + planned(&entry(0, Some(idempotent_batch(NO_SEQUENCE)))), + Err(ERROR_INVALID_RECORD) + ); + } + + #[test] + fn given_an_unknown_producer_when_answered_should_name_a_log_start() { + let unknown = partition_outcome(0, Err(ERROR_UNKNOWN_PRODUCER_ID.into())); + assert_eq!(unknown.error_code, ERROR_UNKNOWN_PRODUCER_ID); + assert_eq!(unknown.log_start_offset, 0); + + let timed_out = partition_outcome(0, Err(ERROR_REQUEST_TIMED_OUT.into())); + assert_eq!(timed_out.log_start_offset, UNKNOWN_OFFSET); } fn planned(entry: &PartitionProduceData) -> std::result::Result<(u32, Vec<IggyMessage>), i16> {
diff --git a/gateways/kafka/tests/bridge_iggy_integration_tests.rs b/gateways/kafka/tests/bridge_iggy_integration_tests.rs index 4a7f6c2..5eefe5d 100644 --- a/gateways/kafka/tests/bridge_iggy_integration_tests.rs +++ b/gateways/kafka/tests/bridge_iggy_integration_tests.rs
@@ -41,7 +41,8 @@ use iggy_gateway_kafka::auth::{IggyAuthenticator, SaslAuthenticator}; use iggy_gateway_kafka::bridge::{ BridgeError, DEFAULT_MAX_MESSAGE_SIZE, IggyBridge, IggyBridgeConfig, IggyTls, KnownNodes, - OffsetCalls, ProduceSessionLimits, Producer, TopicCreationOutcome, TopicMapping, TopicOverride, + OffsetCalls, PartitionBatch, ProduceSessionLimits, Producer, TopicCreationOutcome, + TopicMapping, TopicOverride, }; use iggy_gateway_kafka::protocol::api::ERROR_REQUEST_TIMED_OUT; use iggy_gateway_kafka::protocol::sasl::PlainCredentials; @@ -173,8 +174,11 @@ turn, (), &target, - 0, - vec![IggyMessage::from_str(payload).expect("message")], + PartitionBatch { + partition: 0, + producer: None, + messages: vec![IggyMessage::from_str(payload).expect("message")], + }, deadline, ) .await @@ -618,6 +622,7 @@ max_message_size: DEFAULT_MAX_MESSAGE_SIZE, tls: IggyTls::default(), produce_sessions: ProduceSessionLimits::default(), + idempotent_producers: false, nodes: KnownNodes::default(), }; @@ -644,6 +649,7 @@ max_message_size: DEFAULT_MAX_MESSAGE_SIZE, tls: IggyTls::default(), produce_sessions: ProduceSessionLimits::default(), + idempotent_producers: false, nodes: KnownNodes::default(), };
diff --git a/gateways/kafka/tests/common/gateway_process.rs b/gateways/kafka/tests/common/gateway_process.rs index d29c0ce..09d5876 100644 --- a/gateways/kafka/tests/common/gateway_process.rs +++ b/gateways/kafka/tests/common/gateway_process.rs
@@ -79,6 +79,10 @@ .env("IGGY_KAFKA_IGGY_ADDR", &config.address) .env("IGGY_KAFKA_IGGY_USERNAME", &config.username) .env( + "IGGY_KAFKA_IDEMPOTENT_PRODUCERS", + config.idempotent_producers.to_string(), + ) + .env( "IGGY_KAFKA_IGGY_PASSWORD", secrecy::ExposeSecret::expose_secret(&config.password), );
diff --git a/gateways/kafka/tests/common/iggy_server.rs b/gateways/kafka/tests/common/iggy_server.rs index be55446..1c9ff1c 100644 --- a/gateways/kafka/tests/common/iggy_server.rs +++ b/gateways/kafka/tests/common/iggy_server.rs
@@ -403,6 +403,7 @@ max_message_size: DEFAULT_MAX_MESSAGE_SIZE, tls: IggyTls::default(), produce_sessions: ProduceSessionLimits::default(), + idempotent_producers: true, nodes: KnownNodes::default(), } } @@ -610,6 +611,7 @@ max_message_size: DEFAULT_MAX_MESSAGE_SIZE, tls: IggyTls::default(), produce_sessions: ProduceSessionLimits::default(), + idempotent_producers: true, nodes: KnownNodes::default(), } }
diff --git a/gateways/kafka/tests/kafka_client_e2e_tests.rs b/gateways/kafka/tests/kafka_client_e2e_tests.rs index 810677e..29d8c00 100644 --- a/gateways/kafka/tests/kafka_client_e2e_tests.rs +++ b/gateways/kafka/tests/kafka_client_e2e_tests.rs
@@ -36,18 +36,21 @@ //! `KAFKA_E2E_REQUIRED=1` is set, which turns a skip into a failure so a broken CI step cannot //! leave these silently green. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::io::{BufRead, BufReader, Read}; use std::net::SocketAddr; use std::path::PathBuf; use std::process::{Child, Command, Stdio}; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, mpsc}; +use std::sync::{Arc, Mutex, mpsc}; use std::time::{Duration, Instant}; use iggy::prelude::IggyTimestamp; use secrecy::SecretString; use serde_json::Value; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::tcp::OwnedReadHalf; +use tokio::net::{TcpListener, TcpStream}; use tokio::sync::broadcast; use tokio::task::JoinHandle; @@ -101,6 +104,12 @@ const FAILOVER_RECORDS: u32 = 600; const FAILOVER_RATE: u32 = 60; +/// The topic that a relay loses Produce responses for, and the records written there. +const LOST_REPLY_TOPIC: &str = "lost-replies"; +const LOST_REPLY_RECORDS: u32 = 600; +/// Produce responses the relay drops. Each drop closes that connection, as a lost reply does. +const LOST_REPLIES: usize = 3; + /// Budget for `iggy-server` to start listening. Generous: it is a cold process start, and a debug /// build on a loaded machine is not quick. const SERVER_READY_TIMEOUT: Duration = Duration::from_secs(45); @@ -425,6 +434,7 @@ max_message_size: DEFAULT_MAX_MESSAGE_SIZE, tls: IggyTls::default(), produce_sessions: ProduceSessionLimits::default(), + idempotent_producers: true, nodes: KnownNodes::default(), }; IggyBridge::connect(config) @@ -2318,6 +2328,7 @@ max_message_size: DEFAULT_MAX_MESSAGE_SIZE, produce_sessions: ProduceSessionLimits::default(), tls: IggyTls::default(), + idempotent_producers: true, nodes: KnownNodes::default(), }) .await @@ -2611,9 +2622,11 @@ } /// A Java producer that knows only the first gateway learns the second from Metadata. When the -/// first stops, the producer goes on through the second, and every record lands. +/// first stops, the producer goes on through the second. It is idempotent by default, so each +/// record lands once, also the ones it sends again. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn given_two_listed_gateways_when_the_first_stops_should_keep_producing_through_the_second() { +async fn given_two_listed_gateways_when_the_first_stops_should_write_each_record_once_through_the_second() + { let Some(server) = ready_server() else { return; }; @@ -2692,8 +2705,8 @@ .lines() .filter(|line| line.parse::<u64>().is_ok()) .count(); - assert!( - stored >= FAILOVER_RECORDS as usize, + assert_eq!( + stored, FAILOVER_RECORDS as usize, "{stored} of {FAILOVER_RECORDS} records stored" ); } @@ -2804,3 +2817,216 @@ ) .expect_ran("the batch is produced"); } + +/// A Java producer behind a relay that loses Produce responses sends those batches again, on a new +/// connection. The partitions hold its producer state, so each record lands once. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn given_lost_produce_responses_when_an_idempotent_producer_retries_should_write_each_record_once() + { + let Some(server) = ready_server() else { + return; + }; + let relay_listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind the relay"); + let relay_addr = relay_listener.local_addr().expect("relay addr"); + let gateway = start_gateway_behind(&server, (LOST_REPLY_TOPIC, 1), relay_addr).await; + let relay = LosingRelay::start(relay_listener, gateway.addr, LOST_REPLIES); + + let bootstrap = format!("bootstrap.servers={relay_addr}"); + let records = LOST_REPLY_RECORDS.to_string(); + let output = run_client( + KAFKA_IMAGE, + &[ + "/opt/kafka/bin/kafka-producer-perf-test.sh", + "--topic", + LOST_REPLY_TOPIC, + "--num-records", + &records, + "--throughput", + "200", + "--record-size", + "64", + "--producer-props", + &bootstrap, + "acks=all", + "request.timeout.ms=10000", + "delivery.timeout.ms=60000", + ], + &[], + ) + .expect_ran("the producer runs"); + assert!( + output + .lines() + .any(|line| line.starts_with(&format!("{LOST_REPLY_RECORDS} records sent"))), + "every record must be acknowledged: {output}" + ); + assert_eq!( + relay.lost(), + LOST_REPLIES, + "the relay must lose that many responses: {output}" + ); + + let read = run_client( + KCAT_IMAGE, + &[ + "-b", + &relay_addr.to_string(), + "-C", + "-t", + LOST_REPLY_TOPIC, + "-p", + "0", + "-o", + "beginning", + "-e", + "-f", + "%o\n", + ], + &[], + ) + .expect_ran("kcat reads the topic"); + let stored = read + .lines() + .filter(|line| line.parse::<u64>().is_ok()) + .count(); + assert_eq!( + stored, LOST_REPLY_RECORDS as usize, + "{stored} of {LOST_REPLY_RECORDS} records stored" + ); + gateway.stop().await; +} + +/// A bridged gateway that advertises `relay` in place of its own address, so a client that learns +/// the brokers from Metadata stays behind the relay. +async fn start_gateway_behind( + server: &TestServer, + (topic, partitions): (&str, u32), + relay: SocketAddr, +) -> BridgedGateway { + let bridge = root_bridge(server).await; + bridge + .ensure_stream_and_topic(topic, partitions) + .await + .expect("create the topic"); + let socket = bind_listener("127.0.0.1:0").expect("bind ephemeral port"); + let addr = socket.local_addr().expect("local addr"); + let listener = BoundListener::new( + ListenerConfig::new(SecurityProtocol::Plaintext, addr.to_string()) + .with_advertised(relay.ip().to_string(), relay.port()), + socket, + ); + let gateway = KafkaGateway::new(GatewayConfig::default()).with_bridge(Some(Arc::new(bridge))); + let (shutdown, receiver) = broadcast::channel(1); + let task = tokio::spawn(async move { + let _ = gateway.run(vec![listener], receiver).await; + }); + BridgedGateway { + addr, + shutdown, + task, + } +} + +/// A TCP relay to a gateway that loses its first Produce responses. It reads the Kafka frames of +/// each connection, and drops a Produce response together with its connection. +struct LosingRelay { + lost: Arc<AtomicUsize>, +} + +impl LosingRelay { + fn start(listener: TcpListener, gateway: SocketAddr, losses: usize) -> Self { + let lost = Arc::new(AtomicUsize::new(0)); + let budget = Arc::new(AtomicUsize::new(losses)); + let counted = Arc::clone(&lost); + tokio::spawn(async move { + while let Ok((client, _)) = listener.accept().await { + let Ok(upstream) = TcpStream::connect(gateway).await else { + continue; + }; + tokio::spawn(relay_frames( + client, + upstream, + Arc::clone(&budget), + Arc::clone(&counted), + )); + } + }); + Self { lost } + } + + fn lost(&self) -> usize { + self.lost.load(Ordering::SeqCst) + } +} + +/// Copies frames both ways until one side closes, or until a Produce response is lost. +async fn relay_frames( + client: TcpStream, + upstream: TcpStream, + budget: Arc<AtomicUsize>, + lost: Arc<AtomicUsize>, +) { + const API_KEY_PRODUCE: i16 = 0; + let (mut client_read, mut client_write) = client.into_split(); + let (mut upstream_read, mut upstream_write) = upstream.into_split(); + let produce_requests = Arc::new(Mutex::new(HashSet::new())); + let requests = { + let produce_requests = Arc::clone(&produce_requests); + async move { + while let Some(frame) = read_frame(&mut client_read).await { + // After the length: api key, api version, correlation id. + if frame.len() >= 12 && i16::from_be_bytes([frame[4], frame[5]]) == API_KEY_PRODUCE + { + let correlation = + i32::from_be_bytes([frame[8], frame[9], frame[10], frame[11]]); + produce_requests + .lock() + .expect("the request set") + .insert(correlation); + } + if upstream_write.write_all(&frame).await.is_err() { + return; + } + } + } + }; + let responses = async move { + while let Some(frame) = read_frame(&mut upstream_read).await { + let correlation = i32::from_be_bytes([frame[4], frame[5], frame[6], frame[7]]); + let produce = produce_requests + .lock() + .expect("the request set") + .remove(&correlation); + if produce + && budget + .try_update(Ordering::SeqCst, Ordering::SeqCst, |left| { + left.checked_sub(1) + }) + .is_ok() + { + lost.fetch_add(1, Ordering::SeqCst); + return; + } + if client_write.write_all(&frame).await.is_err() { + return; + } + } + }; + tokio::select! { + () = requests => {} + () = responses => {} + } +} + +/// One length-prefixed Kafka frame, the length included. +async fn read_frame(read: &mut OwnedReadHalf) -> Option<Vec<u8>> { + let mut length = [0; 4]; + read.read_exact(&mut length).await.ok()?; + let size = usize::try_from(i32::from_be_bytes(length)).ok()?; + let mut frame = vec![0; 4 + size]; + frame[..4].copy_from_slice(&length); + read.read_exact(&mut frame[4..]).await.ok()?; + Some(frame) +}
diff --git a/gateways/kafka/tests/produce_real_bridge_tests.rs b/gateways/kafka/tests/produce_real_bridge_tests.rs index 6ce76ed..d7f5c0f 100644 --- a/gateways/kafka/tests/produce_real_bridge_tests.rs +++ b/gateways/kafka/tests/produce_real_bridge_tests.rs
@@ -40,10 +40,13 @@ use tokio::sync::watch; use tokio_util::sync::CancellationToken; -use iggy_gateway_kafka::bridge::{IggyBridge, IggyBridgeConfig, Producer}; +use iggy_gateway_kafka::bridge::{ + IggyBridge, IggyBridgeConfig, PartitionBatch, Producer, kafka_producer_id, +}; use iggy_gateway_kafka::group::{GroupCoordinator, GroupCoordinatorConfig}; use iggy_gateway_kafka::protocol::api::{ - API_KEY_INIT_PRODUCER_ID, API_KEY_PRODUCE, ERROR_NONE, ERROR_NOT_LEADER_OR_FOLLOWER, + API_KEY_INIT_PRODUCER_ID, API_KEY_PRODUCE, ERROR_INVALID_PRODUCER_EPOCH, ERROR_NONE, + ERROR_NOT_LEADER_OR_FOLLOWER, ERROR_OUT_OF_ORDER_SEQUENCE_NUMBER, ERROR_UNKNOWN_PRODUCER_ID, ERROR_UNKNOWN_TOPIC_OR_PARTITION, GatewayState, handle_request_bounded, }; use iggy_gateway_kafka::records::{KEY_HEADER, MAPPING_VERSION, VERSION_HEADER, from_iggy}; @@ -187,6 +190,14 @@ /// Every partition entry of a response, flattened, in the order the response lists them. fn partition_results(version: i16, body: Bytes) -> Vec<(i32, i16, i64)> { + partition_entries(version, body) + .into_iter() + .map(|(index, error_code, base_offset, _)| (index, error_code, base_offset)) + .collect() +} + +/// [`partition_results`], with the log start offset of each entry from v5 on. +fn partition_entries(version: i16, body: Bytes) -> Vec<(i32, i16, i64, Option<i64>)> { let flexible = version >= 9; let mut d = Decoder::new(body); let topics = read_count(&mut d, flexible); @@ -201,9 +212,7 @@ let base_offset = d.read_i64().expect("base offset"); let append_time = d.read_i64().expect("log append time"); assert_eq!(append_time, -1, "CreateTime topics report no append time"); - if version >= 5 { - d.read_i64().expect("log start offset"); - } + let log_start_offset = (version >= 5).then(|| d.read_i64().expect("log start offset")); if version >= 8 { assert_eq!(read_count(&mut d, flexible), 0, "no per-record errors"); read_name(&mut d, flexible); // error_message @@ -211,7 +220,7 @@ if flexible { d.read_tagged_fields().expect("partition tagged fields"); } - results.push((index, error_code, base_offset)); + results.push((index, error_code, base_offset, log_start_offset)); } if flexible { d.read_tagged_fields().expect("topic tagged fields"); @@ -689,7 +698,7 @@ let state = gateway_on(config, 1).await; // Each request here comes on a new connection, so only a producer id keeps one Iggy session. let id = issued_producer_id(&state).await; - let first = [(0, idempotent_batch(id, b"first"))]; + let first = [(0, producer_batch(id, 0, 0, &[b"first"]))]; assert_eq!( produce(&state, 3, 1, TOPIC, &first).await, vec![(0, ERROR_NONE, 0)], @@ -697,7 +706,7 @@ ); relay.go_down().await; - let retried = [(0, idempotent_batch(id, b"retried"))]; + let retried = [(0, producer_batch(id, 0, 1, &[b"retried"]))]; let body = encode_timed_request(3, 1, 1_000, TOPIC, &retried); let response = handle_request_bounded(&state, API_KEY_PRODUCE, 3, body) .await @@ -831,12 +840,18 @@ d.read_i64().expect("producer id") } -/// A one-record batch that carries `producer_id`, as an idempotent producer sends it. -fn idempotent_batch(producer_id: i64, value: &[u8]) -> Bytes { - let mut records = [record(0, None, value, &[])]; - records[0].producer_id = producer_id; - records[0].producer_epoch = 0; - records[0].sequence = 0; +/// A batch of `values` from `producer_id` at `epoch`, the first record at `base_sequence`, as an +/// idempotent producer sends it. +fn producer_batch(producer_id: i64, epoch: i16, base_sequence: i32, values: &[&[u8]]) -> Bytes { + let mut records: Vec<Record> = (0..) + .zip(values) + .map(|(offset, value)| record(offset, None, value, &[])) + .collect(); + for (sequence, record) in (base_sequence..).zip(records.iter_mut()) { + record.producer_id = producer_id; + record.producer_epoch = epoch; + record.sequence = sequence; + } batch(&records, Compression::None) } @@ -864,7 +879,7 @@ "another producer sends on a session of its own" ); - let own = [(0, idempotent_batch(id, b"own"))]; + let own = [(0, producer_batch(id, 0, 0, &[b"own"]))]; let answer = |acks| { let body = encode_timed_request(3, acks, 100, TOPIC, &own); handle_request_bounded(&state, API_KEY_PRODUCE, 3, body) @@ -907,8 +922,11 @@ .expect("the producer's own turn"), (), &target, - 0, - vec![message], + PartitionBatch { + partition: 0, + producer: None, + messages: vec![message], + }, soon_enough(), ) .await; @@ -1033,16 +1051,14 @@ #[tokio::test] #[serial] -async fn given_an_idempotent_batch_sent_twice_when_handled_should_store_both_copies() { +async fn given_an_idempotent_batch_sent_twice_when_handled_should_store_it_once() { let data_dir = tempfile::tempdir().expect("tempdir"); let server = TestServer::spawn(data_dir.path()).await; let state = gateway_with_topic(&server, 1).await; - - let mut records = [record(0, None, b"v", &[])]; - records[0].producer_id = 7; - records[0].producer_epoch = 0; - records[0].sequence = 0; - let entries = [(0, batch(&records, Compression::None))]; + let entries = [( + 0, + producer_batch(kafka_producer_id(7).unwrap(), 0, 0, &[b"v"]), + )]; assert_eq!( produce(&state, 3, -1, TOPIC, &entries).await, @@ -1051,18 +1067,110 @@ ); assert_eq!( produce(&state, 3, -1, TOPIC, &entries).await, + vec![(0, ERROR_NONE, 0)], + "a resend answers the offset of its first write" + ); + assert_eq!( + stored(&server, 0, 10).await.len(), + 1, + "the partition writes the batch once" + ); +} + +/// Older gateways gave ids without the Iggy mark, and two of their producers can hold the same +/// one. Such a batch writes as a plain batch. +#[tokio::test] +#[serial] +async fn given_an_id_without_the_iggy_mark_when_sent_twice_should_store_it_twice() { + let data_dir = tempfile::tempdir().expect("tempdir"); + let server = TestServer::spawn(data_dir.path()).await; + let state = gateway_with_topic(&server, 1).await; + let entries = [(0, producer_batch(7, 0, 0, &[b"v"]))]; + + assert_eq!( + produce(&state, 3, -1, TOPIC, &entries).await, + vec![(0, ERROR_NONE, 0)] + ); + assert_eq!( + produce(&state, 3, -1, TOPIC, &entries).await, vec![(0, ERROR_NONE, 1)], - "the sequence is ignored, so a resend never answers DUPLICATE_SEQUENCE_NUMBER (46)" + "the partition holds no producer for the id" + ); + assert_eq!(stored(&server, 0, 10).await.len(), 2); +} + +#[tokio::test] +#[serial] +async fn given_idempotent_batches_out_of_line_when_handled_should_refuse_them_as_kafka_does() { + const VERSION: i16 = 5; + let data_dir = tempfile::tempdir().expect("tempdir"); + let server = TestServer::spawn(data_dir.path()).await; + let state = gateway_with_topic(&server, 1).await; + let send = |entries: Vec<(i32, Bytes)>| { + let body = encode_request(VERSION, -1, TOPIC, &entries); + let state = &state; + async move { + let response = handle_request_bounded(state, API_KEY_PRODUCE, VERSION, body) + .await + .expect_response("acks=-1 is answered"); + partition_entries(VERSION, response) + } + }; + + let id = kafka_producer_id(7).unwrap(); + let unknown = kafka_producer_id(8).unwrap(); + assert_eq!( + send(vec![(0, producer_batch(id, 1, 0, &[b"a", b"b"]))]).await, + vec![(0, ERROR_NONE, 0, Some(-1))] + ); + assert_eq!( + send(vec![(0, producer_batch(id, 1, 5, &[b"gap"]))]).await, + vec![(0, ERROR_OUT_OF_ORDER_SEQUENCE_NUMBER, -1, Some(-1))], + "a batch after a gap" + ); + assert_eq!( + send(vec![(0, producer_batch(id, 0, 2, &[b"old"]))]).await, + vec![(0, ERROR_INVALID_PRODUCER_EPOCH, -1, Some(-1))], + "a batch of an older epoch" + ); + assert_eq!( + send(vec![(0, producer_batch(unknown, 0, 3, &[b"late"]))]).await, + vec![(0, ERROR_UNKNOWN_PRODUCER_ID, -1, Some(0))], + "a producer the partition does not hold, with a log start so the client starts over" ); assert_eq!( stored(&server, 0, 10).await.len(), 2, - "a retry writes twice until the producer pool in IDEMPOTENCE.md lands" + "nothing refused was written" ); } #[tokio::test] #[serial] +async fn given_a_producer_that_moves_to_another_gateway_when_it_resends_should_store_once() { + let data_dir = tempfile::tempdir().expect("tempdir"); + let server = TestServer::spawn(data_dir.path()).await; + let first = gateway_with_topic(&server, 1).await; + let second = gateway_on(server.test_config(), 1).await; + + let id = issued_producer_id(&first).await; + let other = issued_producer_id(&second).await; + assert_ne!(id, other, "two gateways never give out the same id"); + let entries = [(0, producer_batch(id, 0, 0, &[b"once"]))]; + assert_eq!( + produce(&first, 3, -1, TOPIC, &entries).await, + vec![(0, ERROR_NONE, 0)] + ); + assert_eq!( + produce(&second, 3, -1, TOPIC, &entries).await, + vec![(0, ERROR_NONE, 0)], + "the reply was lost, and the producer sends the batch again through another gateway" + ); + assert_eq!(stored(&server, 0, 10).await.len(), 1); +} + +#[tokio::test] +#[serial] async fn given_no_bridge_when_handled_should_still_answer_the_retriable_stub() { // Every other suite drives this path, so a regression in it surfaces far from Produce. // Pinned here, next to the path that replaced it.