blob: eef024fc326864b1fa966ea524a003ae3a1a42c3 [file]
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
// End-to-end queue-consumer tests against a real scalable-topics broker: a produce -> consume
// round-trip over a Shared subscription. Gated on the PULSAR_ST_E2E environment variable so the
// ordinary (broker-free) unit-test run skips them; the docker harness sets it. The broker URL
// defaults to the standard test service and can be overridden with PULSAR_ST_E2E_SERVICE_URL.
#include <gtest/gtest.h>
#include <pulsar/st/Client.h>
#include <chrono>
#include <cstdint>
#include <cstdlib>
#include <ctime>
#include <set>
#include <string>
#include <thread>
#include <vector>
#include "lib/st/MessageIdImpl.h"
#include "tests/HttpHelper.h"
using namespace pulsar::st;
namespace {
bool e2eEnabled() { return std::getenv("PULSAR_ST_E2E") != nullptr; }
std::string serviceUrl() {
const char* url = std::getenv("PULSAR_ST_E2E_SERVICE_URL");
return url != nullptr ? url : "pulsar://localhost:6650";
}
std::string adminUrl() {
const char* url = std::getenv("PULSAR_ST_E2E_ADMIN_URL");
return url != nullptr ? url : "http://localhost:8080";
}
// A fresh topic name per test run so tests never collide with one another or with a topic left on a
// reused broker (the same convention the classic tests use).
std::string uniqueName(const std::string& prefix) {
static int counter = 0;
return prefix + "-" + std::to_string(std::time(nullptr)) + "-" + std::to_string(counter++);
}
std::string topicUrl(const std::string& name) { return "topic://public/default/" + name; }
// The admin REST base for a scalable topic under public/default.
std::string scalablePath(const std::string& name) {
return adminUrl() + "/admin/v2/scalable/public/default/" + name;
}
// Create a scalable topic with the given number of initial segments. Retries while the
// scalable-topics controller finishes coming up after broker start (only the first test waits).
bool createScalableTopic(const std::string& name, int numInitialSegments = 1) {
const std::string url = scalablePath(name) + "?numInitialSegments=" + std::to_string(numInitialSegments);
for (int attempt = 0; attempt < 30; attempt++) {
const int code = makePutRequest(url, "");
if (code >= 200 && code < 300) return true;
std::this_thread::sleep_for(std::chrono::seconds(1));
}
return false;
}
// Split a segment into two half-range children (POST .../split/{segmentId}).
bool splitSegment(const std::string& name, std::int64_t segmentId) {
const int code = makePostRequest(scalablePath(name) + "/split/" + std::to_string(segmentId), "");
return code >= 200 && code < 300;
}
// The segment id carried by a received message id — the fan-in stamps it on every message.
std::int64_t segmentIdOf(const MessageId& id) {
const auto& impl = MessageIdFactory::impl(id);
return impl ? impl->segmentId : MessageIdImpl::kNoSegment;
}
// Give each message plenty of time to arrive; a healthy broker delivers in milliseconds.
constexpr std::chrono::seconds kReceiveTimeout{20};
TEST(StQueueConsumerE2ETest, testProduceThenConsumeRoundTrip) {
if (!e2eEnabled()) GTEST_SKIP() << "set PULSAR_ST_E2E=1 to run against a scalable-topics broker";
const std::string name = uniqueName("st-e2e-queue");
ASSERT_TRUE(createScalableTopic(name)) << "failed to create scalable topic " << name;
const std::string topic = topicUrl(name);
auto clientResult = PulsarClient::builder().serviceUrl(serviceUrl()).build();
ASSERT_TRUE(clientResult) << clientResult.error();
PulsarClient client = std::move(clientResult).value();
// Subscribe first (Earliest) so the subscription and its per-segment cursors exist before we
// publish — every produced message is then guaranteed to be delivered.
auto consumerResult = client.newQueueConsumer(Schema<std::string>{})
.topic(topic)
.subscriptionName("sub")
.subscriptionInitialPosition(SubscriptionInitialPosition::Earliest)
.subscribe();
ASSERT_TRUE(consumerResult) << consumerResult.error();
QueueConsumer<std::string> consumer = std::move(consumerResult).value();
auto producerResult = client.newProducer(Schema<std::string>{}).topic(topic).create();
ASSERT_TRUE(producerResult) << producerResult.error();
Producer<std::string> producer = std::move(producerResult).value();
constexpr int kCount = 25;
std::set<std::string> produced;
for (int i = 0; i < kCount; i++) {
std::string value = "v-" + std::to_string(i);
auto sent = producer.newMessage().key("key-" + std::to_string(i % 4)).value(value).send();
ASSERT_TRUE(sent) << "send " << i << " failed: " << sent.error();
produced.insert(value);
}
ASSERT_TRUE(producer.flush());
// Receive exactly kCount messages; a Shared subscription gives no cross-segment order, so
// compare the received payloads as a set rather than a sequence.
std::set<std::string> received;
for (int i = 0; i < kCount; i++) {
auto message = consumer.receive(kReceiveTimeout);
ASSERT_TRUE(message) << "receive " << i << " failed: " << message.error();
EXPECT_GE(segmentIdOf(message->id()), 0) << "received message " << i << " has no real segment id";
received.insert(message->value());
consumer.acknowledge(message->id());
}
EXPECT_EQ(received, produced) << "consumed payloads did not match what was produced";
EXPECT_TRUE(producer.close());
EXPECT_TRUE(consumer.close());
EXPECT_TRUE(client.close());
}
// Consume from a topic split (via REST) into two active segments and assert the messages actually
// arrive from both — this is the fan-in the queue consumer exists for: one Shared subscription
// multiplexed across a per-segment classic consumer each, drained through the mux receive queue.
// The single-segment round-trip above never exercises multi-segment fan-in.
TEST(StQueueConsumerE2ETest, testConsumeAcrossSplitSegments) {
if (!e2eEnabled()) GTEST_SKIP() << "set PULSAR_ST_E2E=1 to run against a scalable-topics broker";
const std::string name = uniqueName("st-e2e-queue-split");
ASSERT_TRUE(createScalableTopic(name)) << "failed to create scalable topic " << name;
ASSERT_TRUE(splitSegment(name, 0)) << "failed to split segment 0 of " << name;
const std::string topic = topicUrl(name);
auto clientResult = PulsarClient::builder().serviceUrl(serviceUrl()).build();
ASSERT_TRUE(clientResult) << clientResult.error();
PulsarClient client = std::move(clientResult).value();
auto consumerResult = client.newQueueConsumer(Schema<std::string>{})
.topic(topic)
.subscriptionName("sub")
.subscriptionInitialPosition(SubscriptionInitialPosition::Earliest)
.subscribe();
ASSERT_TRUE(consumerResult) << consumerResult.error();
QueueConsumer<std::string> consumer = std::move(consumerResult).value();
auto producerResult = client.newProducer(Schema<std::string>{}).topic(topic).create();
ASSERT_TRUE(producerResult) << producerResult.error();
Producer<std::string> producer = std::move(producerResult).value();
// 60 distinct keys over two half-range segments hit both with overwhelming probability.
constexpr int kCount = 60;
std::set<std::string> produced;
for (int i = 0; i < kCount; i++) {
std::string value = "v-" + std::to_string(i);
auto sent = producer.newMessage().key("key-" + std::to_string(i)).value(value).send();
ASSERT_TRUE(sent) << "send " << i << " failed: " << sent.error();
produced.insert(value);
}
ASSERT_TRUE(producer.flush());
std::set<std::string> received;
std::set<std::int64_t> segments;
for (int i = 0; i < kCount; i++) {
auto message = consumer.receive(kReceiveTimeout);
ASSERT_TRUE(message) << "receive " << i << " failed: " << message.error();
segments.insert(segmentIdOf(message->id()));
received.insert(message->value());
consumer.acknowledge(message->id());
}
EXPECT_EQ(received, produced) << "consumed payloads did not match what was produced";
EXPECT_GE(segments.size(), 2u) << "messages did not fan in from both split segments";
EXPECT_TRUE(producer.close());
EXPECT_TRUE(consumer.close());
EXPECT_TRUE(client.close());
}
// The headline sealed-segment scenario: a split seals the parent WITHOUT migrating its backlog, so
// the messages produced before the split are only drainable through the sealed segment. Produce
// first, split, then consume: every pre-split message must still arrive (through the sealed
// parent), and the acks must stick — reattaching a second consumer on the same subscription gets
// nothing back. The count deliberately exceeds both the classic prefetch queue and the mux queue
// capacity (1000), so the drain also exercises back-pressure and the broker's end-of-topic
// arriving while messages are still unacked in the application's hands.
TEST(StQueueConsumerE2ETest, testDrainSealedSegmentBacklog) {
if (!e2eEnabled()) GTEST_SKIP() << "set PULSAR_ST_E2E=1 to run against a scalable-topics broker";
const std::string name = uniqueName("st-e2e-queue-drain");
ASSERT_TRUE(createScalableTopic(name)) << "failed to create scalable topic " << name;
const std::string topic = topicUrl(name);
auto clientResult = PulsarClient::builder().serviceUrl(serviceUrl()).build();
ASSERT_TRUE(clientResult) << clientResult.error();
PulsarClient client = std::move(clientResult).value();
// Create the durable subscription up front (and detach), so the backlog produced next is
// retained for it.
{
auto subscriberResult = client.newQueueConsumer(Schema<std::string>{})
.topic(topic)
.subscriptionName("sub")
.subscriptionInitialPosition(SubscriptionInitialPosition::Earliest)
.subscribe();
ASSERT_TRUE(subscriberResult) << subscriberResult.error();
QueueConsumer<std::string> subscriber = std::move(subscriberResult).value();
ASSERT_TRUE(subscriber.close());
}
auto producerResult = client.newProducer(Schema<std::string>{}).topic(topic).create();
ASSERT_TRUE(producerResult) << producerResult.error();
Producer<std::string> producer = std::move(producerResult).value();
// Publish the backlog in bounded async waves (the per-segment pending-send queue is finite).
constexpr int kCount = 1200;
constexpr int kWave = 400;
std::set<std::string> produced;
for (int base = 0; base < kCount; base += kWave) {
std::vector<Future<MessageId>> wave;
wave.reserve(kWave);
for (int i = base; i < base + kWave; i++) {
std::string value = "v-" + std::to_string(i);
wave.push_back(producer.newMessage().key("key-" + std::to_string(i)).value(value).sendAsync());
produced.insert(std::move(value));
}
for (int i = 0; i < kWave; i++) {
auto sent = wave[i].get();
ASSERT_TRUE(sent) << "send " << (base + i) << " failed: " << sent.error();
}
}
ASSERT_TRUE(producer.flush());
ASSERT_TRUE(producer.close());
// Seal the parent: its backlog stays behind in the sealed segment.
ASSERT_TRUE(splitSegment(name, 0)) << "failed to split segment 0 of " << name;
// Drain the sealed segment through a fresh consumer, acking everything.
auto consumerResult = client.newQueueConsumer(Schema<std::string>{})
.topic(topic)
.subscriptionName("sub")
.subscriptionInitialPosition(SubscriptionInitialPosition::Earliest)
.subscribe();
ASSERT_TRUE(consumerResult) << consumerResult.error();
QueueConsumer<std::string> consumer = std::move(consumerResult).value();
std::set<std::string> received;
for (int i = 0; i < kCount; i++) {
auto message = consumer.receive(kReceiveTimeout);
ASSERT_TRUE(message) << "receive " << i << " failed: " << message.error();
EXPECT_EQ(segmentIdOf(message->id()), 0) << "message " << i << " did not come from the sealed parent";
received.insert(std::string(message->value()));
consumer.acknowledge(message->id());
}
EXPECT_EQ(received, produced) << "the sealed segment's backlog did not drain completely";
ASSERT_TRUE(consumer.close());
// The acks must have stuck: a second consumer on the same subscription gets nothing back.
auto verifierResult = client.newQueueConsumer(Schema<std::string>{})
.topic(topic)
.subscriptionName("sub")
.subscriptionInitialPosition(SubscriptionInitialPosition::Earliest)
.subscribe();
ASSERT_TRUE(verifierResult) << verifierResult.error();
QueueConsumer<std::string> verifier = std::move(verifierResult).value();
auto redelivered = verifier.receive(std::chrono::seconds(3));
ASSERT_FALSE(redelivered) << "acks were lost: message \"" << redelivered->value()
<< "\" was redelivered after the drain";
EXPECT_EQ(redelivered.error().result, pulsar::ResultTimeout);
EXPECT_TRUE(verifier.close());
EXPECT_TRUE(client.close());
}
} // namespace