ISSUE #2311: Adds support to decode metadata in the API that list ledgers
Descriptions of the changes in this PR:
### Motivation
The current list ledgers API output the metadata in a serialized binary format, which is not friendly to human operators and external tools, and is not consistent with the output of the [API that gets the metadata](https://bookkeeper.apache.org/docs/4.9.2/admin/http/#endpoint-apiv1ledgermetadataledger_idledger_id).
### Changes
The PR adds a parameter called `decode_meta`, and output the ledger metadata in decoded format when the parameter presents and the value of it is 'true'.
Master Issue: https://github.com/apache/bookkeeper/issues/2311
Reviewers: Sijie Guo <None>, Enrico Olivelli <eolivelli@gmail.com>
This closes #2312 from fantapsody/decode_metadata, closes #2311
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListLedgerService.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListLedgerService.java
index 1683fd4..07ea5af 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListLedgerService.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/server/http/service/ListLedgerService.java
@@ -66,10 +66,14 @@
static final int LIST_LEDGER_BATCH_SIZE = 100;
private void keepLedgerMetadata(long ledgerId, CompletableFuture<Versioned<LedgerMetadata>> future,
- LinkedHashMap<String, String> output)
+ LinkedHashMap<String, Object> output, boolean decodeMeta)
throws Exception {
LedgerMetadata md = future.get().getValue();
- output.put(Long.valueOf(ledgerId).toString(), new String(serDe.serialize(md), UTF_8));
+ if (decodeMeta) {
+ output.put(Long.valueOf(ledgerId).toString(), md);
+ } else {
+ output.put(Long.valueOf(ledgerId).toString(), new String(serDe.serialize(md), UTF_8));
+ }
}
@Override
@@ -84,6 +88,10 @@
&& params.containsKey("print_metadata")
&& params.get("print_metadata").equals("true");
+ // do not decode meta by default for backward compatibility
+ boolean decodeMeta = (params != null)
+ && params.getOrDefault("decode_meta", "false").equals("true");
+
// Page index should start from 1;
int pageIndex = (printMeta && params.containsKey("page"))
? Integer.parseInt(params.get("page")) : -1;
@@ -93,7 +101,7 @@
LedgerManager.LedgerRangeIterator iter = manager.getLedgerRanges(0);
// output <ledgerId: ledgerMetadata>
- LinkedHashMap<String, String> output = Maps.newLinkedHashMap();
+ LinkedHashMap<String, Object> output = Maps.newLinkedHashMap();
// futures for readLedgerMetadata for each page.
Map<Long, CompletableFuture<Versioned<LedgerMetadata>>> futures =
new LinkedHashMap<>(LIST_LEDGER_BATCH_SIZE);
@@ -121,13 +129,13 @@
}
if (futures.size() >= LIST_LEDGER_BATCH_SIZE) {
for (Map.Entry<Long, CompletableFuture<Versioned<LedgerMetadata>> > e : futures.entrySet()) {
- keepLedgerMetadata(e.getKey(), e.getValue(), output);
+ keepLedgerMetadata(e.getKey(), e.getValue(), output, decodeMeta);
}
futures.clear();
}
}
for (Map.Entry<Long, CompletableFuture<Versioned<LedgerMetadata>> > e : futures.entrySet()) {
- keepLedgerMetadata(e.getKey(), e.getValue(), output);
+ keepLedgerMetadata(e.getKey(), e.getValue(), output, decodeMeta);
}
futures.clear();
} else {
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/service/ListLedgerServiceTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/service/ListLedgerServiceTest.java
new file mode 100644
index 0000000..88fb3ff
--- /dev/null
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/server/http/service/ListLedgerServiceTest.java
@@ -0,0 +1,168 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.bookkeeper.server.http.service;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableMap;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.bookkeeper.client.BookKeeper;
+import org.apache.bookkeeper.client.LedgerHandle;
+import org.apache.bookkeeper.client.api.LedgerMetadata;
+import org.apache.bookkeeper.http.HttpServer;
+import org.apache.bookkeeper.http.service.HttpServiceRequest;
+import org.apache.bookkeeper.http.service.HttpServiceResponse;
+import org.apache.bookkeeper.net.BookieSocketAddress;
+import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
+import org.apache.commons.lang3.RandomUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit tests for {@link ListLedgerService}.
+ */
+public class ListLedgerServiceTest extends BookKeeperClusterTestCase {
+ private final ObjectMapper mapper = new ObjectMapper();
+ private ListLedgerService listLedgerService;
+
+ public ListLedgerServiceTest() {
+ super(1);
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+ listLedgerService = new ListLedgerService(bsConfs.get(0), bs.get(0));
+ }
+
+ @Test
+ public void testEmptyList() throws Exception {
+ HttpServiceResponse response = listLedgerService.handle(new HttpServiceRequest());
+ assertEquals(response.getStatusCode(), HttpServer.StatusCode.OK.getValue());
+ JsonNode json = mapper.readTree(response.getBody());
+ assertEquals(0, json.size());
+ }
+
+ @Test
+ public void testListLedgers() throws Exception {
+ int ledgerNum = RandomUtils.nextInt(1, 10);
+ Map<Long, LedgerMetadata> ledgers = new HashMap<>();
+ for (int i = 0; i < ledgerNum; i++) {
+ LedgerHandle ledger = bkc.createLedger(1, 1, 1, BookKeeper.DigestType.CRC32, new byte[0]);
+ ledgers.put(ledger.getId(), ledger.getLedgerMetadata());
+ ledger.close();
+ }
+
+ HttpServiceResponse response = listLedgerService.handle(new HttpServiceRequest());
+ assertEquals(response.getStatusCode(), HttpServer.StatusCode.OK.getValue());
+ JsonNode json = mapper.readTree(response.getBody());
+ assertEquals(ledgerNum, json.size());
+
+ json.fieldNames().forEachRemaining(field -> {
+ assertTrue(ledgers.containsKey(Long.parseLong(field)));
+ assertTrue(json.get(field).isNull());
+ });
+ }
+
+ @Test
+ public void testListLedgersWithMetadata() throws Exception {
+ int ledgerNum = RandomUtils.nextInt(1, 10);
+ Map<Long, LedgerMetadata> ledgers = new HashMap<>();
+ for (int i = 0; i < ledgerNum; i++) {
+ LedgerHandle ledger = bkc.createLedger(1, 1, 1, BookKeeper.DigestType.CRC32, new byte[0]);
+ ledger.close();
+ ledgers.put(ledger.getId(), ledger.getLedgerMetadata());
+ }
+
+ HttpServiceResponse response = listLedgerService.handle(new HttpServiceRequest(null, HttpServer.Method.GET,
+ ImmutableMap.of("print_metadata", "true")));
+ assertEquals(response.getStatusCode(), HttpServer.StatusCode.OK.getValue());
+ JsonNode json = mapper.readTree(response.getBody());
+ assertEquals(ledgerNum, json.size());
+
+ json.fieldNames().forEachRemaining(field -> {
+ LedgerMetadata meta = ledgers.get(Long.parseLong(field));
+ assertNotNull(meta);
+ assertFalse(json.get(field).isNull());
+ });
+ }
+
+ @Test
+ public void testListLedgersWithMetadataDecoded() throws Exception {
+ int ledgerNum = RandomUtils.nextInt(1, 10);
+ Map<Long, LedgerMetadata> ledgers = new HashMap<>();
+ for (int i = 0; i < ledgerNum; i++) {
+ LedgerHandle ledger = bkc.createLedger(1, 1, 1, BookKeeper.DigestType.CRC32, new byte[0],
+ ImmutableMap.of("test_key", "test_value".getBytes()));
+ ledger.close();
+ ledgers.put(ledger.getId(), ledger.getLedgerMetadata());
+ }
+
+ HttpServiceResponse response = listLedgerService.handle(new HttpServiceRequest(null, HttpServer.Method.GET,
+ ImmutableMap.of("print_metadata", "true", "decode_meta", "true")));
+ assertEquals(response.getStatusCode(), HttpServer.StatusCode.OK.getValue());
+ JsonNode json = mapper.readTree(response.getBody());
+ assertEquals(ledgerNum, json.size());
+
+ json.fieldNames().forEachRemaining(field -> {
+ LedgerMetadata meta = ledgers.get(Long.parseLong(field));
+ assertNotNull(meta);
+ JsonNode node = json.get(field);
+ assertEquals(meta.getMetadataFormatVersion(), node.get("metadataFormatVersion").asInt());
+ assertEquals(meta.getEnsembleSize(), node.get("ensembleSize").asInt());
+ assertEquals(meta.getWriteQuorumSize(), node.get("writeQuorumSize").asInt());
+ assertEquals(meta.getAckQuorumSize(), node.get("ackQuorumSize").asInt());
+ assertEquals(meta.getCToken(), node.get("ctoken").asLong());
+// assertEquals(meta.getCtime(), node.get("ctime").asLong());
+ assertEquals(meta.getState().name(), node.get("state").asText());
+ assertEquals(meta.isClosed(), node.get("closed").asBoolean());
+ assertEquals(meta.getLength(), node.get("length").asLong());
+ assertEquals(meta.getLastEntryId(), node.get("lastEntryId").asLong());
+ assertEquals(meta.getDigestType().name(), node.get("digestType").asText());
+ assertEquals(new String(meta.getPassword()), node.get("password").asText());
+
+ for (Map.Entry<String, byte[]> entry : meta.getCustomMetadata().entrySet()) {
+ JsonNode data = node.get("customMetadata").get(entry.getKey());
+ assertArrayEquals(entry.getValue(), Base64.getDecoder().decode(data.asText()));
+ }
+
+ for (Map.Entry<Long, ? extends List<BookieSocketAddress>> entry : meta.getAllEnsembles().entrySet()) {
+ JsonNode members = node.get("allEnsembles")
+ .get(String.valueOf(entry.getKey()));
+ assertEquals(1, entry.getValue().size());
+ assertEquals(entry.getValue().size(), members.size());
+ JsonNode member = members.get(0);
+ assertEquals(entry.getValue().get(0).getHostName(), member.get("hostName").asText());
+ assertEquals(entry.getValue().get(0).getPort(), member.get("port").asInt());
+ }
+ });
+ }
+}
