IGNITE-24287 Add SystemViewHandler (#298)
diff --git a/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsReportBuilder.java b/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsReportBuilder.java index 1560ed5..fdff05d 100644 --- a/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsReportBuilder.java +++ b/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsReportBuilder.java
@@ -44,6 +44,7 @@ import org.apache.ignite.internal.performancestatistics.handlers.ComputeHandler; import org.apache.ignite.internal.performancestatistics.handlers.IgnitePerformanceStatisticsHandler; import org.apache.ignite.internal.performancestatistics.handlers.QueryHandler; +import org.apache.ignite.internal.performancestatistics.handlers.SystemViewHandler; import org.apache.ignite.internal.performancestatistics.handlers.TransactionsHandler; import org.apache.ignite.internal.processors.performancestatistics.FilePerformanceStatisticsReader; import org.apache.ignite.internal.util.typedef.internal.A; @@ -118,7 +119,8 @@ new CacheOperationsHandler(), new TransactionsHandler(), new ComputeHandler(), - new ClusterInfoHandler() + new ClusterInfoHandler(), + new SystemViewHandler() }; new FilePerformanceStatisticsReader(handlers).read(Collections.singletonList(new File(filesDir)));
diff --git a/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/handlers/IgnitePerformanceStatisticsHandler.java b/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/handlers/IgnitePerformanceStatisticsHandler.java index 3da8e12..81e87cb 100644 --- a/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/handlers/IgnitePerformanceStatisticsHandler.java +++ b/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/handlers/IgnitePerformanceStatisticsHandler.java
@@ -17,6 +17,7 @@ package org.apache.ignite.internal.performancestatistics.handlers; +import java.util.List; import java.util.Map; import java.util.UUID; import com.fasterxml.jackson.databind.JsonNode; @@ -43,6 +44,11 @@ } /** {@inheritDoc} */ + @Override default void systemView(UUID nodeId, String viewName, List<String> schema, List<Object> data) { + // No-op. + } + + /** {@inheritDoc} */ @Override default void cacheOperation(UUID nodeId, OperationType type, int cacheId, long startTime, long duration) { // No-op. }
diff --git a/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/handlers/PrintHandler.java b/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/handlers/PrintHandler.java index 5e5e1ae..021bfcd 100644 --- a/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/handlers/PrintHandler.java +++ b/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/handlers/PrintHandler.java
@@ -17,10 +17,13 @@ package org.apache.ignite.internal.performancestatistics.handlers; +import java.io.IOException; import java.io.PrintStream; import java.util.BitSet; +import java.util.List; import java.util.Set; import java.util.UUID; +import com.fasterxml.jackson.core.JsonGenerator; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType; import org.apache.ignite.internal.processors.performancestatistics.OperationType; import org.apache.ignite.internal.processors.performancestatistics.PerformanceStatisticsHandler; @@ -29,6 +32,7 @@ import org.apache.ignite.lang.IgniteUuid; import org.jetbrains.annotations.Nullable; +import static org.apache.ignite.internal.performancestatistics.util.Utils.MAPPER; import static org.apache.ignite.internal.performancestatistics.util.Utils.printEscaped; import static org.apache.ignite.internal.processors.performancestatistics.OperationType.CACHE_START; import static org.apache.ignite.internal.processors.performancestatistics.OperationType.CHECKPOINT; @@ -38,6 +42,7 @@ import static org.apache.ignite.internal.processors.performancestatistics.OperationType.QUERY_PROPERTY; import static org.apache.ignite.internal.processors.performancestatistics.OperationType.QUERY_READS; import static org.apache.ignite.internal.processors.performancestatistics.OperationType.QUERY_ROWS; +import static org.apache.ignite.internal.processors.performancestatistics.OperationType.SYSTEM_VIEW_ROW; import static org.apache.ignite.internal.processors.performancestatistics.OperationType.TASK; import static org.apache.ignite.internal.processors.performancestatistics.OperationType.TX_COMMIT; import static org.apache.ignite.internal.processors.performancestatistics.OperationType.TX_ROLLBACK; @@ -61,6 +66,9 @@ /** Cache identifiers to filter the output. */ @Nullable private final Set<Integer> cacheIds; + /** Generator for system view printing to escape special characters. */ + private final JsonGenerator sysViewGenerator; + /** * @param ps Print stream. * @param ops Set of operations to print. @@ -68,12 +76,13 @@ * @param to The maximum operation start time to filter the output. * @param cacheIds Cache identifiers to filter the output. */ - public PrintHandler(PrintStream ps, @Nullable BitSet ops, long from, long to, @Nullable Set<Integer> cacheIds) { + public PrintHandler(PrintStream ps, @Nullable BitSet ops, long from, long to, @Nullable Set<Integer> cacheIds) throws IOException { this.ps = ps; this.ops = ops; this.from = from; this.to = to; this.cacheIds = cacheIds; + this.sysViewGenerator = MAPPER.getFactory().createGenerator(this.ps); } /** {@inheritDoc} */ @@ -340,6 +349,28 @@ ps.println("}"); } + /** {@inheritDoc} */ + @Override public void systemView(UUID nodeId, String viewName, List<String> schema, List<Object> data) { + try { + sysViewGenerator.writeStartObject(); + + sysViewGenerator.writeStringField("op", SYSTEM_VIEW_ROW.name()); + sysViewGenerator.writeStringField("nodeId", nodeId.toString()); + sysViewGenerator.writeStringField("view", viewName); + + for (int i = 0; i < schema.size(); i++) + sysViewGenerator.writeStringField(schema.get(i), String.valueOf(data.get(i))); + + sysViewGenerator.writeEndObject(); + + sysViewGenerator.flush(); + } catch (IOException e) { + throw new RuntimeException("Unable to write view " + viewName + ".", e); + } + + ps.println(); + } + /** @return {@code True} if the operation should be skipped. */ private boolean skip(OperationType op) { return !(ops == null || ops.get(op.id()));
diff --git a/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/handlers/SystemViewHandler.java b/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/handlers/SystemViewHandler.java new file mode 100644 index 0000000..d51cf95 --- /dev/null +++ b/modules/performance-statistics-ext/src/main/java/org/apache/ignite/internal/performancestatistics/handlers/SystemViewHandler.java
@@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.performancestatistics.handlers; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import static org.apache.ignite.internal.performancestatistics.util.Utils.MAPPER; + +/** + * Builds JSON with system view tables. + * <p> + * Example: + * <pre> + * { + * $nodeId: { + * $systemViewTable: + * schema: [ $column1, $column2 ], + * data:[ + * [ $value1, $value2 ], + * [ $value3, $value4 ] + * ] + * } + * } + * </pre> + */ +public class SystemViewHandler implements IgnitePerformanceStatisticsHandler { + /** */ + private final ObjectNode resNode = MAPPER.createObjectNode(); + + /** {@inheritDoc} */ + @Override public void systemView(UUID nodeId, String viewName, List<String> schema, List<Object> data) { + JsonNode gridNode = resNode.get(nodeId.toString()); + + if (gridNode == null) { + gridNode = MAPPER.createObjectNode(); + resNode.set(nodeId.toString(), gridNode); + } + + JsonNode viewNode = gridNode.get(viewName); + + if (viewNode == null) { + viewNode = MAPPER.createObjectNode(); + ((ObjectNode)gridNode).set(viewName, viewNode); + + ArrayNode schemaNode = MAPPER.createArrayNode(); + schema.forEach(schemaNode::add); + ((ObjectNode)viewNode).set("schema", schemaNode); + + ArrayNode dataNode = MAPPER.createArrayNode(); + ((ObjectNode)viewNode).set("data", dataNode); + } + + ArrayNode dataNode = (ArrayNode)viewNode.get("data"); + + ArrayNode rowNode = MAPPER.createArrayNode(); + data.forEach(attr -> rowNode.add(String.valueOf(attr))); + dataNode.add(rowNode); + } + + /** {@inheritDoc} */ + @Override public Map<String, JsonNode> results() { + return Collections.singletonMap("systemView", resNode); + } +}
diff --git a/modules/performance-statistics-ext/src/test/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsPrinterTest.java b/modules/performance-statistics-ext/src/test/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsPrinterTest.java index 8aa5a54..07acd58 100644 --- a/modules/performance-statistics-ext/src/test/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsPrinterTest.java +++ b/modules/performance-statistics-ext/src/test/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsPrinterTest.java
@@ -20,15 +20,21 @@ import java.io.BufferedReader; import java.io.File; import java.io.FileReader; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType; import org.apache.ignite.internal.processors.performancestatistics.FilePerformanceStatisticsWriter; import org.apache.ignite.internal.processors.performancestatistics.OperationType; @@ -38,6 +44,8 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteUuid; import org.apache.ignite.logger.java.JavaLogger; +import org.apache.ignite.testframework.ListeningTestLogger; +import org.apache.ignite.testframework.LogListener; import org.apache.ignite.testframework.junits.GridTestKernalContext; import org.junit.After; import org.junit.Before; @@ -55,6 +63,7 @@ import static org.apache.ignite.internal.processors.performancestatistics.OperationType.QUERY_PROPERTY; import static org.apache.ignite.internal.processors.performancestatistics.OperationType.QUERY_READS; import static org.apache.ignite.internal.processors.performancestatistics.OperationType.QUERY_ROWS; +import static org.apache.ignite.internal.processors.performancestatistics.OperationType.SYSTEM_VIEW_ROW; import static org.apache.ignite.internal.processors.performancestatistics.OperationType.TASK; import static org.apache.ignite.internal.processors.performancestatistics.OperationType.TX_COMMIT; import static org.apache.ignite.internal.processors.performancestatistics.OperationType.TX_ROLLBACK; @@ -70,6 +79,9 @@ /** Test node ID. */ private static final UUID NODE_ID = UUID.randomUUID(); + /** Timeout. */ + private static final int TIMEOUT = 10_000; + /** */ @Before public void beforeTest() throws Exception { @@ -84,6 +96,47 @@ /** @throws Exception If failed. */ @Test + public void testSystemViewOperation() throws Exception { + IgniteConfiguration cfg = new IgniteConfiguration(); + cfg.setNodeId(NODE_ID); + + ListeningTestLogger logger = new ListeningTestLogger(new JavaLogger()); + cfg.setGridLogger(logger); + + LogListener lsnr = LogListener.matches("Finished writing system views to performance statistics file:").build(); + logger.registerListener(lsnr); + + try (IgniteEx ign = (IgniteEx)Ignition.start(cfg)) { + IgniteCache<String, String> cache = ign.createCache("myCache"); + cache.put("key", "value"); + + ign.context().performanceStatistics().startCollectStatistics(); + + assertTrue("Performance statistics writer did not finish.", waitForCondition(lsnr::check, TIMEOUT)); + + ign.context().performanceStatistics().stopCollectStatistics(); + } + + AtomicBoolean hasSysCache = new AtomicBoolean(false); + AtomicBoolean hasMyCache = new AtomicBoolean(false); + + List<String> args = new ArrayList<>(); + args.add("--ops"); + args.add(SYSTEM_VIEW_ROW.name()); + + readStatistics(args, json -> { + if ("cacheGroups".equals(json.get("view").asText())) { + hasSysCache.compareAndSet(false, "ignite-sys-cache".equals(json.get("cacheGroupName").asText())); + hasMyCache.compareAndSet(false, "myCache".equals(json.get("cacheGroupName").asText())); + } + }); + + assertTrue("Could not find system cache", hasSysCache.get()); + assertTrue("Could not find myCache", hasMyCache.get()); + } + + /** @throws Exception If failed. */ + @Test public void testOperationsFilter() throws Exception { createStatistics(writer -> { writer.cacheStart(0, "cache");
diff --git a/modules/performance-statistics-ext/src/test/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsReportSelfTest.java b/modules/performance-statistics-ext/src/test/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsReportSelfTest.java index 517b777..5285895 100644 --- a/modules/performance-statistics-ext/src/test/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsReportSelfTest.java +++ b/modules/performance-statistics-ext/src/test/java/org/apache/ignite/internal/performancestatistics/PerformanceStatisticsReportSelfTest.java
@@ -18,12 +18,15 @@ package org.apache.ignite.internal.performancestatistics; import java.io.File; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteException; @@ -40,6 +43,7 @@ import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.performancestatistics.handlers.QueryHandler; +import org.apache.ignite.internal.performancestatistics.handlers.SystemViewHandler; import org.apache.ignite.internal.processors.cache.CacheObject; import org.apache.ignite.internal.processors.cache.CacheObjectImpl; import org.apache.ignite.internal.processors.cache.IgniteInternalCache; @@ -251,6 +255,53 @@ } /** */ + @Test + public void testSystemViewHandler() { + SystemViewHandler sysViewHandler = new SystemViewHandler(); + + int nodesNumber = 10; + int viewsNumber = 10; + + List<String> schema = new ArrayList<>(); + schema.add("col1"); + schema.add("col2"); + + for (int id = 0; id < nodesNumber; id++) { + UUID nodeId = new UUID(0, id); + + for (int i = 0; i < viewsNumber; i++) { + List<Object> data = new ArrayList<>(); + data.add(i); + data.add(i); + + sysViewHandler.systemView(nodeId, "view" + i, schema, data); + } + } + + JsonNode res = sysViewHandler.results().get("systemView"); + + for (int id = 0; id < nodesNumber; id++) { + UUID nodeId = new UUID(0, id); + + JsonNode nodeRes = res.get(nodeId.toString()); + + for (int i = 0; i < viewsNumber; i++) { + ObjectNode view = (ObjectNode)nodeRes.get("view" + i); + + ArrayNode schemaNode = (ArrayNode)view.get("schema"); + + assertEquals(schemaNode.get(0).asText(), schema.get(0)); + assertEquals(schemaNode.get(1).asText(), schema.get(1)); + + ArrayNode rowNode = (ArrayNode)view.get("data").get(0); + + assertEquals(Integer.toString(i), rowNode.get(0).asText()); + assertEquals(Integer.toString(i), rowNode.get(1).asText()); + } + } + } + + /** */ private static class TaskWithoutJobs extends ComputeTaskAdapter<Object, Object> { /** {@inheritDoc} */ @Override public @NotNull Map<? extends ComputeJob, ClusterNode> map(