fix healthmgr unit tests (#3018)
* addBUILDforhealthmgr
* fixhealthmgrunit
diff --git a/heron/healthmgr/tests/java/BUILD b/heron/healthmgr/tests/java/BUILD
new file mode 100644
index 0000000..af6b53e
--- /dev/null
+++ b/heron/healthmgr/tests/java/BUILD
@@ -0,0 +1,86 @@
+test_deps_files = \
+ heron_java_proto_files() + [
+ "//heron/api/src/java:api-java-low-level",
+ "//heron/api/src/java:classification",
+ "//heron/common/src/java:basics-java",
+ "//heron/common/src/java:config-java",
+ "//heron/common/src/java:network-java",
+ "//heron/common/src/java:utils-java",
+ "//heron/healthmgr/src/java:healthmgr-java",
+ "//heron/packing/src/java:roundrobin-packing",
+ "//heron/scheduler-core/src/java:scheduler-java",
+ "//heron/spi/src/java:common-spi-java",
+ "//heron/spi/src/java:metricsmgr-spi-java",
+ "//heron/spi/src/java:packing-spi-java",
+ "//heron/spi/src/java:statemgr-spi-java",
+ "//heron/spi/src/java:utils-spi-java",
+ "//third_party/java:jackson",
+ "//third_party/java:junit4",
+ "//third_party/java:mockito",
+ "//third_party/java:powermock",
+ "@aopalliance_aopalliance//jar",
+ "@com_fasterxml_jackson_jaxrs_jackson_jaxrs_base//jar",
+ "@com_fasterxml_jackson_jaxrs_jackson_jaxrs_json_provider//jar",
+ "@com_fasterxml_jackson_module_jackson_module_jaxb_annotations//jar",
+ "@com_google_guava_guava//jar",
+ "@com_google_inject_extensions_guice_assistedinject//jar",
+ "@com_google_inject_guice//jar",
+ "@com_jayway_jsonpath//jar",
+ "@com_microsoft_dhalion//jar",
+ "@commons_cli_commons_cli//jar",
+ "@it_unimi_dsi_fastutil//jar",
+ "@javax_annotation_javax_annotation_api//jar",
+ "@javax_inject_javax_inject//jar",
+ "@javax_ws_rs_javax_ws_rs_api//jar",
+ "@net_minidev_json_smart//jar",
+ "@org_apache_commons_commons_math3//jar",
+ "@org_glassfish_hk2_hk2_api//jar",
+ "@org_glassfish_hk2_hk2_locator//jar",
+ "@org_glassfish_hk2_hk2_utils//jar",
+ "@org_glassfish_jersey_bundles_repackaged_jersey_guava//jar",
+ "@org_glassfish_jersey_core_jersey_client//jar",
+ "@org_glassfish_jersey_core_jersey_common//jar",
+ "@org_glassfish_jersey_ext_jersey_entity_filtering//jar",
+ "@org_glassfish_jersey_media_jersey_media_json_jackson//jar",
+ "@org_roaringbitmap//jar",
+ "@org_yaml_snakeyaml//jar",
+ "@tech_tablesaw//jar",
+ ]
+
+java_library(
+ name = "healthmgr-tests",
+ srcs = glob(["**/*.java"]) + ["//heron/healthmgr/src/java:healthmgr-main"],
+ deps = test_deps_files,
+)
+
+java_tests(
+ test_classes = [
+ "org.apache.heron.healthmgr.HealthManagerTest",
+ "org.apache.heron.healthmgr.HealthPolicyConfigReaderTest",
+
+ "org.apache.heron.healthmgr.common.PackingPlanProviderTest",
+
+ "org.apache.heron.healthmgr.detectors.BackPressureDetectorTest",
+ "org.apache.heron.healthmgr.detectors.GrowingWaitQueueDetectorTest",
+ "org.apache.heron.healthmgr.detectors.LargeWaitQueueDetectorTest",
+ "org.apache.heron.healthmgr.detectors.ProcessingRateSkewDetectorTest",
+ "org.apache.heron.healthmgr.detectors.WaitQueueSkewDetectorTest",
+
+ "org.apache.heron.healthmgr.diagnosers.DataSkewDiagnoserTest",
+ "org.apache.heron.healthmgr.diagnosers.SlowInstanceDiagnoserTest",
+ "org.apache.heron.healthmgr.diagnosers.UnderProvisioningDiagnoserTest",
+
+ "org.apache.heron.healthmgr.resolvers.ScaleUpResolverTest",
+
+ "org.apache.heron.healthmgr.sensors.BackPressureSensorTest",
+ "org.apache.heron.healthmgr.sensors.BufferSizeSensorTest",
+ "org.apache.heron.healthmgr.sensors.ExecuteCountSensorTest",
+ "org.apache.heron.healthmgr.sensors.MetricsCacheMetricsProviderTest",
+# "org.apache.heron.healthmgr.sensors.TrackerMetricsProviderTest",
+ ],
+ runtime_deps = [
+ ":healthmgr-tests",
+ ],
+ data = ["//heron/config/src/yaml:test-config-internals-yaml"],
+ size = "small",
+)
diff --git a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/HealthManagerTest.java b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/HealthManagerTest.java
index 40e36f8..5d20845 100644
--- a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/HealthManagerTest.java
+++ b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/HealthManagerTest.java
@@ -68,8 +68,9 @@
.build();
when(adaptor.getSchedulerLocation(anyString())).thenReturn(schedulerLocation);
+ HealthManagerMetrics publishingMetrics = mock(HealthManagerMetrics.class);
AbstractModule baseModule = HealthManager
- .buildBaseModule("127.0.0.1", TrackerMetricsProvider.class.getName());
+ .buildBaseModule("127.0.0.1", TrackerMetricsProvider.class.getName(), publishingMetrics);
HealthManager healthManager = new HealthManager(config, baseModule);
diff --git a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/common/TopologyProviderTest.java b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/common/TopologyProviderTest.java
deleted file mode 100644
index 4b373ba..0000000
--- a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/common/TopologyProviderTest.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.heron.healthmgr.common;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import com.microsoft.dhalion.events.EventManager;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import org.apache.heron.api.Config;
-import org.apache.heron.api.generated.TopologyAPI.Topology;
-import org.apache.heron.common.utils.topology.TopologyTests;
-import org.apache.heron.healthmgr.common.HealthManagerEvents.TopologyUpdate;
-import org.apache.heron.proto.system.PhysicalPlans;
-import org.apache.heron.spi.statemgr.SchedulerStateManagerAdaptor;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-public class TopologyProviderTest {
- private String topology = "topology";
- private EventManager eventManager = new EventManager();
-
- @Test
- public void fetchesAndCachesPackingFromStateMgr() {
- SchedulerStateManagerAdaptor adaptor = getMockSchedulerStateManagerAdaptor();
- TopologyProvider provider = new TopologyProvider(adaptor, eventManager, topology);
- Assert.assertEquals(2, provider.get().getBoltsCount());
-
- // once fetched it is cached
- provider.get();
- verify(adaptor, times(1)).getPhysicalPlan(topology);
- }
-
- @Test
- public void refreshesPackingPlanOnUpdate() {
- SchedulerStateManagerAdaptor adaptor = getMockSchedulerStateManagerAdaptor();
-
- TopologyProvider provider = new TopologyProvider(adaptor, eventManager, topology);
- Assert.assertEquals(2, provider.get().getBoltsCount());
-
- // once fetched it is cached
- provider.onEvent(new TopologyUpdate(null, null));
- provider.get();
- verify(adaptor, times(2)).getPhysicalPlan(topology);
- }
-
- private SchedulerStateManagerAdaptor getMockSchedulerStateManagerAdaptor() {
- Topology testTopology
- = TopologyTests.createTopology(topology, new Config(), getSpouts(), getBolts());
- PhysicalPlans.PhysicalPlan plan =
- PhysicalPlans.PhysicalPlan.newBuilder().setTopology(testTopology).build();
- SchedulerStateManagerAdaptor adaptor = mock(SchedulerStateManagerAdaptor.class);
- when(adaptor.getPhysicalPlan(topology)).thenReturn(plan);
- return adaptor;
- }
-
- @Test
- public void providesBoltNames() {
- SchedulerStateManagerAdaptor adaptor = getMockSchedulerStateManagerAdaptor();
- TopologyProvider topologyProvider = new TopologyProvider(adaptor, eventManager, topology);
-
- Map<String, Integer> bolts = getBolts();
- assertEquals(2, bolts.size());
- String[] boltNames = topologyProvider.getBoltNames();
- assertEquals(bolts.size(), boltNames.length);
- for (String boltName : boltNames) {
- bolts.remove(boltName);
- }
- assertEquals(0, bolts.size());
- }
-
- private Map<String, Integer> getBolts() {
- Map<String, Integer> bolts = new HashMap<>();
- bolts.put("bolt-1", 1);
- bolts.put("bolt-2", 1);
- return bolts;
- }
-
- private Map<String, Integer> getSpouts() {
- Map<String, Integer> spouts = new HashMap<>();
- spouts.put("spout", 1);
- return spouts;
- }
-}
diff --git a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/detectors/BackPressureDetectorTest.java b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/detectors/BackPressureDetectorTest.java
index 4160fa0..06c47f7 100644
--- a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/detectors/BackPressureDetectorTest.java
+++ b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/detectors/BackPressureDetectorTest.java
@@ -19,6 +19,7 @@
package org.apache.heron.healthmgr.detectors;
+import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
@@ -32,6 +33,7 @@
import org.junit.Before;
import org.junit.Test;
+import org.apache.heron.healthmgr.HealthManagerMetrics;
import org.apache.heron.healthmgr.HealthPolicyConfig;
import static org.apache.heron.healthmgr.detectors.BackPressureDetector.CONF_NOISE_FILTER;
@@ -49,7 +51,7 @@
now = Instant.now();
}
@Test
- public void testConfigAndFilter() {
+ public void testConfigAndFilter() throws IOException {
HealthPolicyConfig config = mock(HealthPolicyConfig.class);
when(config.getConfig(CONF_NOISE_FILTER, 20)).thenReturn(50);
@@ -65,7 +67,8 @@
metrics.add(measurement2);
metrics.add(measurement3);
- BackPressureDetector detector = new BackPressureDetector(config);
+ HealthManagerMetrics publishingMetrics = mock(HealthManagerMetrics.class);
+ BackPressureDetector detector = new BackPressureDetector(config, publishingMetrics);
PoliciesExecutor.ExecutionContext context = mock(PoliciesExecutor.ExecutionContext.class);
when(context.checkpoint()).thenReturn(now);
detector.initialize(context);
@@ -92,7 +95,7 @@
metrics.add(measurement1);
metrics.add(measurement2);
- detector = new BackPressureDetector(config);
+ detector = new BackPressureDetector(config, publishingMetrics);
detector.initialize(context);
symptoms = detector.detect(metrics);
diff --git a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/diagnosers/SlowInstanceDiagnoserTest.java b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/diagnosers/SlowInstanceDiagnoserTest.java
index fb5bfb0..029fe92 100644
--- a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/diagnosers/SlowInstanceDiagnoserTest.java
+++ b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/diagnosers/SlowInstanceDiagnoserTest.java
@@ -19,6 +19,7 @@
package org.apache.heron.healthmgr.diagnosers;
+import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
@@ -35,6 +36,7 @@
import org.junit.Before;
import org.junit.Test;
+import org.apache.heron.healthmgr.HealthManagerMetrics;
import org.apache.heron.healthmgr.sensors.BaseSensor;
import static org.apache.heron.healthmgr.detectors.BaseDetector.SymptomType.SYMPTOM_COMP_BACK_PRESSURE;
@@ -55,14 +57,15 @@
private ExecutionContext context;
@Before
- public void initTestData() {
+ public void initTestData() throws IOException {
now = Instant.now();
measurements = new ArrayList<>();
context = mock(ExecutionContext.class);
when(context.checkpoint()).thenReturn(now);
- diagnoser = new SlowInstanceDiagnoser();
+ HealthManagerMetrics publishingMetrics = mock(HealthManagerMetrics.class);
+ diagnoser = new SlowInstanceDiagnoser(publishingMetrics);
diagnoser.initialize(context);
}
diff --git a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/resolvers/ScaleUpResolverTest.java b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/resolvers/ScaleUpResolverTest.java
index 0c36e91..e9846c9 100644
--- a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/resolvers/ScaleUpResolverTest.java
+++ b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/resolvers/ScaleUpResolverTest.java
@@ -21,6 +21,7 @@
import java.time.Instant;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -40,9 +41,11 @@
import org.apache.heron.api.generated.TopologyAPI;
import org.apache.heron.common.utils.topology.TopologyTests;
import org.apache.heron.healthmgr.common.PackingPlanProvider;
-import org.apache.heron.healthmgr.common.TopologyProvider;
+import org.apache.heron.healthmgr.common.PhysicalPlanProvider;
+import org.apache.heron.healthmgr.HealthManagerMetrics;
import org.apache.heron.packing.roundrobin.RoundRobinPacking;
import org.apache.heron.proto.scheduler.Scheduler.UpdateTopologyRequest;
+import org.apache.heron.proto.system.PhysicalPlans.PhysicalPlan;
import org.apache.heron.scheduler.client.ISchedulerClient;
import org.apache.heron.spi.common.Config;
import org.apache.heron.spi.common.Key;
@@ -103,7 +106,7 @@
@Test
public void testBuildPackingPlan() {
TopologyAPI.Topology topology = createTestTopology();
- TopologyProvider topologyProvider = createTopologyProvider(topology);
+ PhysicalPlanProvider topologyProvider = createPhysicalPlanProvider(topology);
Config config = createConfig(topology);
PackingPlan currentPlan = createPacking(topology, config);
@@ -139,10 +142,12 @@
.build();
}
- private TopologyProvider createTopologyProvider(TopologyAPI.Topology topology) {
- TopologyProvider topologyProvider = mock(TopologyProvider.class);
- when(topologyProvider.get()).thenReturn(topology);
- return topologyProvider;
+ private PhysicalPlanProvider createPhysicalPlanProvider(TopologyAPI.Topology topology) {
+ PhysicalPlan pp = PhysicalPlan.newBuilder().setTopology(topology).build();
+
+ PhysicalPlanProvider physicalPlanProvider = mock(PhysicalPlanProvider.class);
+ when(physicalPlanProvider.get()).thenReturn(pp);
+ return physicalPlanProvider;
}
private TopologyAPI.Topology createTestTopology() {
diff --git a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/sensors/BackPressureSensorTest.java b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/sensors/BackPressureSensorTest.java
index ddc429f..15da5a4 100644
--- a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/sensors/BackPressureSensorTest.java
+++ b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/sensors/BackPressureSensorTest.java
@@ -19,7 +19,9 @@
package org.apache.heron.healthmgr.sensors;
+import java.io.IOException;
import java.time.Instant;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -32,8 +34,9 @@
import org.junit.Test;
import org.mockito.Mockito;
+import org.apache.heron.healthmgr.HealthManagerMetrics;
import org.apache.heron.healthmgr.common.PackingPlanProvider;
-import org.apache.heron.healthmgr.common.TopologyProvider;
+import org.apache.heron.healthmgr.common.PhysicalPlanProvider;
import static org.apache.heron.healthmgr.sensors.BaseSensor.DEFAULT_METRIC_DURATION;
import static org.apache.heron.healthmgr.sensors.BaseSensor.MetricName.METRIC_BACK_PRESSURE;
@@ -45,9 +48,10 @@
public class BackPressureSensorTest {
@Test
- public void providesBackPressureMetricForBolts() {
- TopologyProvider topologyProvider = mock(TopologyProvider.class);
- when(topologyProvider.getBoltNames()).thenReturn(new String[]{"bolt-1", "bolt-2"});
+ public void providesBackPressureMetricForBolts() throws IOException {
+ PhysicalPlanProvider topologyProvider = mock(PhysicalPlanProvider.class);
+ when(topologyProvider.getBoltNames()).thenReturn(
+ Arrays.asList(new String[]{"bolt-1", "bolt-2"}));
String[] boltIds = new String[]{"container_1_bolt-1_1",
"container_2_bolt-2_22",
@@ -70,8 +74,10 @@
}
+ HealthManagerMetrics publishingMetrics = mock(HealthManagerMetrics.class);
BackPressureSensor backPressureSensor =
- new BackPressureSensor(packingPlanProvider, topologyProvider, null, metricsProvider);
+ new BackPressureSensor(
+ packingPlanProvider, topologyProvider, null, metricsProvider, publishingMetrics);
ExecutionContext context = mock(ExecutionContext.class);
when(context.checkpoint()).thenReturn(Instant.now());
diff --git a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/sensors/BufferSizeSensorTest.java b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/sensors/BufferSizeSensorTest.java
index e68ec27..f19621c 100644
--- a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/sensors/BufferSizeSensorTest.java
+++ b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/sensors/BufferSizeSensorTest.java
@@ -20,6 +20,7 @@
package org.apache.heron.healthmgr.sensors;
import java.time.Instant;
+import java.util.Arrays;
import java.util.Collection;
import com.microsoft.dhalion.api.MetricsProvider;
@@ -30,7 +31,7 @@
import org.junit.Test;
import org.apache.heron.healthmgr.common.PackingPlanProvider;
-import org.apache.heron.healthmgr.common.TopologyProvider;
+import org.apache.heron.healthmgr.common.PhysicalPlanProvider;
import org.apache.heron.healthmgr.sensors.BaseSensor.MetricName;
import static org.apache.heron.healthmgr.sensors.BaseSensor.MetricName.METRIC_WAIT_Q_SIZE;
@@ -41,8 +42,9 @@
public class BufferSizeSensorTest {
@Test
public void providesBufferSizeMetricForBolts() {
- TopologyProvider topologyProvider = mock(TopologyProvider.class);
- when(topologyProvider.getBoltNames()).thenReturn(new String[]{"bolt-1", "bolt-2"});
+ PhysicalPlanProvider topologyProvider = mock(PhysicalPlanProvider.class);
+ when(topologyProvider.getBoltNames()).thenReturn(
+ Arrays.asList(new String[]{"bolt-1", "bolt-2"}));
String[] boltIds = new String[]{"container_1_bolt-1_1",
"container_2_bolt-2_22",
diff --git a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/sensors/ExecuteCountSensorTest.java b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/sensors/ExecuteCountSensorTest.java
index 811717f..77dfbc6 100644
--- a/heron/healthmgr/tests/java/org/apache/heron/healthmgr/sensors/ExecuteCountSensorTest.java
+++ b/heron/healthmgr/tests/java/org/apache/heron/healthmgr/sensors/ExecuteCountSensorTest.java
@@ -32,7 +32,7 @@
import org.junit.Test;
-import org.apache.heron.healthmgr.common.TopologyProvider;
+import org.apache.heron.healthmgr.common.PhysicalPlanProvider;
import static org.apache.heron.healthmgr.sensors.BaseSensor.DEFAULT_METRIC_DURATION;
import static org.apache.heron.healthmgr.sensors.BaseSensor.MetricName.METRIC_EXE_COUNT;
@@ -47,8 +47,9 @@
public void providesBoltExecutionCountMetrics() {
Instant now = Instant.now();
String metric = METRIC_EXE_COUNT.text();
- TopologyProvider topologyProvider = mock(TopologyProvider.class);
- when(topologyProvider.getBoltNames()).thenReturn(new String[]{"bolt-1", "bolt-2"});
+ PhysicalPlanProvider topologyProvider = mock(PhysicalPlanProvider.class);
+ when(topologyProvider.getBoltNames()).thenReturn(
+ Arrays.asList(new String[]{"bolt-1", "bolt-2"}));
MetricsProvider metricsProvider = mock(MetricsProvider.class);
