SLIDER 25. Have Slider-Agent report back the latest config of any type
git-svn-id: https://svn.apache.org/repos/asf/incubator/slider/trunk@1594031 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/app-packages/accumulo-v1_5/appConfig.json b/app-packages/accumulo-v1_5/appConfig.json
index 18a5c8f..9debe7b 100644
--- a/app-packages/accumulo-v1_5/appConfig.json
+++ b/app-packages/accumulo-v1_5/appConfig.json
@@ -4,7 +4,7 @@
},
"global": {
"agent.conf": "/slider/agent/conf/agent.ini",
- "application.def": "/slider/accumulo_v151.tar",
+ "application.def": "/slider/accumulo_v151.zip",
"config_types": "accumulo-site",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"package_list": "files/accumulo-1.5.1-bin.tar.gz",
@@ -45,27 +45,22 @@
},
"components": {
"ACCUMULO_MASTER": {
- "wait.heartbeat": "3",
- "role.script": "scripts/accumulo_master.py"
+ "wait.heartbeat": "3"
},
"slider-appmaster": {
"jvm.heapsize": "256M"
},
"ACCUMULO_TSERVER": {
- "wait.heartbeat": "6",
- "role.script": "scripts/accumulo_tserver.py"
+ "wait.heartbeat": "6"
},
"ACCUMULO_MONITOR": {
- "wait.heartbeat": "8",
- "role.script": "scripts/accumulo_monitor.py"
+ "wait.heartbeat": "8"
},
"ACCUMULO_GC": {
- "wait.heartbeat": "6",
- "role.script": "scripts/accumulo_gc.py"
+ "wait.heartbeat": "6"
},
"ACCUMULO_TRACER": {
- "wait.heartbeat": "8",
- "role.script": "scripts/accumulo_tracer.py"
+ "wait.heartbeat": "8"
}
}
}
diff --git a/app-packages/hbase-v0_96/appConfig.json b/app-packages/hbase-v0_96/appConfig.json
index 5a0a9eb..bd844b0 100644
--- a/app-packages/hbase-v0_96/appConfig.json
+++ b/app-packages/hbase-v0_96/appConfig.json
@@ -4,7 +4,7 @@
},
"global": {
"agent.conf": "/slider/agent/conf/agent.ini",
- "application.def": "/slider/hbase_v096.tar",
+ "application.def": "/slider/hbase_v096.zip",
"config_types": "core-site,hdfs-site,hbase-site",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"package_list": "files/hbase-0.96.1-hadoop2-bin.tar.gz",
@@ -54,15 +54,13 @@
},
"components": {
"HBASE_MASTER": {
- "wait.heartbeat": "5",
- "role.script": "scripts/hbase_master.py"
+ "wait.heartbeat": "5"
},
"slider-appmaster": {
"jvm.heapsize": "256M"
},
"HBASE_REGIONSERVER": {
- "wait.heartbeat": "3",
- "role.script": "scripts/hbase_regionserver.py"
+ "wait.heartbeat": "3"
}
}
}
diff --git a/app-packages/storm-v0_91/appConfig.json b/app-packages/storm-v0_91/appConfig.json
index 56404d1..dac64f5 100644
--- a/app-packages/storm-v0_91/appConfig.json
+++ b/app-packages/storm-v0_91/appConfig.json
@@ -4,7 +4,7 @@
},
"global": {
"agent.conf": "/slider/agent/conf/agent.ini",
- "application.def": "/slider/storm_v091.tar",
+ "application.def": "/slider/storm_v091.zip",
"config_types": "storm-site",
"java_home": "/usr/jdk64/jdk1.7.0_45",
"package_list": "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz",
@@ -107,23 +107,18 @@
"jvm.heapsize": "256M"
},
"NIMBUS": {
- "role.script": "scripts/nimbus.py"
},
"STORM_REST_API": {
- "wait.heartbeat": "3",
- "role.script": "scripts/rest_api.py"
+ "wait.heartbeat": "3"
},
"STORM_UI_SERVER": {
- "wait.heartbeat": "3",
- "role.script": "scripts/ui_server.py"
+ "wait.heartbeat": "3"
},
"DRPC_SERVER": {
- "wait.heartbeat": "7",
- "role.script": "scripts/drpc_server.py"
+ "wait.heartbeat": "7"
},
"SUPERVISOR": {
- "wait.heartbeat": "10",
- "role.script": "scripts/supervisor.py"
+ "wait.heartbeat": "10"
}
}
}
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index f114d03..021a1a8 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -134,6 +134,26 @@
raise AgentException(message)
return path
+ def getConfig(self, command):
+ if 'commandParams' in command and 'config_type' in command['commandParams']:
+ config_type = command['commandParams']['config_type']
+ logger.info("Requesting applied config for type {0}".format(config_type))
+ if config_type in self.applied_configs:
+ return {
+ 'configurations': {config_type: self.applied_configs[config_type]}
+ }
+ else:
+ return {
+ 'configurations': {}
+ }
+ pass
+ else:
+ logger.info("Requesting all applied config.")
+ return {
+ 'configurations': self.applied_configs
+ }
+ pass
+
def requestComponentStatus(self, command):
"""
Component status is determined by exit code, returned by runCommand().
@@ -145,24 +165,7 @@
override_output_files = False
if command['roleCommand'] == "GET_CONFIG":
- if 'commandParams' in command and 'config_type' in command['commandParams']:
- config_type = command['commandParams']['config_type']
- logger.info("Requesting applied config for type {0}".format(config_type))
- if config_type in self.applied_configs:
- return {
- 'configurations': {config_type: self.applied_configs[config_type]}
- }
- else:
- return {
- 'configurations': {}
- }
- pass
- else:
- logger.info("Requesting all applied config.")
- return {
- 'configurations': self.applied_configs
- }
- pass
+ return self.getConfig(command)
else:
res = self.runCommand(command, self.status_commands_stdout,
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index d03ca31..2859f33 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -39,8 +39,6 @@
import org.apache.slider.core.launch.CommandLineBuilder;
import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.info.RegisteredEndpoint;
-import org.apache.slider.core.registry.info.ServiceInstanceData;
import org.apache.slider.providers.AbstractProviderService;
import org.apache.slider.providers.ProviderCore;
import org.apache.slider.providers.ProviderRole;
@@ -53,6 +51,7 @@
import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
import org.apache.slider.server.appmaster.web.rest.agent.AgentRestOperations;
import org.apache.slider.server.appmaster.web.rest.agent.CommandReport;
+import org.apache.slider.server.appmaster.web.rest.agent.ComponentStatus;
import org.apache.slider.server.appmaster.web.rest.agent.ExecutionCommand;
import org.apache.slider.server.appmaster.web.rest.agent.HeartBeat;
import org.apache.slider.server.appmaster.web.rest.agent.HeartBeatResponse;
@@ -92,6 +91,7 @@
private static final ProviderUtils providerUtils = new ProviderUtils(log);
private static final String LABEL_MAKER = "___";
private static final String CONTAINER_ID = "container_id";
+ private static final String GLOBAL_CONFIG_TAG = "global";
private AgentClientProvider clientProvider;
private Map<String, ComponentInstanceState> componentStatuses = new HashMap<String, ComponentInstanceState>();
private Map<String, List<String>> roleHostMapping = new HashMap<String, List<String>>();
@@ -147,7 +147,7 @@
// No need to synchronize as there is low chance of multiple simultaneous reads
if (metainfo == null) {
metainfo = getApplicationMetainfo(fileSystem, appDef);
- if(metainfo == null) {
+ if (metainfo == null) {
log.error("metainfo.xml is unavailable or malformed at {}.", appDef);
throw new SliderException("metainfo.xml is required in app package.");
}
@@ -239,7 +239,7 @@
log.info("Reading metainfo at {}", appDef);
InputStream metainfoStream = SliderUtils.getApplicationResourceInputStream(
fileSystem.getFileSystem(), new Path(appDef), "metainfo.xml");
- if(metainfoStream == null) {
+ if (metainfoStream == null) {
log.error("metainfo.xml is unavailable at {}.", appDef);
throw new IOException("metainfo.xml is required in app package.");
}
@@ -250,7 +250,7 @@
}
protected void publishComponentConfiguration(String name, String description,
- Iterable<Map.Entry<String, String>> entries) {
+ Iterable<Map.Entry<String, String>> entries) {
PublishedConfiguration pubconf = new PublishedConfiguration();
pubconf.description = description;
pubconf.putValues(entries);
@@ -368,12 +368,8 @@
String roleName = getRoleName(label);
String containerId = getContainerId(label);
StateAccessForProviders accessor = getStateAccessor();
- String scriptPath = null;
+ String scriptPath = getScriptPathFromMetainfo(roleName);
- scriptPath = getScriptPathFromMetainfo(roleName);
-
- //scriptPath = accessor.getInstanceDefinitionSnapshot().
- // getAppConfOperations().getComponentOpt(roleName, AgentKeys.COMPONENT_SCRIPT, null);
if (scriptPath == null) {
log.error("role.script is unavailable for " + roleName + ". Commands will not be sent.");
return response;
@@ -382,7 +378,9 @@
if (!componentStatuses.containsKey(label)) {
return response;
}
+
ComponentInstanceState componentStatus = componentStatuses.get(label);
+ processReturnedStatus(heartBeat, componentStatus);
List<CommandReport> reports = heartBeat.getReports();
if (reports != null && !reports.isEmpty()) {
@@ -403,8 +401,8 @@
}
Command command = componentStatus.getNextCommand();
- if (Command.NOP != command) {
- try {
+ try {
+ if (Command.NOP != command) {
componentStatus.commandIssued(command);
if (command == Command.INSTALL) {
log.info("Installing component ...");
@@ -413,15 +411,42 @@
log.info("Starting component ...");
addStartCommand(roleName, containerId, response, scriptPath);
}
- } catch (SliderException e) {
- componentStatus.applyCommandResult(CommandResult.FAILED, command);
- log.warn("Component instance failed operation.", e);
}
+ // if there is no outstanding command then retrieve config
+ Boolean isMaster = isMaster(roleName);
+ if (isMaster && componentStatus.getState() == State.STARTED
+ && command == Command.NOP) {
+ if (!componentStatus.getConfigReported()) {
+ addGetConfigCommand(roleName, containerId, response);
+ }
+ }
+ } catch (SliderException e) {
+ componentStatus.applyCommandResult(CommandResult.FAILED, command);
+ log.warn("Component instance failed operation.", e);
}
return response;
}
+ protected void processReturnedStatus(HeartBeat heartBeat, ComponentInstanceState componentStatus) {
+ List<ComponentStatus> statuses = heartBeat.getComponentStatus();
+ if (statuses != null && statuses.size() > 0) {
+ log.info("Processing {} status reports.", statuses.size());
+ for (ComponentStatus status : statuses) {
+ log.info("Status report: " + status.toString());
+ if (status.getConfigs() != null) {
+ for (String key : status.getConfigs().keySet()) {
+ if (!key.equals(GLOBAL_CONFIG_TAG)) {
+ Map<String, String> configs = status.getConfigs().get(key);
+ publishComponentConfiguration(key, key, configs.entrySet());
+ }
+ }
+ componentStatus.setConfigReported(true);
+ }
+ }
+ }
+ }
+
protected String getScriptPathFromMetainfo(String roleName) {
String scriptPath = null;
List<Service> services = getMetainfo().getServices();
@@ -438,6 +463,26 @@
return scriptPath;
}
+ protected Boolean isMaster(String roleName) throws SliderException {
+ String scriptPath = null;
+ List<Service> services = getMetainfo().getServices();
+ if (services.size() != 1) {
+ log.error("Malformed app definition: Expect only one service in the metainfo.xml");
+ } else {
+ Service service = services.get(0);
+ for (Component component : service.getComponents()) {
+ if (component.getName().equals(roleName)) {
+ if (component.getCategory().equals("MASTER")) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+ }
+ throw new SliderException(String.format("Rolename %s not found in metainfo.xml", roleName));
+ }
+
private String getRoleName(String label) {
return label.substring(label.indexOf(LABEL_MAKER) + LABEL_MAKER.length());
}
@@ -610,7 +655,7 @@
// for now, reading this from appConf. In the future, modify this method to
// process metainfo.xml
List<String> configList = new ArrayList<String>();
- configList.add("global");
+ configList.add(GLOBAL_CONFIG_TAG);
String configTypes = appConf.get("config_types");
String[] configs = configTypes.split(",");
@@ -625,7 +670,7 @@
Map<String, Map<String, String>> configurations,
Map<String, String> tokens) {
Map<String, String> config = new HashMap<String, String>();
- if (configName.equals("global")) {
+ if (configName.equals(GLOBAL_CONFIG_TAG)) {
addDefaultGlobalConfig(config);
}
// add role hosts to tokens
@@ -669,10 +714,10 @@
registry.listInstancesByType(SliderKeys.APP_TYPE);
assert services.size() >= 1;
Map payload = (Map) services.get(0);
- Map<String,Map> endpointMap =
- (Map<String,Map>) ((Map)payload.get("externalView")).get("endpoints");
- for (Map.Entry<String,Map> endpoint : endpointMap.entrySet()) {
- Map<String,String> val = endpoint.getValue();
+ Map<String, Map> endpointMap =
+ (Map<String, Map>) ((Map) payload.get("externalView")).get("endpoints");
+ for (Map.Entry<String, Map> endpoint : endpointMap.entrySet()) {
+ Map<String, String> val = endpoint.getValue();
if ("http".equals(val.get("protocol"))) {
URL url = new URL(val.get("value"));
details.put(val.get("description"), url);
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
index eea77d7..35e7706 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
@@ -29,13 +29,13 @@
private static int MAX_FAILURE_TOLERATED = 3;
private static String INVALID_TRANSITION_ERROR =
"Result {0} for command {1} is not expected for component {2} in state {3}.";
- private State state = State.INIT;
- private State targetState = State.STARTED;
- private int failuresSeen = 0;
private final String compName;
private final String containerId;
private final String applicationId;
-
+ private State state = State.INIT;
+ private State targetState = State.STARTED;
+ private int failuresSeen = 0;
+ private Boolean configReported = false;
public ComponentInstanceState(String compName,
String containerId,
@@ -45,6 +45,14 @@
this.applicationId = applicationId;
}
+ public Boolean getConfigReported() {
+ return configReported;
+ }
+
+ public void setConfigReported(Boolean configReported) {
+ this.configReported = configReported;
+ }
+
public void commandIssued(Command command) {
Command expected = getNextCommand();
if (expected != command) {
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java
index 071432b..63bdd0c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java
@@ -17,6 +17,7 @@
package org.apache.slider.server.appmaster.web.rest.agent;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
import java.util.Map;
@@ -32,8 +33,18 @@
String status;
String serviceName;
String clusterName;
+ String roleCommand;
+ @JsonProperty("configurations")
private Map<String, Map<String, String>> configurations;
+ public String getRoleCommand() {
+ return roleCommand;
+ }
+
+ public void setRoleCommand(String roleCommand) {
+ this.roleCommand = roleCommand;
+ }
+
public String getComponentName() {
return this.componentName;
}
@@ -75,13 +86,13 @@
}
/** @return the config tags that match this command, or <code>null</code> if none are present */
- public Map<String, Map<String, String>> getConfiguration() {
+ public Map<String, Map<String, String>> getConfigs() {
return configurations;
}
- /** @param configurations the config tags that match this status */
- public void setConfiguration(Map<String, Map<String, String>> configurations) {
- this.configurations = configurations;
+ /** @param configs the config tags that match this status */
+ public void setConfigs(Map<String, Map<String, String>> configs) {
+ this.configurations = configs;
}
@Override
@@ -92,6 +103,7 @@
", status='" + status + '\'' +
", serviceName='" + serviceName + '\'' +
", clusterName='" + clusterName + '\'' +
+ ", roleCommand='" + roleCommand + '\'' +
'}';
}
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
index 4ad94cf..48a58dd 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/AgentMiniClusterTestBase.groovy
@@ -48,9 +48,8 @@
agentConf.createNewFile()
agentDef = new File(destDir, "agentdef")
agentDef.createNewFile()
- imagePath = new File(destDir, "imagePath.tar.gz")
- imagePath.createNewFile()
-
+ File slider_dir = new File(new File(".").absoluteFile, "src/test/python");
+ imagePath = new File(slider_dir, "appdef_1.zip")
}
@Override
public String getTestConfigurationPath() {
@@ -89,7 +88,7 @@
*/
@Override
List<String> getImageCommands() {
- [Arguments.ARG_IMAGE, imagePath.toURI().toString()]
+ [Arguments.ARG_IMAGE, agentDef.toURI().toString()]
}
/**
@@ -113,8 +112,8 @@
deleteExistingData,
blockUntilRunning,
[
- (AgentKeys.APP_DEF): agentConf.toURI().toString(),
- (AgentKeys.AGENT_CONF): agentDef.toURI().toString()
+ (AgentKeys.APP_DEF): imagePath.toURI().toString(),
+ (AgentKeys.AGENT_CONF): agentConf.toURI().toString()
])
}
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
index b887d5b..bcda69a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/web/rest/publisher/TestPublisherRestResources.groovy
@@ -58,7 +58,7 @@
false)
Map<String, Integer> roles = [:]
File slider_core = new File(new File(".").absoluteFile, "src/test/python");
- String app_def = "appdef_1.tar"
+ String app_def = "appdef_1.zip"
File app_def_path = new File(slider_core, app_def)
String agt_ver = "version"
File agt_ver_path = new File(slider_core, agt_ver)
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 7b74520..e758b04 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -44,6 +44,7 @@
import org.apache.slider.server.appmaster.model.mock.MockFileSystem;
import org.apache.slider.server.appmaster.model.mock.MockNodeId;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.web.rest.agent.ComponentStatus;
import org.apache.slider.server.appmaster.web.rest.agent.HeartBeat;
import org.apache.slider.server.appmaster.web.rest.agent.HeartBeatResponse;
import org.apache.slider.server.appmaster.web.rest.agent.Register;
@@ -57,6 +58,8 @@
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -65,8 +68,11 @@
import static org.easymock.EasyMock.createNiceMock;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyCollection;
import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.eq;
@@ -116,6 +122,7 @@
doReturn(new Metainfo()).when(mockAps).getApplicationMetainfo(any(SliderFileSystem.class), anyString());
try {
+ doReturn(true).when(mockAps).isMaster(anyString());
doNothing().when(mockAps).addInstallCommand(
eq("HBASE_MASTER"),
eq("mockcontainer_1"),
@@ -183,6 +190,31 @@
}
@Test
+ public void testProcessConfig() throws Exception {
+ AgentProviderService aps = new AgentProviderService();
+ HeartBeat hb = new HeartBeat();
+ ComponentStatus status = new ComponentStatus();
+ status.setClusterName("test");
+ status.setComponentName("HBASE_MASTER");
+ status.setRoleCommand("GET_CONFIG");
+ Map<String, String> hbaseSite = new HashMap<String, String>();
+ hbaseSite.put("a", "b");
+ hbaseSite.put("c", "d");
+ Map<String, Map<String, String>> configs = new HashMap<String,Map<String, String>>();
+ configs.put("hbase-site", hbaseSite);
+ configs.put("global", hbaseSite);
+ status.setConfigs(configs);
+ hb.setComponentStatus(new ArrayList<ComponentStatus>(Arrays.asList(status)));
+
+ ComponentInstanceState componentStatus = new ComponentInstanceState("HBASE_MASTER", "aid", "cid");
+ AgentProviderService mockAps = Mockito.spy(aps);
+ doNothing().when(mockAps).publishComponentConfiguration(anyString(), anyString(), anyCollection());
+ mockAps.processReturnedStatus(hb, componentStatus);
+ assert componentStatus.getConfigReported() == true;
+ Mockito.verify(mockAps, Mockito.times(1)).publishComponentConfiguration(anyString(), anyString(), anyCollection());
+ }
+
+ @Test
public void testMetainfoParsing() throws Exception {
String metainfo_1_str = "<metainfo>\n"
+ " <schemaVersion>2.0</schemaVersion>\n"
diff --git a/src/site/markdown/debugging.md b/src/site/markdown/debugging.md
index 897db53..b98af1d 100644
--- a/src/site/markdown/debugging.md
+++ b/src/site/markdown/debugging.md
@@ -74,7 +74,7 @@
## IDE-based remote debugging of the Application Master
-For situtations in which the logging does not yield enough information to debug an issue,
+For situations in which the logging does not yield enough information to debug an issue,
the user has the option of specifying JVM command line options for the
Application Master that enable attaching to the running process with a debugger
(e.g. the remote debugging facilities in Eclipse or Intellij IDEA).
