SLIDER-525. Storm app package for windows does not report component status correctly
HBase Win package has the ability to provide custom ports for Thrift and Thrift2
diff --git a/app-packages/hbase-win/README.txt b/app-packages/hbase-win/README.txt
index a413e76..6389fb2 100644
--- a/app-packages/hbase-win/README.txt
+++ b/app-packages/hbase-win/README.txt
@@ -30,7 +30,7 @@
-Dpkg.name=<file name of app zip file> -Dpkg.src=<folder location where the pkg is available>
Example:
-mvn clean package -Phbase-app-package -Dpkg.version=0.98.5-hadoop2
+mvn clean package -Phbase-app-package-win -Dpkg.version=0.98.5-hadoop2
-Dpkg.name=hbase-0.98.5-hadoop2-bin.zip
-Dpkg.src=/Users/user1/Downloads
diff --git a/app-packages/hbase-win/package/scripts/hbase_service.py b/app-packages/hbase-win/package/scripts/hbase_service.py
index 9a8dc56..e269531 100644
--- a/app-packages/hbase-win/package/scripts/hbase_service.py
+++ b/app-packages/hbase-win/package/scripts/hbase_service.py
@@ -28,6 +28,8 @@
import params
pid_file = format("{pid_dir}/hbase-{hbase_user}-{name}.pid")
+ custom_port = None
+ custom_info_port = None
heap_size = params.master_heapsize
main_class = "org.apache.hadoop.hbase.master.HMaster"
if name == "regionserver":
@@ -36,12 +38,17 @@
if name == "rest":
heap_size = params.restserver_heapsize
main_class = "org.apache.hadoop.hbase.rest.RESTServer"
+ custom_port = params.rest_port
if name == "thrift":
heap_size = params.thriftserver_heapsize
main_class = "org.apache.hadoop.hbase.thrift.ThriftServer"
+ custom_port = params.thrift_port
+ custom_info_port = params.thrift_info_port
if name == "thrift2":
heap_size = params.thrift2server_heapsize
main_class = "org.apache.hadoop.hbase.thrift2.ThriftServer"
+ custom_port = params.thrift2_port
+ custom_info_port = params.thrift2_info_port
role_user = format("{hbase_user}-{name}")
@@ -49,6 +56,12 @@
process_cmd = format("{java64_home}\\bin\\java {rest_of_the_command} {main_class} {action}")
+ if custom_port:
+ process_cmd = format("{process_cmd} -p {custom_port}")
+
+ if custom_info_port:
+ process_cmd = format("{process_cmd} --infoport {custom_info_port}")
+
Execute(process_cmd,
logoutput=False,
wait_for_finish=False,
diff --git a/app-packages/hbase-win/package/scripts/params.py b/app-packages/hbase-win/package/scripts/params.py
index 4f55cd0..90d274c 100644
--- a/app-packages/hbase-win/package/scripts/params.py
+++ b/app-packages/hbase-win/package/scripts/params.py
@@ -42,11 +42,13 @@
hbase_hdfs_root_dir = config['configurations']['hbase-site']['hbase.rootdir']
"""
-Read various ports, unused as of now
+Read various ports
"""
-rest_port = config['configurations']['global']['hbase_rest_port']
-thrift_port = config['configurations']['global']['hbase_thrift_port']
-thrift2_port = config['configurations']['global']['hbase_thrift2_port']
+rest_port = default("configurations/global/hbase_rest_port", 1700)
+thrift_port = default("configurations/global/hbase_thrift_port", 9090)
+thrift2_port = default("configurations/global/hbase_thrift2_port", 9091)
+thrift_info_port = default("configurations/global/hbase_info_thrift_port", 9095)
+thrift2_info_port = default("configurations/global/hbase_info_thrift2_port", 9096)
"""
Compute or read various heap sizes
diff --git a/app-packages/hbase-win/resources-default.json b/app-packages/hbase-win/resources-default.json
index e451c51..93dc17c 100644
--- a/app-packages/hbase-win/resources-default.json
+++ b/app-packages/hbase-win/resources-default.json
@@ -10,7 +10,7 @@
"HBASE_MASTER": {
"yarn.role.priority": "1",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "512"
},
"slider-appmaster": {
"yarn.memory": "1024"
@@ -18,22 +18,22 @@
"HBASE_REGIONSERVER": {
"yarn.role.priority": "2",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "512"
},
"HBASE_REST": {
"yarn.role.priority": "3",
"yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.memory": "512"
},
"HBASE_THRIFT": {
"yarn.role.priority": "4",
- "yarn.component.instances": "0",
- "yarn.memory": "256"
+ "yarn.component.instances": "1",
+ "yarn.memory": "512"
},
"HBASE_THRIFT2": {
"yarn.role.priority": "5",
- "yarn.component.instances": "1",
- "yarn.memory": "256"
+ "yarn.component.instances": "0",
+ "yarn.memory": "512"
}
}
}
diff --git a/app-packages/storm-win/package/scripts/nimbus.py b/app-packages/storm-win/package/scripts/nimbus.py
index fafc97a..c7c3120 100644
--- a/app-packages/storm-win/package/scripts/nimbus.py
+++ b/app-packages/storm-win/package/scripts/nimbus.py
@@ -49,7 +49,7 @@
def status(self, env):
import status_params
env.set_params(status_params)
- #check_process_status(status_params.pid_nimbus)
+ check_process_status(status_params.pid_nimbus)
if __name__ == "__main__":
Nimbus().execute()
diff --git a/app-packages/storm-win/package/scripts/rest_api.py b/app-packages/storm-win/package/scripts/rest_api.py
index 6a09e4e..33d8924 100644
--- a/app-packages/storm-win/package/scripts/rest_api.py
+++ b/app-packages/storm-win/package/scripts/rest_api.py
@@ -51,7 +51,7 @@
def status(self, env):
import status_params
env.set_params(status_params)
- #check_process_status(status_params.pid_rest_api)
+ check_process_status(status_params.pid_rest_api)
if __name__ == "__main__":
StormRestApi().execute()
diff --git a/app-packages/storm-win/package/scripts/service.py b/app-packages/storm-win/package/scripts/service.py
index b17c636..aa7d339 100644
--- a/app-packages/storm-win/package/scripts/service.py
+++ b/app-packages/storm-win/package/scripts/service.py
@@ -20,9 +20,9 @@
from resource_management import *
-import time
import os
import sys
+import xml.etree.ElementTree as et
"""
Slider package uses jps as pgrep does not list the whole process start command
@@ -40,14 +40,32 @@
os.environ['STORM_LOG_DIR'] = params.log_dir
os.environ['STORM_HOME'] = params.app_root
os.environ['STORM_CONF_DIR'] = params.conf_dir
- cmd = format("{storm_bin} {name} > {log_dir}/{name}.out 2>&1")
- Execute(cmd,
- logoutput=False,
- wait_for_finish=False,
- pid_file = pid_file
+ generate_xml = format("{storm_bin} --service {name} > {log_dir}/{name}.cmd")
+
+ Execute(generate_xml,
+ logoutput=True,
+ wait_for_finish=True
)
+ tree = et.parse(format("{log_dir}/{name}.cmd"))
+ root = tree.getroot()
+ cmd_part = None
+ for child in root:
+ if child.tag == "arguments":
+ cmd_part = child.text
+
+ if cmd_part:
+ cmd = format("{java64_home}\\bin\\java {cmd_part}")
+
+ Execute(cmd,
+ logoutput=False,
+ wait_for_finish=False,
+ pid_file=pid_file
+ )
+ else:
+ Logger.warn("Valid command file did not get generated at " + format("{log_dir}/{name}.cmd"))
+
elif action == "stop":
pid = format("`cat {pid_file}` >/dev/null 2>&1")
Execute(format("kill {pid}")
diff --git a/app-packages/storm-win/package/scripts/supervisor.py b/app-packages/storm-win/package/scripts/supervisor.py
index eba9fa9..47c20c9 100644
--- a/app-packages/storm-win/package/scripts/supervisor.py
+++ b/app-packages/storm-win/package/scripts/supervisor.py
@@ -53,7 +53,7 @@
import status_params
env.set_params(status_params)
- #check_process_status(status_params.pid_supervisor)
+ check_process_status(status_params.pid_supervisor)
if __name__ == "__main__":
diff --git a/app-packages/storm-win/package/scripts/ui_server.py b/app-packages/storm-win/package/scripts/ui_server.py
index 0875b76..0fe7cd2 100644
--- a/app-packages/storm-win/package/scripts/ui_server.py
+++ b/app-packages/storm-win/package/scripts/ui_server.py
@@ -49,7 +49,7 @@
def status(self, env):
import status_params
env.set_params(status_params)
- #check_process_status(status_params.pid_ui)
+ check_process_status(status_params.pid_ui)
if __name__ == "__main__":
UiServer().execute()
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index fe7047d..387bc7e 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -295,7 +295,7 @@
if 'restartEnabled' in response:
restartEnabled = response['restartEnabled']
if restartEnabled:
- logger.info("Component auto-restart is enabled.")
+ logger.debug("Component auto-restart is enabled.")
if 'hasMappedComponents' in response.keys():
self.hasMappedComponents = response['hasMappedComponents'] != False
