jaas fix for StormServer
diff --git a/app-packages/storm/README.txt b/app-packages/storm/README.txt
index 49adfce..288604a 100644
--- a/app-packages/storm/README.txt
+++ b/app-packages/storm/README.txt
@@ -17,16 +17,19 @@
How to create a Slider app package for Storm?
-To create the app package you will need the Storm tarball and invoke mvn command
-with appropriate parameters.
+To create the app package you will need the Storm tarball copied to a specific location.
+Various configurations provided in this sample are customized for apache-storm-0.9.1.2.1.1.0-237.tar.gz.
+So if you use a different version you may need to edit a few config values.
-Command:
-mvn clean package -Pstorm-app-package -Dpkg.version=<version>
- -Dpkg.name=<file name of app tarball> -Dpkg.src=<folder location where the pkg is available>
+Replace the placeholder tarball for Storm.
+ cp ~/Downloads/apache-storm-0.9.3.0.2.5.0-100.tar.gz package/files/
+ rm package/files/apache-storm-0.9.1.2.1.1.0-237.tar.gz.REPLACE
-Example:
-mvn clean package -Pstorm-app-package -Dpkg.version=0.9.3.2.2.0.0-578
- -Dpkg.name=apache-storm-0.9.3.2.2.0.0-578.tar.gz -Dpkg.src=/Users/user1/Downloads
+Create a zip package at the root of the package (<slider enlistment>/app-packages/storm-v0_91/)
+ zip -r Apache_Storm_v_0_9_3.zip .
+
+Verify the content using
+ unzip -l "$@" Apache_Storm_v_0_9_3.zip
While appConfig.json and resources.json are not required for the package they work
well as the default configuration for Slider apps. So its advisable that when you
diff --git a/app-packages/storm/appConfig.json b/app-packages/storm/appConfig.json
index 1eae599..33e38ac 100644
--- a/app-packages/storm/appConfig.json
+++ b/app-packages/storm/appConfig.json
@@ -11,7 +11,7 @@
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}",
"site.global.user_group": "hadoop",
-
+ "site.global.security_enabled": "false",
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_id": "Application2",
"site.global.ganglia_enabled":"true",
@@ -32,7 +32,7 @@
"site.storm-site.nimbus.host": "${NIMBUS_HOST}",
"site.storm-site.ui.port": "${STORM_UI_SERVER.ALLOCATED_PORT}",
"site.storm-site.supervisor.slots.ports": "[${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE},${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}]",
- "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${@//site/global/ganglia_server_host},port=${@//site/global/ganglia_server_port},wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-${pkg.version}/external/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
+ "site.storm-site.supervisor.childopts": "-Xmx256m -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=0 -javaagent:${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/lib/jmxetric-1.0.4.jar=host=${NN_HOST},port=8668,wireformat31x=true,mode=multicast,config=${AGENT_WORK_ROOT}/app/install/apache-storm-0.9.1.2.1.1.0-237/contrib/storm-jmxetric/conf/jmxetric-conf.xml,process=Supervisor_JVM",
"site.storm-site.drpc.port": "0",
"site.storm-site.logviewer.port": "${SUPERVISOR.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index bfd84e8..2c314b4 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -130,7 +130,7 @@
<packages>
<package>
<type>tarball</type>
- <name>files/${pkg.name}</name>
+ <name>files/apache-storm-0.9.1.2.1.1.0-237.tar.gz</name>
</package>
</packages>
</osSpecific>
diff --git a/app-packages/storm/package/scripts/params.py b/app-packages/storm/package/scripts/params.py
index 38e25fb..ff85765 100644
--- a/app-packages/storm/package/scripts/params.py
+++ b/app-packages/storm/package/scripts/params.py
@@ -43,8 +43,7 @@
ganglia_server = config['configurations']['global']['ganglia_server_host']
ganglia_port = config['configurations']['global']['ganglia_server_port']
-_authentication = config['configurations']['core-site']['hadoop.security.authentication']
-security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
+security_enabled = config['configurations']['global']['security_enabled']
if security_enabled:
_hostname_lowercase = config['hostname'].lower()
@@ -54,6 +53,7 @@
storm_jaas_client_principal = _storm_client_principal_name.replace('_HOST', _hostname_lowercase)
storm_client_keytab_path = config['configurations']['storm-env']['storm_client_keytab']
- storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST', _hostname_lowercase)
+ storm_jaas_server_principal = _storm_server_principal_name.replace('_HOST',nimbus_host.lower())
+ storm_jaas_stormclient_servicename = storm_jaas_server_principal.split("/")[0]
storm_server_keytab_path = config['configurations']['storm-env']['storm_server_keytab']
kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
diff --git a/app-packages/storm/package/scripts/service.py b/app-packages/storm/package/scripts/service.py
index 3a5e795..2150cd4 100644
--- a/app-packages/storm/package/scripts/service.py
+++ b/app-packages/storm/package/scripts/service.py
@@ -47,14 +47,14 @@
crt_pid_cmd = format("{jps_path} -vl | grep \"^[0-9 ]*backtype.storm.daemon.{name}\" {grep_and_awk} > {pid_file}")
if action == "start":
- cmd = format("env JAVA_HOME={java64_home} PATH=$PATH:{java64_home}/bin STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {python_binary} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
+ cmd = format("env JAVA_HOME={java64_home} PATH={java64_home}/bin:$PATH STORM_BASE_DIR={app_root} STORM_CONF_DIR={conf_dir} {python_binary} {storm_bin} {name} > {log_dir}/{name}.out 2>&1")
if params.security_enabled:
if name == "nimbus":
- Execute(format("{kinit_path_local} -kt {storm_server_keytab_path} {storm_user}"),
+ Execute(format("{kinit_path_local} -kt {storm_server_keytab_path} {storm_jaas_server_principal}"),
user=params.storm_user)
else:
- Execute(format("{kinit_path_local} -kt {storm_client_keytab_path} {storm_user}"),
+ Execute(format("{kinit_path_local} -kt {storm_client_keytab_path} {storm_jaas_client_principal}"),
user=params.storm_user)
Execute(cmd,
diff --git a/app-packages/storm/package/scripts/storm.py b/app-packages/storm/package/scripts/storm.py
index efca073..8ecb3a1 100644
--- a/app-packages/storm/package/scripts/storm.py
+++ b/app-packages/storm/package/scripts/storm.py
@@ -50,6 +50,9 @@
)
if params.security_enabled:
- TemplateConfig( format("{conf_dir}/storm_jaas.conf"),
- owner = params.storm_user
- )
\ No newline at end of file
+ File(format("{conf_dir}/storm_jaas.conf"),
+ content=Template("storm_jaas.conf.j2"),
+ owner = params.storm_user,
+ group = params.user_group
+ )
+
diff --git a/app-packages/storm/package/scripts/yaml_config.py b/app-packages/storm/package/scripts/yaml_config.py
index d25089c..5f763cc 100644
--- a/app-packages/storm/package/scripts/yaml_config.py
+++ b/app-packages/storm/package/scripts/yaml_config.py
@@ -19,9 +19,13 @@
"""
import re
+import socket
from resource_management import *
def escape_yaml_propetry(value):
+ # pre-process value for any "_HOST" tokens
+ value = value.replace('_HOST', socket.getfqdn())
+
unquouted = False
unquouted_values = ["null","Null","NULL","true","True","TRUE","false","False","FALSE","YES","Yes","yes","NO","No","no","ON","On","on","OFF","Off","off"]
@@ -31,7 +35,11 @@
# if is list [a,b,c]
if re.match('^\w*\[.+\]\w*$', value):
unquouted = True
-
+
+ # if is map {'a':'b'}
+ if re.match('^\w*\{.+\}\w*$', value):
+ unquouted = True
+
try:
int(value)
unquouted = True
diff --git a/app-packages/storm/package/templates/storm_jaas.conf.j2 b/app-packages/storm/package/templates/storm_jaas.conf.j2
index e68cd57..a1ba6ea 100644
--- a/app-packages/storm/package/templates/storm_jaas.conf.j2
+++ b/app-packages/storm/package/templates/storm_jaas.conf.j2
@@ -20,8 +20,7 @@
useKeyTab=true
keyTab="{{storm_server_keytab_path}}"
storeKey=true
- useTicketCache=true
- renewTGT=true
+ useTicketCache=false
principal="{{storm_jaas_server_principal}}";
};
StormClient {
@@ -29,9 +28,8 @@
useKeyTab=true
keyTab="{{storm_client_keytab_path}}"
storeKey=true
- useTicketCache=true
- renewTGT=true
- serviceName="nimbus"
+ useTicketCache=false
+ serviceName="{{storm_jaas_stormclient_servicename}}"
debug=true
principal="{{storm_jaas_client_principal}}";
};
@@ -40,8 +38,7 @@
useKeyTab=true
keyTab="{{storm_client_keytab_path}}"
storeKey=true
- useTicketCache=true
- renewTGT=true
+ useTicketCache=false
serviceName="zookeeper"
principal="{{storm_jaas_client_principal}}";
};
diff --git a/app-packages/storm/resources.json b/app-packages/storm/resources.json
index 017766c..f14b2c8 100644
--- a/app-packages/storm/resources.json
+++ b/app-packages/storm/resources.json
@@ -13,7 +13,7 @@
},
"STORM_UI_SERVER": {
"yarn.role.priority": "2",
- "yarn.component.instances": "1"
+ "yarn.component.instances": "0"
},
"DRPC_SERVER": {
"yarn.role.priority": "3",
@@ -24,4 +24,4 @@
"yarn.component.instances": "1"
}
}
-}
\ No newline at end of file
+}
