| #!/usr/bin/env python |
| """ |
| Licensed to the Apache Software Foundation (ASF) under one |
| or more contributor license agreements. See the NOTICE file |
| distributed with this work for additional information |
| regarding copyright ownership. The ASF licenses this file |
| to you under the Apache License, Version 2.0 (the |
| "License"); you may not use this file except in compliance |
| with the License. You may obtain a copy of the License at |
| |
| http://www.apache.org/licenses/LICENSE-2.0 |
| |
| Unless required by applicable law or agreed to in writing, software |
| distributed under the License is distributed on an "AS IS" BASIS, |
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| See the License for the specific language governing permissions and |
| limitations under the License. |
| |
| """ |
| import functools |
| import glob |
| import os |
| import re |
| import sys |
| import tarfile |
| import tempfile |
| import time |
| |
| from contextlib import closing |
| from optparse import OptionParser |
| from xml.dom import minidom |
| |
| os.environ["PATH"] += os.pathsep + "/var/lib/ambari-agent" |
| sys.path.append("/usr/lib/ambari-server/lib") |
| |
| from ambari_server.serverClassPath import JDBC_DRIVER_PATH_PROPERTY |
| from ambari_server.serverConfiguration import get_value_from_properties, get_ambari_properties |
| |
| from resource_management.core import File |
| from resource_management.core import shell |
| from resource_management.core.environment import Environment |
| from resource_management.core.logger import Logger |
| from resource_management.core.resources.system import Directory |
| from resource_management.core.resources.system import Execute |
| from resource_management.core.source import StaticFile |
| from resource_management.libraries import ConfigDictionary |
| from resource_management.libraries.functions.format import format |
| from resource_management.libraries.functions.oozie_prepare_war import prepare_war |
| from resource_management.libraries.resources.hdfs_resource import HdfsResource |
| |
| DEFAULT_SQL_DRIVER_PATH = "/var/lib/ambari-server/resources/sqljdbc41.jar" |
| |
| with Environment() as env: |
| def get_stack_version(): |
| if not options.hdp_version: |
| # Ubuntu returns: "stdin: is not a tty", as subprocess32 output. |
| tmpfile = tempfile.NamedTemporaryFile() |
| out = None |
| with open(tmpfile.name, 'r+') as file: |
| get_stack_version_cmd = '/usr/bin/hdp-select status %s > %s' % ('hadoop-mapreduce-historyserver', tmpfile.name) |
| code, stdoutdata = shell.call(get_stack_version_cmd) |
| out = file.read() |
| pass |
| if code != 0 or out is None: |
| Logger.warning("Could not verify HDP version by calling '%s'. Return Code: %s, Output: %s." % |
| (get_stack_version_cmd, str(code), str(out))) |
| return 1 |
| |
| matches = re.findall(r"([\d\.]+\-\d+)", out) |
| stack_version = matches[0] if matches and len(matches) > 0 else None |
| |
| if not stack_version: |
| Logger.error("Could not parse HDP version from output of hdp-select: %s" % str(out)) |
| return 1 |
| else: |
| stack_version = options.hdp_version |
| |
| return stack_version |
| |
| parser = OptionParser() |
| parser.add_option("-d", "--database-driver", dest="sql_driver_path", |
| default=get_value_from_properties(get_ambari_properties(), JDBC_DRIVER_PATH_PROPERTY, DEFAULT_SQL_DRIVER_PATH), |
| help="Path to JDBC driver") |
| parser.add_option("-f", "--fs-type", dest="fs_type", default="wasb", |
| help="Expected protocol of fs.defaultFS") |
| parser.add_option("-v", "--hdp-version", dest="hdp_version", default="", |
| help="hdp-version used in path of tarballs") |
| parser.add_option("-u", "--upgrade", dest="upgrade", action="store_true", |
| help="flag to indicate script is being run for upgrade", default=False) |
| (options, args) = parser.parse_args() |
| |
| if not os.path.exists(options.sql_driver_path): |
| Logger.error("SQL driver file {} does not exist".format(options.sql_driver_path)) |
| if os.path.exists(DEFAULT_SQL_DRIVER_PATH): |
| Logger.warning("Fallback to SQL driver {}".format(DEFAULT_SQL_DRIVER_PATH)) |
| options.sql_driver_path = DEFAULT_SQL_DRIVER_PATH |
| else: |
| sys.exit(1) |
| |
| Logger.info("Using SQL driver from {}".format(options.sql_driver_path)) |
| sql_driver_filename = os.path.basename(options.sql_driver_path) |
| |
| # See if hdfs path prefix is provided on the command line. If yes, use that value, if no |
| # use empty string as default. |
| hdfs_path_prefix = "" |
| if len(args) > 0: |
| hdfs_path_prefix = args[0] |
| |
| stack_version = get_stack_version() |
| |
| def getPropertyValueFromConfigXMLFile(xmlfile, name, defaultValue=None): |
| xmldoc = minidom.parse(xmlfile) |
| propNodes = [node.parentNode for node in xmldoc.getElementsByTagName("name") if node.childNodes[0].nodeValue == name] |
| if len(propNodes) > 0: |
| for node in propNodes[-1].childNodes: |
| if node.nodeName == "value": |
| if len(node.childNodes) > 0: |
| return node.childNodes[0].nodeValue |
| else: |
| return defaultValue |
| return defaultValue |
| |
| def get_fs_root(fsdefaultName=None): |
| fsdefaultName = "fake" |
| expected_fs_protocol = options.fs_type + '://' |
| |
| while True: |
| fsdefaultName = getPropertyValueFromConfigXMLFile("/etc/hadoop/conf/core-site.xml", "fs.defaultFS") |
| |
| if fsdefaultName and fsdefaultName.startswith(expected_fs_protocol): |
| break |
| |
| Logger.info("Waiting to read appropriate value of fs.defaultFS from /etc/hadoop/conf/core-site.xml ...") |
| time.sleep(10) |
| |
| Logger.info("Returning fs.defaultFS -> " + fsdefaultName) |
| return fsdefaultName |
| |
| # These values must be the suffix of the properties in cluster-env.xml |
| TAR_SOURCE_SUFFIX = "_tar_source" |
| TAR_DESTINATION_FOLDER_SUFFIX = "_tar_destination_folder" |
| |
| class params: |
| hdfs_path_prefix = hdfs_path_prefix |
| hdfs_user = "hdfs" |
| mapred_user ="mapred" |
| hadoop_bin_dir="/usr/hdp/" + stack_version + "/hadoop/bin" |
| hadoop_conf_dir = "/etc/hadoop/conf" |
| user_group = "hadoop" |
| security_enabled = False |
| oozie_user = "oozie" |
| execute_path = "/usr/hdp/" + stack_version + "/hadoop/bin" |
| ambari_libs_dir = "/var/lib/ambari-agent/lib" |
| hdfs_site = ConfigDictionary({'dfs.webhdfs.enabled':False, |
| }) |
| fs_default = get_fs_root() |
| slider_home_dir = '/usr/hdp/' + stack_version + '/slider' |
| slider_lib_dir = slider_home_dir + '/lib' |
| slider_tarball = slider_lib_dir + "/slider.tar.gz" |
| oozie_secure = '' |
| oozie_home="/usr/hdp/" + stack_version + "/oozie" |
| oozie_setup_sh=format("/usr/hdp/" + stack_version + "/oozie/bin/oozie-setup.sh") |
| oozie_setup_sh_current="/usr/hdp/current/oozie-server/bin/oozie-setup.sh" |
| oozie_tmp_dir = "/var/tmp/oozie" |
| oozie_libext_dir = "/usr/hdp/" + stack_version + "/oozie/libext" |
| oozie_env_sh_template = \ |
| ''' |
| #!/bin/bash |
| |
| export OOZIE_CONFIG=${{OOZIE_CONFIG:-/usr/hdp/{0}/oozie/conf}} |
| export OOZIE_DATA=${{OOZIE_DATA:-/var/lib/oozie/data}} |
| export OOZIE_LOG=${{OOZIE_LOG:-/var/log/oozie}} |
| export CATALINA_BASE=${{CATALINA_BASE:-/usr/hdp/{0}/oozie/oozie-server}} |
| export CATALINA_TMPDIR=${{CATALINA_TMPDIR:-/var/tmp/oozie}} |
| export CATALINA_PID=${{CATALINA_PID:-/var/run/oozie/oozie.pid}} |
| export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat |
| '''.format(stack_version) |
| |
| HdfsResource = functools.partial( |
| HdfsResource, |
| user=hdfs_user, |
| security_enabled = False, |
| keytab = None, |
| kinit_path_local = None, |
| hadoop_bin_dir = hadoop_bin_dir, |
| hadoop_conf_dir = hadoop_conf_dir, |
| principal_name = None, |
| hdfs_site = hdfs_site, |
| default_fs = fs_default, |
| hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore", |
| ) |
| |
| def _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed): |
| """ |
| :param source_and_dest_pairs: List of tuples (x, y), where x is the source file in the local file system, |
| and y is the destination file path in HDFS |
| :param file_owner: Owner to set for the file copied to HDFS (typically hdfs account) |
| :param group_owner: Owning group to set for the file copied to HDFS (typically hadoop group) |
| :param kinit_if_needed: kinit command if it is needed, otherwise an empty string |
| :return: Returns 0 if at least one file was copied and no exceptions occurred, and 1 otherwise. |
| |
| Must kinit before calling this function. |
| """ |
| |
| for (source, destination) in source_and_dest_pairs: |
| params.HdfsResource(destination, |
| action="create_on_execute", |
| type = 'file', |
| mode=0444, |
| owner=file_owner, |
| group=group_owner, |
| source=source, |
| ) |
| |
| |
| def copy_tarballs_to_hdfs(source, dest, stack_select_component_name, component_user, file_owner, group_owner): |
| """ |
| :param tarball_prefix: Prefix of the tarball must be one of tez, hive, mr, pig |
| :param stack_select_component_name: Component name to get the status to determine the version |
| :param component_user: User that will execute the Hadoop commands |
| :param file_owner: Owner of the files copied to HDFS (typically hdfs account) |
| :param group_owner: Group owner of the files copied to HDFS (typically hadoop group) |
| :return: Returns 0 on success, 1 if no files were copied, and in some cases may raise an exception. |
| |
| In order to call this function, params.py must have all of the following, |
| stack_version_formatted, kinit_path_local, security_enabled, hdfs_user, hdfs_principal_name, hdfs_user_keytab, |
| hadoop_bin_dir, hadoop_conf_dir, and HdfsDirectory as a partial function. |
| """ |
| |
| component_tar_source_file, component_tar_destination_folder = source, dest |
| |
| if not os.path.exists(component_tar_source_file): |
| Logger.warning("Could not find file: %s" % str(component_tar_source_file)) |
| return 1 |
| |
| |
| |
| file_name = os.path.basename(component_tar_source_file) |
| destination_file = os.path.join(component_tar_destination_folder, file_name) |
| destination_file = destination_file.replace("{{ stack_version_formatted }}", stack_version) |
| |
| |
| kinit_if_needed = "" |
| if params.security_enabled: |
| kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name};") |
| |
| if kinit_if_needed: |
| Execute(kinit_if_needed, |
| user=component_user, |
| path='/bin' |
| ) |
| |
| source_and_dest_pairs = [(component_tar_source_file, destination_file), ] |
| return _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed) |
| |
| def createHdfsResources(): |
| Logger.info("Creating hdfs directories...") |
| params.HdfsResource(format('{hdfs_path_prefix}/atshistory'), user='hdfs', change_permissions_for_parents=True, owner='yarn', group='hadoop', type='directory', action= ['create_on_execute'], mode=0755) |
| params.HdfsResource(format('{hdfs_path_prefix}/user/hcat'), owner='hcat', type='directory', action=['create_on_execute'], mode=0755) |
| params.HdfsResource(format('{hdfs_path_prefix}/hive/warehouse'), owner='hive', type='directory', action=['create_on_execute'], mode=0777) |
| params.HdfsResource(format('{hdfs_path_prefix}/user/hive'), owner='hive', type='directory', action=['create_on_execute'], mode=0755) |
| params.HdfsResource(format('{hdfs_path_prefix}/tmp'), mode=0777, action=['create_on_execute'], type='directory', owner='hdfs') |
| params.HdfsResource(format('{hdfs_path_prefix}/user/ambari-qa'), type='directory', action=['create_on_execute'], mode=0770) |
| params.HdfsResource(format('{hdfs_path_prefix}/user/oozie'), owner='oozie', type='directory', action=['create_on_execute'], mode=0775) |
| params.HdfsResource(format('{hdfs_path_prefix}/app-logs'), recursive_chmod=True, owner='yarn', group='hadoop', type='directory', action=['create_on_execute'], mode=0777) |
| params.HdfsResource(format('{hdfs_path_prefix}/tmp/entity-file-history/active'), owner='yarn', group='hadoop', type='directory', action=['create_on_execute']) |
| params.HdfsResource(format('{hdfs_path_prefix}/mapred'), owner='mapred', type='directory', action=['create_on_execute']) |
| params.HdfsResource(format('{hdfs_path_prefix}/mapred/system'), owner='hdfs', type='directory', action=['create_on_execute']) |
| params.HdfsResource(format('{hdfs_path_prefix}/mr-history/done'), change_permissions_for_parents=True, owner='mapred', group='hadoop', type='directory', action=['create_on_execute'], mode=0777) |
| params.HdfsResource(format('{hdfs_path_prefix}/atshistory/done'), owner='yarn', group='hadoop', type='directory', action=['create_on_execute'], mode=0700) |
| params.HdfsResource(format('{hdfs_path_prefix}/atshistory/active'), owner='yarn', group='hadoop', type='directory', action=['create_on_execute'], mode=01777) |
| params.HdfsResource(format('{hdfs_path_prefix}/ams/hbase'), owner='ams', type='directory', action=['create_on_execute'], mode=0775) |
| params.HdfsResource(format('{hdfs_path_prefix}/amshbase/staging'), owner='ams', type='directory', action=['create_on_execute'], mode=0711) |
| params.HdfsResource(format('{hdfs_path_prefix}/user/ams/hbase'), owner='ams', type='directory', action=['create_on_execute'], mode=0775) |
| params.HdfsResource(format('{hdfs_path_prefix}/hdp'), owner='hdfs', type='directory', action=['create_on_execute'], mode=0755) |
| params.HdfsResource(format('{hdfs_path_prefix}/user/spark'), owner='spark', group='hadoop', type='directory', action=['create_on_execute'], mode=0775) |
| params.HdfsResource(format('{hdfs_path_prefix}/user/livy'), owner='livy', group='hadoop', type='directory', action=['create_on_execute'], mode=0775) |
| params.HdfsResource(format('{hdfs_path_prefix}/hdp/spark-events'), owner='spark', group='hadoop', type='directory', action=['create_on_execute'], mode=0777) |
| params.HdfsResource(format('{hdfs_path_prefix}/hdp/spark2-events'), owner='spark', group='hadoop', type='directory', action=['create_on_execute'], mode=0777) |
| params.HdfsResource(format('{hdfs_path_prefix}/hbase'), owner='hbase', type='directory', action=['create_on_execute']) |
| params.HdfsResource(format('{hdfs_path_prefix}/apps/hbase/staging'), owner='hbase', type='directory', action=['create_on_execute'], mode=0711) |
| params.HdfsResource(format('{hdfs_path_prefix}/user/hbase'), owner='hbase', type='directory', action=['create_on_execute'], mode=0755) |
| params.HdfsResource(format('{hdfs_path_prefix}/apps/zeppelin'), owner='zeppelin', group='hadoop', type='directory', action=['create_on_execute']) |
| params.HdfsResource(format('{hdfs_path_prefix}/user/zeppelin'), owner='zeppelin', group='hadoop', type='directory', action=['create_on_execute']) |
| params.HdfsResource(format('{hdfs_path_prefix}/user/zeppelin/test'), owner='zeppelin', group='hadoop', type='directory', action=['create_on_execute']) |
| |
| def copy_zeppelin_dependencies_to_hdfs(file_pattern): |
| spark_deps_full_path = glob.glob(file_pattern) |
| if spark_deps_full_path and os.path.exists(spark_deps_full_path[0]): |
| copy_tarballs_to_hdfs(spark_deps_full_path[0], hdfs_path_prefix+'/apps/zeppelin/', 'hadoop-mapreduce-historyserver', params.hdfs_user, 'zeppelin', 'zeppelin') |
| else: |
| Logger.info('zeppelin-spark-dependencies not found at %s.' % file_pattern) |
| |
| def putCreatedHdfsResourcesToIgnore(env): |
| if not 'hdfs_files' in env.config: |
| Logger.info("Not creating .hdfs_resource_ignore as no resources to use.") |
| return |
| |
| file_content = "" |
| for file in env.config['hdfs_files']: |
| if not file['target'].startswith(hdfs_path_prefix): |
| raise Exception("Something created outside hdfs_path_prefix!") |
| file_content += file['target'][len(hdfs_path_prefix):] |
| file_content += "\n" |
| |
| with open("/var/lib/ambari-agent/data/.hdfs_resource_ignore", "a+") as fp: |
| fp.write(file_content) |
| |
| def putSQLDriverToOozieShared(): |
| params.HdfsResource(hdfs_path_prefix + '/user/oozie/share/lib/sqoop/{0}'.format(sql_driver_filename), |
| owner='hdfs', type='file', action=['create_on_execute'], mode=0644, source=options.sql_driver_path) |
| |
| def recreate_slider_tarball(): |
| """ |
| Re-create tarball to include extra jars, which were put into slider lib dir. |
| """ |
| Logger.info(format("Re-creating {slider_tarball}")) |
| with closing(tarfile.open(params.slider_tarball, "w:gz")) as tar: |
| for filepath in glob.glob(format("{slider_lib_dir}/*.jar")): |
| tar.add(os.path.realpath(filepath), arcname=os.path.basename(filepath)) |
| |
| env.set_params(params) |
| hadoop_conf_dir = params.hadoop_conf_dir |
| |
| Directory('/var/lib/ambari-agent/tmp/hadoop_java_io_tmpdir', |
| owner=params.hdfs_user, |
| group=params.user_group, |
| mode=01777 |
| ) |
| Directory('/var/log/hadoop', |
| create_parents = True, |
| owner='root', |
| group=params.user_group, |
| mode=0775, |
| cd_access='a', |
| ) |
| Directory('/var/run/hadoop', |
| create_parents = True, |
| owner='root', |
| group='root', |
| cd_access='a', |
| ) |
| Directory('/tmp/hadoop-hdfs', |
| create_parents = True, |
| owner=params.hdfs_user, |
| cd_access='a', |
| ) |
| Directory('/tmp/hbase-hbase', |
| owner='hbase', |
| mode=0775, |
| create_parents = True, |
| cd_access="a", |
| ) |
| |
| oozie_libext_dir = params.oozie_libext_dir |
| oozie_home=params.oozie_home |
| configure_cmds = [] |
| configure_cmds.append(('tar','-xvf', oozie_home + '/oozie-sharelib.tar.gz','-C', oozie_home)) |
| configure_cmds.append(('cp', "/usr/share/HDP-oozie/ext-2.2.zip", options.sql_driver_path, oozie_libext_dir)) |
| configure_cmds.append(('chown', 'oozie:hadoop', oozie_libext_dir + "/ext-2.2.zip", oozie_libext_dir + "/" + sql_driver_filename)) |
| |
| no_op_test = "ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1" |
| |
| File("/etc/oozie/conf/oozie-env.sh", |
| owner=params.oozie_user, |
| content=params.oozie_env_sh_template |
| ) |
| |
| hashcode_file = format("{oozie_home}/.hashcode") |
| skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share") |
| |
| Execute( configure_cmds, |
| not_if = format("{no_op_test} || {skip_recreate_sharelib}"), |
| sudo = True, |
| ) |
| |
| File(hashcode_file, |
| mode = 0644, |
| ) |
| |
| prepare_war(params) |
| |
| oozie_shared_lib = format("/usr/hdp/{stack_version}/oozie/share") |
| oozie_user = 'oozie' |
| oozie_hdfs_user_dir = format("{hdfs_path_prefix}/user/{oozie_user}") |
| kinit_if_needed = '' |
| |
| recreate_slider_tarball() |
| |
| if options.upgrade: |
| Logger.info("Skipping uploading oozie shared lib during upgrade") |
| else: |
| params.HdfsResource(format("{oozie_hdfs_user_dir}/share/"), |
| action="delete_on_execute", |
| type = 'directory' |
| ) |
| |
| spark_client_dir = format("/usr/hdp/{stack_version}/spark") |
| |
| if os.path.exists(spark_client_dir): |
| try: |
| # Rename /usr/hdp/{stack_version}/oozie/share/lib/spark to spark-orig |
| if not os.path.exists(format("{oozie_shared_lib}/lib/spark-orig")): |
| Execute(("mv", |
| format("{oozie_shared_lib}/lib/spark"), |
| format("{oozie_shared_lib}/lib/spark-orig")), |
| sudo=True) |
| |
| # Create /usr/hdp/{stack_version}/oozie/share/lib/spark |
| if not os.path.exists(format("{oozie_shared_lib}/lib/spark")): |
| Execute(('mkdir', format('{oozie_shared_lib}/lib/spark')), |
| sudo=True) |
| |
| # Copy oozie-sharelib-spark from /usr/hdp/{stack_version}/oozie/share/lib/spark-orig to spark |
| Execute(format("cp -f {oozie_shared_lib}/lib/spark-orig/oozie-sharelib-spark*.jar {oozie_shared_lib}/lib/spark")) |
| |
| # Copy /usr/hdp/{stack_version}/spark-client/*.jar except spark-examples*.jar |
| Execute(format("cp -P {spark_client_dir}/lib/*.jar {oozie_shared_lib}/lib/spark")) |
| Execute(format("find {oozie_shared_lib}/lib/spark/ -type l -delete")) |
| try: |
| Execute(format("rm -f {oozie_shared_lib}/lib/spark/spark-examples*.jar")) |
| except: |
| Logger.warning("No spark-examples jar files found in Spark client lib.") |
| |
| # Copy /usr/hdp/{stack_version}/spark-client/python/lib/*.zip & *.jar to /usr/hdp/{stack_version}/oozie/share/lib/spark |
| Execute(format("cp -f {spark_client_dir}/python/lib/*.zip {oozie_shared_lib}/lib/spark")) |
| |
| try: |
| Execute(format("cp -f {spark_client_dir}/python/lib/*.jar {oozie_shared_lib}/lib/spark")) |
| except: |
| Logger.warning("No jar files found in Spark client python lib.") |
| |
| Execute(("chmod", "-R", "0755", format('{oozie_shared_lib}/lib/spark')), |
| sudo=True) |
| |
| # Skipping this step since it might cause issues to automated scripts that rely on hdfs://user/oozie/share/lib |
| # Rename /usr/hdp/{stack_version}/oozie/share/lib to lib_ts |
| # millis = int(round(time.time() * 1000)) |
| # Execute(("mv", |
| # format("{oozie_shared_lib}/lib"), |
| # format("{oozie_shared_lib}/lib_{millis}")), |
| # sudo=True) |
| except Exception, e: |
| Logger.warning('Exception occurred while preparing oozie share lib: '+ repr(e)) |
| |
| params.HdfsResource(format("{oozie_hdfs_user_dir}/share"), |
| action="create_on_execute", |
| type = 'directory', |
| mode=0755, |
| recursive_chmod = True, |
| owner=oozie_user, |
| source = oozie_shared_lib, |
| ) |
| |
| Logger.info("Copying tarballs...") |
| # TODO, these shouldn't hardcode the stack root or destination stack name. |
| copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/hadoop/mapreduce.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/mapreduce/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group) |
| copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/tez/lib/tez.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/tez/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group) |
| copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/hive/hive.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/hive/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group) |
| |
| # Needed by Hive Interactive |
| copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/tez_hive2/lib/tez.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/tez_hive2/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group) |
| |
| copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/pig/pig.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/pig/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group) |
| copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/hadoop-mapreduce/hadoop-streaming.jar"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/mapreduce/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group) |
| copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/sqoop/sqoop.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/sqoop/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group) |
| copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/slider/lib/slider.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/slider/", 'hadoop-mapreduce-historyserver', params.hdfs_user, params.hdfs_user, params.user_group) |
| |
| createHdfsResources() |
| copy_zeppelin_dependencies_to_hdfs(format("/usr/hdp/{stack_version}/zeppelin/interpreter/spark/dep/zeppelin-spark-dependencies*.jar")) |
| putSQLDriverToOozieShared() |
| putCreatedHdfsResourcesToIgnore(env) |
| |
| # jar shouldn't be used before (read comment below) |
| File(format("{ambari_libs_dir}/fast-hdfs-resource.jar"), |
| mode=0644, |
| content=StaticFile("/var/lib/ambari-agent/cache/stacks/HDP/2.0.6/hooks/before-START/files/fast-hdfs-resource.jar") |
| ) |
| # Create everything in one jar call (this is fast). |
| # (! Before everything should be executed with action="create_on_execute/delete_on_execute" for this time-optimization to work) |
| try: |
| params.HdfsResource(None, |
| logoutput=True, |
| action="execute" |
| ) |
| except: |
| os.remove("/var/lib/ambari-agent/data/.hdfs_resource_ignore") |
| raise |
| Logger.info("Completed tarball copy.") |
| |
| if not options.upgrade: |
| Logger.info("Executing stack-selector-tool for stack {0} ...".format(stack_version)) |
| Execute( |
| ('/usr/bin/hdp-select', 'set', 'all', stack_version), |
| sudo = True |
| ) |
| |
| Logger.info("Ambari preupload script completed.") |
