| #!/usr/bin/env python |
| # -*- coding: utf-8 -*- |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one or more |
| # contributor license agreements. See the NOTICE file distributed with |
| # this work for additional information regarding copyright ownership. |
| # The ASF licenses this file to You under the Apache License, Version 2.0 |
| # (the "License"); you may not use this file except in compliance with |
| # the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| |
| """Invokes hbase shell after retrieving effective hbase-site.xml from a live Slider HBase cluster |
| First argument is the name of cluster instance |
| """ |
| import os |
| import subprocess |
| from os.path import expanduser |
| from os.path import exists |
| import glob |
| import getopt |
| import re |
| import fnmatch |
| import shutil |
| import logging |
| import socket |
| from string import Template |
| import time |
| import fileinput |
| import sys |
| import tempfile |
| import json |
| import datetime |
| from xml.dom import minidom |
| from xml.dom.minidom import parseString |
| import xml.etree.ElementTree as ET |
| import urllib2 |
| import hashlib |
| import random |
| import httplib, ssl |
| |
| # find path to given command |
| def which(program): |
| def is_exe(fpath): |
| return os.path.isfile(fpath) and os.access(fpath, os.X_OK) |
| |
| fpath, fname = os.path.split(program) |
| if fpath: |
| if is_exe(program): |
| return program |
| else: |
| for path in os.environ["PATH"].split(os.pathsep): |
| path = path.strip('"') |
| exe_file = os.path.join(path, program) |
| if is_exe(exe_file): |
| return exe_file |
| return None |
| |
| SLIDER_DIR = os.getenv('SLIDER_HOME', None) |
| if SLIDER_DIR == None or (not os.path.exists(SLIDER_DIR)): |
| SLIDER_CMD = which("slider") |
| if SLIDER_DIR == None: |
| if os.path.exists("/usr/bin/slider"): |
| SLIDER_CMD = "/usr/bin/slider" |
| else: |
| print "Unable to find SLIDER_HOME or slider command. Please configure SLIDER_HOME before running hbase-slider" |
| sys.exit(1) |
| else: |
| SLIDER_CMD = os.path.join(SLIDER_DIR, 'bin', 'slider.py') |
| |
| HBASE_TMP_DIR=os.path.join(tempfile.gettempdir(), "hbase-temp") |
| |
| # call slider command |
| def call(cmd): |
| print "Running: " + " ".join(cmd) |
| retcode = subprocess.call(cmd) |
| if retcode != 0: |
| raise Exception("return code from running %s was %d" % (cmd[0], retcode)) |
| |
| # Write text into a file |
| # wtext - Text to write |
| def writeToFile(wtext, outfile, isAppend=False): |
| mode = 'w' |
| if isAppend: |
| mode = 'a+' |
| outf = open(outfile, mode) |
| try: |
| outf.write(wtext) |
| finally: |
| outf.close() |
| |
| # Update the XML configuration properties and write to another file |
| # infile - Input config XML file |
| # outfile - Output config XML file |
| # propertyMap - Properties to add/update |
| # {'name1':'value1', 'name2':'value2',...} |
| def writePropertiesToConfigXMLFile(infile, outfile, propertyMap): |
| xmldoc = minidom.parse(infile) |
| cfgnode = xmldoc.getElementsByTagName("configuration") |
| if len(cfgnode) == 0: |
| raise Exception("Invalid Config XML file: " + infile) |
| cfgnode = cfgnode[0] |
| propertyMapKeys = propertyMap.keys() |
| removeProp = [] |
| modified = [] |
| for node in xmldoc.getElementsByTagName("name"): |
| name = node.childNodes[0].nodeValue.strip() |
| if name in propertyMapKeys: |
| modified.append(name) |
| for vnode in node.parentNode.childNodes: |
| if vnode.nodeName == "value": |
| if vnode.childNodes == []: |
| removeProp.append(name) |
| modified.remove(name) |
| else: |
| vnode.childNodes[0].nodeValue = propertyMap[name] |
| remaining = list(set(propertyMapKeys) - set(modified)) |
| # delete properties whose value is set to None e.g.<value></value> |
| for node in xmldoc.getElementsByTagName("name"): |
| name = node.childNodes[0].nodeValue.strip() |
| if name in removeProp: |
| parent = node.parentNode |
| super = parent.parentNode |
| super.removeChild(parent) |
| for property in remaining: |
| pn = xmldoc.createElement("property") |
| nn = xmldoc.createElement("name") |
| ntn = xmldoc.createTextNode(property) |
| nn.appendChild(ntn) |
| pn.appendChild(nn) |
| vn = xmldoc.createElement("value") |
| vtn = xmldoc.createTextNode(str(propertyMap[property])) |
| vn.appendChild(vtn) |
| pn.appendChild(vn) |
| cfgnode.appendChild(pn) |
| writeToFile(xmldoc.toxml(), outfile) |
| |
| def install(cluster_instance, dir): |
| """Syntax: [hbase-slider cluster_instance install dir] |
| Installs a fully configured hbase client in the specified dir |
| The resulting client may be used on its own without hbase-slider |
| """ |
| if os.path.exists(dir): |
| raise Exception("Install dir must not exist: " + dir) |
| |
| workdir = os.path.join(tempfile.gettempdir(), 'install-work-dir') |
| |
| statusfile = os.path.join(workdir, 'status.json') |
| cmd = [SLIDER_CMD, "status", cluster_instance, "--out", statusfile] |
| call(cmd) |
| |
| infile = open(statusfile) |
| try: |
| content = json.load(infile) |
| finally: |
| infile.close() |
| |
| appdef = content['options']['application.def'] |
| appdeffile = appdef[appdef.rfind('/')+1:] |
| cmd = ["hadoop", "fs", "-copyToLocal", appdef, workdir] |
| call(cmd) |
| |
| cmd = ["unzip", os.path.join(workdir, appdeffile), "-d", workdir] |
| call(cmd) |
| |
| gzfile = glob.glob(os.path.join(workdir, 'package', 'files', 'hbase*gz')) |
| if len(gzfile) != 1: |
| raise Exception("got " + gzfile + " from glob") |
| cmd = ["tar", "xvzf", gzfile[0], '-C', workdir] |
| call(cmd) |
| |
| tmp_hbase = glob.glob(os.path.join(workdir, 'hbase-[.0-9]*')) |
| if len(tmp_hbase) != 1: |
| raise Exception("got " + tmp_hbase + " from glob") |
| tmp_hbase = tmp_hbase[0] |
| |
| confdir = os.path.join(tmp_hbase, 'conf') |
| tmpHBaseConfFile=os.path.join(tempfile.gettempdir(), "hbase-site.xml") |
| |
| call([SLIDER_CMD, "registry", "--getconf", "hbase-site", "--user", "hbase", "--format", "xml", "--dest", tmpHBaseConfFile, "--name", cluster_instance]) |
| global HBASE_TMP_DIR |
| propertyMap = {'hbase.tmp.dir' : HBASE_TMP_DIR, "instance" : cluster_instance} |
| writePropertiesToConfigXMLFile(tmpHBaseConfFile, os.path.join(confdir, "hbase-site.xml"), propertyMap) |
| |
| libdir = os.path.join(tmp_hbase, 'lib') |
| for jar in glob.glob(os.path.join(workdir, 'package', 'files', '*jar')): |
| shutil.move(jar, libdir) |
| shutil.move(tmp_hbase, dir) |
| |
| def quicklinks(app_name): |
| """Syntax: [hbase-slider appname quicklinks] |
| Prints the quicklinks information of hbase-slider registry |
| """ |
| cmd = [SLIDER_CMD, "registry", "--getconf", "quicklinks", "--format", "json", |
| "--name", app_name] |
| |
| call(cmd) |
| |
| home = expanduser("~") |
| if len(sys.argv) < 2: |
| print "optionally you can specify the output directory for conf dir using:" |
| print " --appconf=<dir>" |
| print "optionally you can specify the (existing) directory for hbase conf files (as template) using:" |
| print " --hbaseconf=<dir>" |
| print "optionally you can specify the age in number of hours beyond which hbase-site.xml would be retrieved from slider cluster" |
| print " --ttl=<age of hbase-site.xml>" |
| print "the name of cluster instance is required as the first parameter following options" |
| print "the second parameter can be:" |
| print " shell (default) - activates hbase shell based on retrieved hbase-site.xml" |
| print " quicklinks - prints quicklinks from registry" |
| print " install <dir> - installs hbase client into <dir>" |
| sys.exit(1) |
| |
| try: |
| opts, args = getopt.getopt(sys.argv[1:], "", ["appconf=", "hbaseconf=", "ttl="]) |
| except getopt.GetoptError as err: |
| # print help information and exit: |
| print str(err) |
| sys.exit(2) |
| |
| cluster_instance=args[0] |
| local_conf_dir=os.path.join(home, cluster_instance, 'conf') |
| hbase_conf_dir="/etc/hbase/conf" |
| ttl=0 |
| for o, a in opts: |
| if o == "--appconf": |
| local_conf_dir = a |
| elif o == "--hbaseconf": |
| hbase_conf_dir = a |
| elif o == "--ttl": |
| ttl = a |
| |
| if len(args) > 1: |
| if args[1] == 'quicklinks': |
| quicklinks(cluster_instance) |
| sys.exit(0) |
| elif args[1] == 'install': |
| install(cluster_instance, args[2]) |
| sys.exit(0) |
| |
| needToRetrieve=True |
| HBaseConfFile=os.path.join(local_conf_dir, "hbase-site.xml") |
| if not exists(local_conf_dir): |
| shutil.copytree(hbase_conf_dir, local_conf_dir) |
| else: |
| if exists(HBaseConfFile): |
| diff = os.path.getmtime(HBaseConfFile)-int(time.time()) |
| diff = diff / 60 / 60 |
| print HBaseConfFile + " is " + str(diff) + " hours old" |
| if diff < ttl: |
| needToRetrieve=False |
| |
| if needToRetrieve: |
| tmpHBaseConfFile=os.path.join(tempfile.gettempdir(), "hbase-site.xml") |
| |
| call([SLIDER_CMD, "registry", "--getconf", "hbase-site", "--user", "hbase", "--format", "xml", "--dest", tmpHBaseConfFile, "--name", cluster_instance]) |
| propertyMap = {'hbase.tmp.dir' : HBASE_TMP_DIR, "instance" : cluster_instance} |
| writePropertiesToConfigXMLFile(tmpHBaseConfFile, HBaseConfFile, propertyMap) |
| print "hbase configuration is saved in " + HBaseConfFile |
| |
| call(["hbase", "--config", local_conf_dir, "shell"]) |