src/java/org/apache/hcatalog/cli/HCatCli.java - hcatalog - Git at Google

 /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.hcatalog.cli;

 import java.io.BufferedReader;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.PrintStream;
 import java.io.PrintWriter;
 import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
 import java.util.Properties;

 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.GnuParser;
 import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
 import org.apache.commons.cli.Parser;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hive.cli.CliSessionState;
 import org.apache.hadoop.hive.common.LogUtils;
 import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
 import org.apache.hadoop.hive.ql.CommandNeedRetryException;
 import org.apache.hadoop.hive.ql.processors.DfsProcessor;
 import org.apache.hadoop.hive.ql.processors.SetProcessor;
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
 import org.apache.hcatalog.common.HCatConstants;
 import org.apache.hcatalog.common.HCatUtil;

 public class HCatCli {

     @SuppressWarnings("static-access")
     public static void main(String[] args) {

         try {
             LogUtils.initHiveLog4j();
         } catch (LogInitializationException e) {

         }

         CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class));
         ss.in = System.in;
         try {
             ss.out = new PrintStream(System.out, true, "UTF-8");
             ss.err = new PrintStream(System.err, true, "UTF-8");
         } catch (UnsupportedEncodingException e) {
             System.exit(1);
         }

         HiveConf conf = ss.getConf();

         HiveConf.setVar(conf, ConfVars.SEMANTIC_ANALYZER_HOOK, HCatSemanticAnalyzer.class.getName());

         SessionState.start(ss);

         Options options = new Options();

         // -e 'quoted-query-string'
         options.addOption(OptionBuilder
                 .hasArg()
                 .withArgName("exec")
                 .withDescription("hcat command given from command line")
                 .create('e'));

         // -f <query-file>
         options.addOption(OptionBuilder
                 .hasArg()
                 .withArgName("file")
                 .withDescription("hcat commands in file")
                 .create('f'));

         // -g
         options.addOption(OptionBuilder
                 .hasArg().
                 withArgName("group").
                 withDescription("group for the db/table specified in CREATE statement").
                 create('g'));

         // -p
         options.addOption(OptionBuilder
                 .hasArg()
                 .withArgName("perms")
                 .withDescription("permissions for the db/table specified in CREATE statement")
                 .create('p'));

         // -D
         options.addOption(OptionBuilder
                 .hasArgs(2)
                 .withArgName("property=value")
                 .withValueSeparator()
                 .withDescription("use hadoop value for given property")
                 .create('D'));

         // [-h|--help]
         options.addOption(new Option("h", "help", false, "Print help information"));

         Parser parser = new GnuParser();
         CommandLine cmdLine = null;

         try {
             cmdLine = parser.parse(options, args);

         } catch (ParseException e) {
             printUsage(options, ss.err);
             System.exit(1);
         }
         // -e
         String execString = (String) cmdLine.getOptionValue('e');
         // -f
         String fileName = (String) cmdLine.getOptionValue('f');
         // -h
         if (cmdLine.hasOption('h')) {
             printUsage(options, ss.out);
             System.exit(0);
         }

         if (execString != null && fileName != null) {
             ss.err.println("The '-e' and '-f' options cannot be specified simultaneously");
             printUsage(options, ss.err);
             System.exit(1);
         }

         // -p
         String perms = (String) cmdLine.getOptionValue('p');
         if (perms != null) {
             validatePermissions(ss, conf, perms);
         }

         // -g
         String grp = (String) cmdLine.getOptionValue('g');
         if (grp != null) {
             conf.set(HCatConstants.HCAT_GROUP, grp);
         }

         // -D
         setConfProperties(conf, cmdLine.getOptionProperties("D"));

         if (execString != null) {
             System.exit(processLine(execString));
         }

         try {
             if (fileName != null) {
                 System.exit(processFile(fileName));
             }
         } catch (FileNotFoundException e) {
             ss.err.println("Input file not found. (" + e.getMessage() + ")");
             System.exit(1);
         } catch (IOException e) {
             ss.err.println("Could not open input file for reading. (" + e.getMessage() + ")");
             System.exit(1);
         }

         // -h
         printUsage(options, ss.err);
         System.exit(1);
     }

     private static void setConfProperties(HiveConf conf, Properties props) {
         for (java.util.Map.Entry<Object, Object> e : props.entrySet())
             conf.set((String) e.getKey(), (String) e.getValue());
     }

     private static int processLine(String line) {
         int ret = 0;

         String command = "";
         for (String oneCmd : line.split(";")) {

             if (StringUtils.endsWith(oneCmd, "\\")) {
                 command += StringUtils.chop(oneCmd) + ";";
                 continue;
             } else {
                 command += oneCmd;
             }
             if (StringUtils.isBlank(command)) {
                 continue;
             }

             ret = processCmd(command);
             command = "";
         }
         return ret;
     }

     private static int processFile(String fileName) throws IOException {
         FileReader fileReader = null;
         BufferedReader reader = null;
         try {
             fileReader = new FileReader(fileName);
             reader = new BufferedReader(fileReader);
             String line;
             StringBuilder qsb = new StringBuilder();

             while ((line = reader.readLine()) != null) {
                 qsb.append(line + "\n");
             }

             return (processLine(qsb.toString()));
         } finally {
             if (fileReader != null) {
                 fileReader.close();
             }
             if (reader != null) {
                 reader.close();
             }
         }
     }

     private static int processCmd(String cmd) {

         SessionState ss = SessionState.get();
         long start = System.currentTimeMillis();

         cmd = cmd.trim();
         String firstToken = cmd.split("\\s+")[0].trim();

         if (firstToken.equalsIgnoreCase("set")) {
             return new SetProcessor().run(cmd.substring(firstToken.length()).trim()).getResponseCode();
         } else if (firstToken.equalsIgnoreCase("dfs")) {
             return new DfsProcessor(ss.getConf()).run(cmd.substring(firstToken.length()).trim()).getResponseCode();
         }

         HCatDriver driver = new HCatDriver();

         int ret = driver.run(cmd).getResponseCode();

         if (ret != 0) {
             driver.close();
             System.exit(ret);
         }

         ArrayList<String> res = new ArrayList<String>();
         try {
             while (driver.getResults(res)) {
                 for (String r : res) {
                     ss.out.println(r);
                 }
                 res.clear();
             }
         } catch (IOException e) {
             ss.err.println("Failed with exception " + e.getClass().getName() + ":"
                 + e.getMessage() + "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
             ret = 1;
         } catch (CommandNeedRetryException e) {
             ss.err.println("Failed with exception " + e.getClass().getName() + ":"
                 + e.getMessage() + "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
             ret = 1;
         }

         int cret = driver.close();
         if (ret == 0) {
             ret = cret;
         }

         long end = System.currentTimeMillis();
         if (end > start) {
             double timeTaken = (end - start) / 1000.0;
             ss.err.println("Time taken: " + timeTaken + " seconds");
         }
         return ret;
     }

     private static void printUsage(Options options, OutputStream os) {
         PrintWriter pw = new PrintWriter(os);
         new HelpFormatter().printHelp(pw, 2 * HelpFormatter.DEFAULT_WIDTH,
             "hcat { -e \"<query>\" | -f \"<filepath>\" } [ -g \"<group>\" ] [ -p \"<perms>\" ] [ -D\"<name>=<value>\" ]",
             null, options, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD,
             null, false);
         pw.flush();
     }

     private static void validatePermissions(CliSessionState ss, HiveConf conf, String perms) {
         perms = perms.trim();
         FsPermission fp = null;

         if (perms.matches("^\\s*([r,w,x,-]{9})\\s*$")) {
             fp = FsPermission.valueOf("d" + perms);
         } else if (perms.matches("^\\s*([0-7]{3})\\s*$")) {
             fp = new FsPermission(Short.decode("0" + perms));
         } else {
             ss.err.println("Invalid permission specification: " + perms);
             System.exit(1);
         }

         if (!HCatUtil.validateMorePermissive(fp.getUserAction(), fp.getGroupAction())) {
             ss.err.println("Invalid permission specification: " + perms + " : user permissions must be more permissive than group permission ");
             System.exit(1);
         }
         if (!HCatUtil.validateMorePermissive(fp.getGroupAction(), fp.getOtherAction())) {
             ss.err.println("Invalid permission specification: " + perms + " : group permissions must be more permissive than other permission ");
             System.exit(1);
         }
         if ((!HCatUtil.validateExecuteBitPresentIfReadOrWrite(fp.getUserAction())) ||
             (!HCatUtil.validateExecuteBitPresentIfReadOrWrite(fp.getGroupAction())) ||
             (!HCatUtil.validateExecuteBitPresentIfReadOrWrite(fp.getOtherAction()))) {
             ss.err.println("Invalid permission specification: " + perms + " : permissions must have execute permissions if read or write permissions are specified ");
             System.exit(1);
         }

         conf.set(HCatConstants.HCAT_PERMS, "d" + fp.toString());

     }


 }
	/**
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.hcatalog.cli;

	import java.io.BufferedReader;
	import java.io.FileNotFoundException;
	import java.io.FileReader;
	import java.io.IOException;
	import java.io.OutputStream;
	import java.io.PrintStream;
	import java.io.PrintWriter;
	import java.io.UnsupportedEncodingException;
	import java.util.ArrayList;
	import java.util.Properties;

	import org.apache.commons.cli.CommandLine;
	import org.apache.commons.cli.GnuParser;
	import org.apache.commons.cli.HelpFormatter;
	import org.apache.commons.cli.Option;
	import org.apache.commons.cli.OptionBuilder;
	import org.apache.commons.cli.Options;
	import org.apache.commons.cli.ParseException;
	import org.apache.commons.cli.Parser;
	import org.apache.commons.lang.StringUtils;
	import org.apache.hadoop.fs.permission.FsPermission;
	import org.apache.hadoop.hive.cli.CliSessionState;
	import org.apache.hadoop.hive.common.LogUtils;
	import org.apache.hadoop.hive.common.LogUtils.LogInitializationException;
	import org.apache.hadoop.hive.conf.HiveConf;
	import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
	import org.apache.hadoop.hive.ql.CommandNeedRetryException;
	import org.apache.hadoop.hive.ql.processors.DfsProcessor;
	import org.apache.hadoop.hive.ql.processors.SetProcessor;
	import org.apache.hadoop.hive.ql.session.SessionState;
	import org.apache.hcatalog.cli.SemanticAnalysis.HCatSemanticAnalyzer;
	import org.apache.hcatalog.common.HCatConstants;
	import org.apache.hcatalog.common.HCatUtil;

	public class HCatCli {

	@SuppressWarnings("static-access")
	public static void main(String[] args) {

	try {
	LogUtils.initHiveLog4j();
	} catch (LogInitializationException e) {

	}

	CliSessionState ss = new CliSessionState(new HiveConf(SessionState.class));
	ss.in = System.in;
	try {
	ss.out = new PrintStream(System.out, true, "UTF-8");
	ss.err = new PrintStream(System.err, true, "UTF-8");
	} catch (UnsupportedEncodingException e) {
	System.exit(1);
	}

	HiveConf conf = ss.getConf();

	HiveConf.setVar(conf, ConfVars.SEMANTIC_ANALYZER_HOOK, HCatSemanticAnalyzer.class.getName());

	SessionState.start(ss);

	Options options = new Options();

	// -e 'quoted-query-string'
	options.addOption(OptionBuilder
	.hasArg()
	.withArgName("exec")
	.withDescription("hcat command given from command line")
	.create('e'));

	// -f <query-file>
	options.addOption(OptionBuilder
	.hasArg()
	.withArgName("file")
	.withDescription("hcat commands in file")
	.create('f'));

	// -g
	options.addOption(OptionBuilder
	.hasArg().
	withArgName("group").
	withDescription("group for the db/table specified in CREATE statement").
	create('g'));

	// -p
	options.addOption(OptionBuilder
	.hasArg()
	.withArgName("perms")
	.withDescription("permissions for the db/table specified in CREATE statement")
	.create('p'));

	// -D
	options.addOption(OptionBuilder
	.hasArgs(2)
	.withArgName("property=value")
	.withValueSeparator()
	.withDescription("use hadoop value for given property")
	.create('D'));

	// [-h\|--help]
	options.addOption(new Option("h", "help", false, "Print help information"));

	Parser parser = new GnuParser();
	CommandLine cmdLine = null;

	try {
	cmdLine = parser.parse(options, args);

	} catch (ParseException e) {
	printUsage(options, ss.err);
	System.exit(1);
	}
	// -e
	String execString = (String) cmdLine.getOptionValue('e');
	// -f
	String fileName = (String) cmdLine.getOptionValue('f');
	// -h
	if (cmdLine.hasOption('h')) {
	printUsage(options, ss.out);
	System.exit(0);
	}

	if (execString != null && fileName != null) {
	ss.err.println("The '-e' and '-f' options cannot be specified simultaneously");
	printUsage(options, ss.err);
	System.exit(1);
	}

	// -p
	String perms = (String) cmdLine.getOptionValue('p');
	if (perms != null) {
	validatePermissions(ss, conf, perms);
	}

	// -g
	String grp = (String) cmdLine.getOptionValue('g');
	if (grp != null) {
	conf.set(HCatConstants.HCAT_GROUP, grp);
	}

	// -D
	setConfProperties(conf, cmdLine.getOptionProperties("D"));

	if (execString != null) {
	System.exit(processLine(execString));
	}

	try {
	if (fileName != null) {
	System.exit(processFile(fileName));
	}
	} catch (FileNotFoundException e) {
	ss.err.println("Input file not found. (" + e.getMessage() + ")");
	System.exit(1);
	} catch (IOException e) {
	ss.err.println("Could not open input file for reading. (" + e.getMessage() + ")");
	System.exit(1);
	}

	// -h
	printUsage(options, ss.err);
	System.exit(1);
	}

	private static void setConfProperties(HiveConf conf, Properties props) {
	for (java.util.Map.Entry<Object, Object> e : props.entrySet())
	conf.set((String) e.getKey(), (String) e.getValue());
	}

	private static int processLine(String line) {
	int ret = 0;

	String command = "";
	for (String oneCmd : line.split(";")) {

	if (StringUtils.endsWith(oneCmd, "\\")) {
	command += StringUtils.chop(oneCmd) + ";";
	continue;
	} else {
	command += oneCmd;
	}
	if (StringUtils.isBlank(command)) {
	continue;
	}

	ret = processCmd(command);
	command = "";
	}
	return ret;
	}

	private static int processFile(String fileName) throws IOException {
	FileReader fileReader = null;
	BufferedReader reader = null;
	try {
	fileReader = new FileReader(fileName);
	reader = new BufferedReader(fileReader);
	String line;
	StringBuilder qsb = new StringBuilder();

	while ((line = reader.readLine()) != null) {
	qsb.append(line + "\n");
	}

	return (processLine(qsb.toString()));
	} finally {
	if (fileReader != null) {
	fileReader.close();
	}
	if (reader != null) {
	reader.close();
	}
	}
	}

	private static int processCmd(String cmd) {

	SessionState ss = SessionState.get();
	long start = System.currentTimeMillis();

	cmd = cmd.trim();
	String firstToken = cmd.split("\\s+")[0].trim();

	if (firstToken.equalsIgnoreCase("set")) {
	return new SetProcessor().run(cmd.substring(firstToken.length()).trim()).getResponseCode();
	} else if (firstToken.equalsIgnoreCase("dfs")) {
	return new DfsProcessor(ss.getConf()).run(cmd.substring(firstToken.length()).trim()).getResponseCode();
	}

	HCatDriver driver = new HCatDriver();

	int ret = driver.run(cmd).getResponseCode();

	if (ret != 0) {
	driver.close();
	System.exit(ret);
	}

	ArrayList<String> res = new ArrayList<String>();
	try {
	while (driver.getResults(res)) {
	for (String r : res) {
	ss.out.println(r);
	}
	res.clear();
	}
	} catch (IOException e) {
	ss.err.println("Failed with exception " + e.getClass().getName() + ":"
	+ e.getMessage() + "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
	ret = 1;
	} catch (CommandNeedRetryException e) {
	ss.err.println("Failed with exception " + e.getClass().getName() + ":"
	+ e.getMessage() + "\n" + org.apache.hadoop.util.StringUtils.stringifyException(e));
	ret = 1;
	}

	int cret = driver.close();
	if (ret == 0) {
	ret = cret;
	}

	long end = System.currentTimeMillis();
	if (end > start) {
	double timeTaken = (end - start) / 1000.0;
	ss.err.println("Time taken: " + timeTaken + " seconds");
	}
	return ret;
	}

	private static void printUsage(Options options, OutputStream os) {
	PrintWriter pw = new PrintWriter(os);
	new HelpFormatter().printHelp(pw, 2 * HelpFormatter.DEFAULT_WIDTH,
	"hcat { -e \"<query>\" \| -f \"<filepath>\" } [ -g \"<group>\" ] [ -p \"<perms>\" ] [ -D\"<name>=<value>\" ]",
	null, options, HelpFormatter.DEFAULT_LEFT_PAD, HelpFormatter.DEFAULT_DESC_PAD,
	null, false);
	pw.flush();
	}

	private static void validatePermissions(CliSessionState ss, HiveConf conf, String perms) {
	perms = perms.trim();
	FsPermission fp = null;

	if (perms.matches("^\\s([r,w,x,-]{9})\\s$")) {
	fp = FsPermission.valueOf("d" + perms);
	} else if (perms.matches("^\\s([0-7]{3})\\s$")) {
	fp = new FsPermission(Short.decode("0" + perms));
	} else {
	ss.err.println("Invalid permission specification: " + perms);
	System.exit(1);
	}

	if (!HCatUtil.validateMorePermissive(fp.getUserAction(), fp.getGroupAction())) {
	ss.err.println("Invalid permission specification: " + perms + " : user permissions must be more permissive than group permission ");
	System.exit(1);
	}
	if (!HCatUtil.validateMorePermissive(fp.getGroupAction(), fp.getOtherAction())) {
	ss.err.println("Invalid permission specification: " + perms + " : group permissions must be more permissive than other permission ");
	System.exit(1);
	}
	if ((!HCatUtil.validateExecuteBitPresentIfReadOrWrite(fp.getUserAction())) \|\|
	(!HCatUtil.validateExecuteBitPresentIfReadOrWrite(fp.getGroupAction())) \|\|
	(!HCatUtil.validateExecuteBitPresentIfReadOrWrite(fp.getOtherAction()))) {
	ss.err.println("Invalid permission specification: " + perms + " : permissions must have execute permissions if read or write permissions are specified ");
	System.exit(1);
	}

	conf.set(HCatConstants.HCAT_PERMS, "d" + fp.toString());

	}


	}