| /* |
| * Licensed to the Apache Software Foundation (ASF) under one |
| * or more contributor license agreements. See the NOTICE file |
| * distributed with this work for additional information |
| * regarding copyright ownership. The ASF licenses this file |
| * to you under the Apache License, Version 2.0 (the |
| * "License"); you may not use this file except in compliance |
| * with the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package org.apache.slider.common.tools; |
| |
| import com.google.common.base.Preconditions; |
| import org.apache.hadoop.conf.Configuration; |
| import org.apache.hadoop.fs.FSDataInputStream; |
| import org.apache.hadoop.fs.FSDataOutputStream; |
| import org.apache.hadoop.fs.FileSystem; |
| import org.apache.hadoop.fs.Path; |
| import org.apache.hadoop.io.IOUtils; |
| import org.apache.hadoop.registry.client.api.RegistryConstants; |
| import org.apache.slider.common.SliderKeys; |
| import org.apache.slider.common.SliderXmlConfKeys; |
| import org.apache.slider.core.exceptions.BadConfigException; |
| import org.slf4j.Logger; |
| import org.slf4j.LoggerFactory; |
| import org.w3c.dom.Document; |
| import org.xml.sax.SAXException; |
| |
| import javax.xml.parsers.DocumentBuilder; |
| import javax.xml.parsers.DocumentBuilderFactory; |
| import javax.xml.parsers.ParserConfigurationException; |
| import java.io.ByteArrayInputStream; |
| import java.io.File; |
| import java.io.FileNotFoundException; |
| import java.io.FileOutputStream; |
| import java.io.IOException; |
| import java.io.InputStream; |
| import java.io.OutputStream; |
| import java.io.StringWriter; |
| import java.net.MalformedURLException; |
| import java.net.URL; |
| import java.util.HashMap; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.TreeSet; |
| |
| /** |
| * Methods to aid in config, both in the Configuration class and |
| * with other parts of setting up Slider-initated processes. |
| * |
| * Some of the methods take an argument of a map iterable for their sources; this allows |
| * the same method |
| */ |
| public class ConfigHelper { |
| private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class); |
| |
| /** |
| * Dump the (sorted) configuration |
| * @param conf config |
| * @return the sorted keyset |
| */ |
| public static Set<String> dumpConf(Configuration conf) { |
| Set<String> keys = sortedConfigKeys(conf); |
| for (String key : keys) { |
| log.info("{}={}", key, conf.get(key)); |
| } |
| return keys; |
| } |
| |
| /** |
| * Take a configuration and return a sorted set |
| * @param conf config |
| * @return the sorted keyset |
| |
| */ |
| public static Set<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) { |
| TreeSet<String> sorted = new TreeSet<String>(); |
| for (Map.Entry<String, String> entry : conf) { |
| sorted.add(entry.getKey()); |
| } |
| return sorted; |
| } |
| |
| /** |
| * Set an entire map full of values |
| * |
| * @param config config to patch |
| * @param map map of data |
| * @param origin origin data |
| */ |
| public static void addConfigMap(Configuration config, |
| Map<String, String> map, |
| String origin) throws BadConfigException { |
| addConfigMap(config, map.entrySet(), origin); |
| } |
| |
| /** |
| * Set an entire map full of values |
| * |
| * @param config config to patch |
| * @param map map of data |
| * @param origin origin data |
| */ |
| public static void addConfigMap(Configuration config, |
| Iterable<Map.Entry<String, String>> map, |
| String origin) throws BadConfigException { |
| for (Map.Entry<String, String> mapEntry : map) { |
| String key = mapEntry.getKey(); |
| String value = mapEntry.getValue(); |
| if (value == null) { |
| throw new BadConfigException("Null value for property " + key); |
| } |
| config.set(key, value, origin); |
| } |
| } |
| |
| |
| /** |
| * Save a config file in a destination directory on a given filesystem |
| * @param systemConf system conf used for creating filesystems |
| * @param confToSave config to save |
| * @param confdir the directory path where the file is to go |
| * @param filename the filename |
| * @return the destination path where the file was saved |
| * @throws IOException IO problems |
| */ |
| public static Path saveConfig(Configuration systemConf, |
| Configuration confToSave, |
| Path confdir, |
| String filename) throws IOException { |
| FileSystem fs = FileSystem.get(confdir.toUri(), systemConf); |
| Path destPath = new Path(confdir, filename); |
| saveConfig(fs, destPath, confToSave); |
| return destPath; |
| } |
| |
| /** |
| * Save a config |
| * @param fs filesystem |
| * @param destPath dest to save |
| * @param confToSave config to save |
| * @throws IOException IO problems |
| */ |
| public static void saveConfig(FileSystem fs, |
| Path destPath, |
| Configuration confToSave) throws |
| IOException { |
| FSDataOutputStream fos = fs.create(destPath); |
| try { |
| confToSave.writeXml(fos); |
| } finally { |
| IOUtils.closeStream(fos); |
| } |
| } |
| |
| /** |
| * Convert to an XML string |
| * @param conf configuration |
| * @return conf |
| * @throws IOException |
| */ |
| public static String toXml(Configuration conf) throws IOException { |
| StringWriter writer = new StringWriter(); |
| conf.writeXml(writer); |
| return writer.toString(); |
| } |
| |
| /** |
| * This will load and parse a configuration to an XML document |
| * @param fs filesystem |
| * @param path path |
| * @return an XML document |
| * @throws IOException IO failure |
| */ |
| public Document parseConfiguration(FileSystem fs, |
| Path path) throws |
| IOException { |
| |
| |
| byte[] data = loadBytes(fs, path); |
| //this is here to track down a parse issue |
| //related to configurations |
| String s = new String(data, 0, data.length); |
| log.debug("XML resource {} is \"{}\"", path, s); |
| /* JDK7 |
| try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { |
| Document document = parseConfigXML(in); |
| return document; |
| } catch (ParserConfigurationException | SAXException e) { |
| throw new IOException(e); |
| } |
| */ |
| ByteArrayInputStream in= null; |
| try { |
| in = new ByteArrayInputStream(data); |
| Document document = parseConfigXML(in); |
| return document; |
| } catch (ParserConfigurationException e) { |
| throw new IOException(e); |
| } catch (SAXException e) { |
| throw new IOException(e); |
| } finally { |
| IOUtils.closeStream(in); |
| } |
| } |
| |
| public static byte[] loadBytes(FileSystem fs, Path path) throws IOException { |
| int len = (int) fs.getLength(path); |
| byte[] data = new byte[len]; |
| /* JDK7 |
| try(FSDataInputStream in = fs.open(path)) { |
| in.readFully(0, data); |
| } |
| */ |
| FSDataInputStream in = null; |
| in = fs.open(path); |
| try { |
| in.readFully(0, data); |
| } finally { |
| IOUtils.closeStream(in); |
| } |
| return data; |
| } |
| |
| /** |
| * Load a configuration from ANY FS path. The normal Configuration |
| * loader only works with file:// URIs |
| * @param fs filesystem |
| * @param path path |
| * @return a loaded resource |
| * @throws IOException |
| */ |
| public static Configuration loadConfiguration(FileSystem fs, |
| Path path) throws IOException { |
| byte[] data = loadBytes(fs, path); |
| |
| ByteArrayInputStream in2; |
| |
| in2 = new ByteArrayInputStream(data); |
| Configuration conf1 = new Configuration(false); |
| conf1.addResource(in2); |
| //now clone it while dropping all its sources |
| Configuration conf2 = new Configuration(false); |
| String src = path.toString(); |
| for (Map.Entry<String, String> entry : conf1) { |
| String key = entry.getKey(); |
| String value = entry.getValue(); |
| conf2.set(key, value, src); |
| } |
| return conf2; |
| } |
| |
| |
| /** |
| * Generate a config file in a destination directory on the local filesystem |
| * @param confdir the directory path where the file is to go |
| * @param filename the filename |
| * @return the destination path |
| */ |
| public static File saveConfig(Configuration generatingConf, |
| File confdir, |
| String filename) throws IOException { |
| |
| |
| File destPath = new File(confdir, filename); |
| OutputStream fos = new FileOutputStream(destPath); |
| try { |
| generatingConf.writeXml(fos); |
| } finally { |
| IOUtils.closeStream(fos); |
| } |
| return destPath; |
| } |
| |
| /** |
| * Parse an XML Hadoop configuration into an XML document. x-include |
| * is supported, but as the location isn't passed in, relative |
| * URIs are out. |
| * @param in instream |
| * @return a document |
| * @throws ParserConfigurationException parser feature problems |
| * @throws IOException IO problems |
| * @throws SAXException XML is invalid |
| */ |
| public static Document parseConfigXML(InputStream in) throws |
| ParserConfigurationException, |
| IOException, |
| SAXException { |
| DocumentBuilderFactory docBuilderFactory |
| = DocumentBuilderFactory.newInstance(); |
| //ignore all comments inside the xml file |
| docBuilderFactory.setIgnoringComments(true); |
| |
| //allow includes in the xml file |
| docBuilderFactory.setNamespaceAware(true); |
| docBuilderFactory.setXIncludeAware(true); |
| DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); |
| return builder.parse(in); |
| } |
| |
| /** |
| * Load a Hadoop configuration from a local file. |
| * @param file file to load |
| * @return a configuration which hasn't actually had the load triggered |
| * yet. |
| * @throws FileNotFoundException file is not there |
| * @throws IOException any other IO problem |
| */ |
| public static Configuration loadConfFromFile(File file) throws |
| IOException { |
| return loadConfFromFile(file, false); |
| } |
| |
| /** |
| * |
| * Load a Hadoop configuration from a local file. |
| * @param file file to load |
| * @param loadDefaults flag to indicate if the defaults should be loaded yet |
| * @return a configuration which hasn't actually had the load triggered |
| * yet. |
| * @throws FileNotFoundException file is not there |
| * @throws IOException any other IO problem |
| */ |
| public static Configuration loadConfFromFile(File file, |
| boolean loadDefaults) throws IOException { |
| if (!file.exists()) { |
| throw new FileNotFoundException("File not found :" |
| + file.getAbsoluteFile()); |
| } |
| Configuration conf = new Configuration(loadDefaults); |
| try { |
| conf.addResource(file.toURI().toURL()); |
| } catch (MalformedURLException e) { |
| // should never happen... |
| throw new IOException( |
| "File " + file.toURI() + " doesn't have a valid URL"); |
| } |
| return conf; |
| } |
| |
| /** |
| * Add a configuration from a file to an existing configuration |
| * @param conf existing configuration |
| * @param file file to load |
| * @param overwrite flag to indicate new values should overwrite the predecessor |
| * @return the merged configuration |
| * @throws IOException |
| */ |
| public static Configuration addConfigurationFile(Configuration conf, |
| File file, boolean overwrite) |
| throws IOException { |
| Configuration c2 = loadConfFromFile(file, false); |
| mergeConfigurations(conf, c2, file.getAbsolutePath(), overwrite); |
| return conf; |
| } |
| |
| /** |
| * Add the system env variables with the given prefix (by convention, env.) |
| * @param conf existing configuration |
| * @param prefix prefix |
| */ |
| public static void addEnvironmentVariables(Configuration conf, String prefix) { |
| Map<String, String> env = System.getenv(); |
| for (Map.Entry<String, String> entry : env.entrySet()) { |
| conf.set(prefix + entry.getKey(),entry.getValue(), "env"); |
| } |
| } |
| |
| /** |
| * looks for the config under $confdir/$templateFilename; if not there |
| * loads it from /conf/templateFile. |
| * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the |
| * origin to help debug what's happening |
| * @param systemConf system conf |
| * @param confdir conf dir in FS |
| * @param templateFilename filename in the confdir |
| * @param fallbackResource resource to fall back on |
| * @return loaded conf |
| * @throws IOException IO problems |
| */ |
| public static Configuration loadTemplateConfiguration(Configuration systemConf, |
| Path confdir, |
| String templateFilename, |
| String fallbackResource) throws |
| IOException { |
| FileSystem fs = FileSystem.get(confdir.toUri(), systemConf); |
| |
| Path templatePath = new Path(confdir, templateFilename); |
| return loadTemplateConfiguration(fs, templatePath, fallbackResource); |
| } |
| |
| /** |
| * looks for the config under $confdir/$templateFilename; if not there |
| * loads it from /conf/templateFile. |
| * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the |
| * origin to help debug what's happening. |
| * @param fs Filesystem |
| * @param templatePath HDFS path for template |
| * @param fallbackResource resource to fall back on, or "" for no fallback |
| * @return loaded conf |
| * @throws IOException IO problems |
| * @throws FileNotFoundException if the path doesn't have a file and there |
| * was no fallback. |
| */ |
| public static Configuration loadTemplateConfiguration(FileSystem fs, |
| Path templatePath, |
| String fallbackResource) |
| throws IOException { |
| Configuration conf; |
| String origin; |
| if (fs.exists(templatePath)) { |
| log.debug("Loading template configuration {}", templatePath); |
| conf = loadConfiguration(fs, templatePath); |
| origin = templatePath.toString(); |
| } else { |
| if (fallbackResource.isEmpty()) { |
| throw new FileNotFoundException("No config file found at " + templatePath); |
| } |
| log.debug("Template {} not found" + |
| " -reverting to classpath resource {}", templatePath, fallbackResource); |
| conf = new Configuration(false); |
| conf.addResource(fallbackResource); |
| origin = "Resource " + fallbackResource; |
| } |
| //force a get |
| conf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN); |
| //now set the origin |
| conf.set(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN, origin); |
| return conf; |
| } |
| |
| |
| /** |
| * For testing: dump a configuration |
| * @param conf configuration |
| * @return listing in key=value style |
| */ |
| public static String dumpConfigToString(Configuration conf) { |
| Set<String> sorted = sortedConfigKeys(conf); |
| |
| StringBuilder builder = new StringBuilder(); |
| for (String key : sorted) { |
| |
| builder.append(key) |
| .append("=") |
| .append(conf.get(key)) |
| .append("\n"); |
| } |
| return builder.toString(); |
| } |
| |
| /** |
| * Merge in one configuration above another |
| * @param base base config |
| * @param merge one to merge. This MUST be a non-default-load config to avoid |
| * merge origin confusion |
| * @param origin description of the origin for the put operation |
| * @param overwrite flag to indicate new values should overwrite the predecessor |
| * @return the base with the merged values |
| */ |
| public static Configuration mergeConfigurations(Configuration base, |
| Iterable<Map.Entry<String, String>> merge, |
| String origin, |
| boolean overwrite) { |
| for (Map.Entry<String, String> entry : merge) { |
| String key = entry.getKey(); |
| if (overwrite || base.get(key) == null) { |
| base.set(key, entry.getValue(), origin); |
| } |
| } |
| return base; |
| } |
| |
| /** |
| * Register a resource as a default resource. |
| * Do not attempt to use this unless you understand that the |
| * order in which default resources are loaded affects the outcome, |
| * and that subclasses of Configuration often register new default |
| * resources |
| * @param resource the resource name |
| * @return the URL or null |
| */ |
| public static URL registerDefaultResource(String resource) { |
| URL resURL = ConfigHelper.class.getClassLoader() |
| .getResource(resource); |
| if (resURL != null) { |
| Configuration.addDefaultResource(resource); |
| } |
| return resURL; |
| } |
| |
| /** |
| * Load a configuration from a resource on this classpath. |
| * If the resource is not found, an empty configuration is returned |
| * @param resource the resource name |
| * @return the loaded configuration. |
| */ |
| public static Configuration loadFromResource(String resource) { |
| Configuration conf = new Configuration(false); |
| URL resURL = ConfigHelper.class.getClassLoader() |
| .getResource(resource); |
| if (resURL != null) { |
| log.debug("loaded resources from {}", resURL); |
| conf.addResource(resource); |
| } else{ |
| log.debug("failed to find {} on the classpath", resource); |
| } |
| return conf; |
| |
| } |
| |
| /** |
| * Load a resource that must be there |
| * @param resource the resource name |
| * @return the loaded configuration |
| * @throws FileNotFoundException if the resource is missing |
| */ |
| public static Configuration loadMandatoryResource(String resource) |
| throws FileNotFoundException { |
| Configuration conf = new Configuration(false); |
| URL resURL = ConfigHelper.class.getClassLoader() |
| .getResource(resource); |
| if (resURL != null) { |
| log.debug("loaded resources from {}", resURL); |
| conf.addResource(resource); |
| } else { |
| throw new FileNotFoundException(resource); |
| } |
| return conf; |
| } |
| |
| /** |
| * Propagate a property from a source to a dest config, with a best-effort |
| * attempt at propagating the origin. |
| * If the |
| * @param dest destination |
| * @param src source |
| * @param key key to try to copy |
| * @return true if the key was found and propagated |
| */ |
| public static boolean propagate(Configuration dest, |
| Configuration src, |
| String key) { |
| String val = src.get(key); |
| if (val != null) { |
| String[] origin = src.getPropertySources(key); |
| if (origin != null && origin.length > 0) { |
| dest.set(key, val, origin[0]); |
| } else { |
| dest.set(key, val); |
| return true; |
| } |
| } |
| return false; |
| } |
| |
| |
| /** |
| * Take a configuration, return a hash map |
| * @param conf conf |
| * @return hash map |
| */ |
| public static Map<String, String> buildMapFromConfiguration(Configuration conf) { |
| Map<String, String> map = new HashMap<String, String>(); |
| return SliderUtils.mergeEntries(map, conf); |
| } |
| |
| /** |
| * This goes through the keyset of one configuration and retrieves each value |
| * from a value source -a different or the same configuration. This triggers |
| * the property resolution process of the value, resolving any variables against |
| * in-config or inherited configurations |
| * @param keysource source of keys |
| * @param valuesource the source of values |
| * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code> |
| */ |
| public static Configuration resolveConfiguration( |
| Iterable<Map.Entry<String, String>> keysource, |
| Configuration valuesource) { |
| Configuration result = new Configuration(false); |
| for (Map.Entry<String, String> entry : keysource) { |
| String key = entry.getKey(); |
| String value = valuesource.get(key); |
| Preconditions.checkState(value != null, |
| "no reference for \"%s\" in values", key); |
| result.set(key, value); |
| } |
| return result; |
| } |
| |
| /** |
| * Register anything we consider deprecated |
| */ |
| public static void registerDeprecatedConfigItems() { |
| Configuration.addDeprecation( |
| SliderXmlConfKeys.REGISTRY_ZK_QUORUM, |
| RegistryConstants.KEY_REGISTRY_ZK_QUORUM); |
| Configuration.addDeprecation( |
| SliderXmlConfKeys.REGISTRY_PATH, |
| RegistryConstants.KEY_REGISTRY_ZK_ROOT); |
| |
| } |
| } |