/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.slider.common.tools;

import com.google.common.base.Preconditions;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.core.exceptions.BadConfigException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;

/**
 * Methods to aid in config, both in the Configuration class and
 * with other parts of setting up Slider-initated processes.
 * 
 * Some of the methods take an argument of a map iterable for their sources; this allows
 * the same method
 */
public class ConfigHelper {
  private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class);

  /**
   * Dump the (sorted) configuration
   * @param conf config
   * @return the sorted keyset
   */
  public static Set<String> dumpConf(Configuration conf) {
    Set<String> keys = sortedConfigKeys(conf);
    for (String key : keys) {
      log.info("{}={}", key, conf.get(key));
    }
    return keys;
  }

  /**
   * Take a configuration and return a sorted set
   * @param conf config
   * @return the sorted keyset

   */
  public static Set<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) {
    TreeSet<String> sorted = new TreeSet<String>();
    for (Map.Entry<String, String> entry : conf) {
      sorted.add(entry.getKey());
    }
    return sorted;
  }

  /**
   * Set an entire map full of values
   *
   * @param config config to patch
   * @param map map of data
   * @param origin origin data
   */
  public static void addConfigMap(Configuration config,
                                  Map<String, String> map,
                                  String origin) throws BadConfigException {
    addConfigMap(config, map.entrySet(), origin);
  }
  
  /**
   * Set an entire map full of values
   *
   * @param config config to patch
   * @param map map of data
   * @param origin origin data
   */
  public static void addConfigMap(Configuration config,
                                  Iterable<Map.Entry<String, String>> map,
                                  String origin) throws BadConfigException {
    for (Map.Entry<String, String> mapEntry : map) {
      String key = mapEntry.getKey();
      String value = mapEntry.getValue();
      if (value == null) {
        throw new BadConfigException("Null value for property " + key);
      }
      config.set(key, value, origin);
    }
  }


  /**
   * Save a config file in a destination directory on a given filesystem
   * @param systemConf system conf used for creating filesystems
   * @param confToSave config to save
   * @param confdir the directory path where the file is to go
   * @param filename the filename
   * @return the destination path where the file was saved
   * @throws IOException IO problems
   */
  public static Path saveConfig(Configuration systemConf,
                                Configuration confToSave,
                                Path confdir,
                                String filename) throws IOException {
    FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);
    Path destPath = new Path(confdir, filename);
    saveConfig(fs, destPath, confToSave);
    return destPath;
  }

  /**
   * Save a config
   * @param fs filesystem
   * @param destPath dest to save
   * @param confToSave  config to save
   * @throws IOException IO problems
   */
  public static void saveConfig(FileSystem fs,
                                Path destPath,
                                Configuration confToSave) throws
                                                              IOException {
    FSDataOutputStream fos = fs.create(destPath);
    try {
      confToSave.writeXml(fos);
    } finally {
      IOUtils.closeStream(fos);
    }
  }

  /**
   * Convert to an XML string
   * @param conf configuration
   * @return conf
   * @throws IOException
   */
  public static String toXml(Configuration conf) throws IOException {
    StringWriter writer = new StringWriter();
    conf.writeXml(writer);
    return writer.toString();
  }
  
  /**
   * This will load and parse a configuration to an XML document
   * @param fs filesystem
   * @param path path
   * @return an XML document
   * @throws IOException IO failure
   */
  public Document parseConfiguration(FileSystem fs,
                                     Path path) throws
                                                IOException {


    byte[] data = loadBytes(fs, path);
    //this is here to track down a parse issue
    //related to configurations
    String s = new String(data, 0, data.length);
    log.debug("XML resource {} is \"{}\"", path, s);
/* JDK7
    try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
      Document document = parseConfigXML(in);
      return document;
    } catch (ParserConfigurationException | SAXException e) {
      throw new IOException(e);
    }
*/
    ByteArrayInputStream in= null;
    try {
      in = new ByteArrayInputStream(data);
      Document document = parseConfigXML(in);
      return document;
    } catch (ParserConfigurationException e) {
      throw new IOException(e);
    } catch (SAXException e) {
      throw new IOException(e);
    } finally {
      IOUtils.closeStream(in);
    }
  }

  public static byte[] loadBytes(FileSystem fs, Path path) throws IOException {
    int len = (int) fs.getLength(path);
    byte[] data = new byte[len];
    /* JDK7
    try(FSDataInputStream in = fs.open(path)) {
      in.readFully(0, data);
    }
*/
    FSDataInputStream in = null;
    in = fs.open(path);
    try {
      in.readFully(0, data);
    } finally {
      IOUtils.closeStream(in);
    }
    return data;
  }

  /**
   * Load a configuration from ANY FS path. The normal Configuration
   * loader only works with file:// URIs
   * @param fs filesystem
   * @param path path
   * @return a loaded resource
   * @throws IOException
   */
  public static Configuration loadConfiguration(FileSystem fs,
                                                Path path) throws IOException {
    byte[] data = loadBytes(fs, path);

    ByteArrayInputStream in2;

    in2 = new ByteArrayInputStream(data);
    Configuration conf1 = new Configuration(false);
    conf1.addResource(in2);
    //now clone it while dropping all its sources
    Configuration conf2   = new Configuration(false);
    String src = path.toString();
    for (Map.Entry<String, String> entry : conf1) {
      String key = entry.getKey();
      String value = entry.getValue();
      conf2.set(key, value, src);
    }
    return conf2;
  }


  /**
   * Generate a config file in a destination directory on the local filesystem
   * @param confdir the directory path where the file is to go
   * @param filename the filename
   * @return the destination path
   */
  public static File saveConfig(Configuration generatingConf,
                                    File confdir,
                                    String filename) throws IOException {


    File destPath = new File(confdir, filename);
    OutputStream fos = new FileOutputStream(destPath);
    try {
      generatingConf.writeXml(fos);
    } finally {
      IOUtils.closeStream(fos);
    }
    return destPath;
  }

  /**
   * Parse an XML Hadoop configuration into an XML document. x-include
   * is supported, but as the location isn't passed in, relative
   * URIs are out.
   * @param in instream
   * @return a document
   * @throws ParserConfigurationException parser feature problems
   * @throws IOException IO problems
   * @throws SAXException XML is invalid
   */
  public static Document parseConfigXML(InputStream in) throws
                                               ParserConfigurationException,
                                               IOException,
                                               SAXException {
    DocumentBuilderFactory docBuilderFactory
      = DocumentBuilderFactory.newInstance();
    //ignore all comments inside the xml file
    docBuilderFactory.setIgnoringComments(true);

    //allow includes in the xml file
    docBuilderFactory.setNamespaceAware(true);
    docBuilderFactory.setXIncludeAware(true);
    DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
    return builder.parse(in);
  }

  /**
   * Load a Hadoop configuration from a local file.
   * @param file file to load
   * @return a configuration which hasn't actually had the load triggered
   * yet.
   * @throws FileNotFoundException file is not there
   * @throws IOException any other IO problem
   */
  public static Configuration loadConfFromFile(File file) throws
                                                          IOException {
    return loadConfFromFile(file, false);
  }

  /**
   *
   * Load a Hadoop configuration from a local file.
   * @param file file to load
   * @param loadDefaults flag to indicate if the defaults should be loaded yet
   * @return a configuration which hasn't actually had the load triggered
   * yet.
   * @throws FileNotFoundException file is not there
   * @throws IOException any other IO problem
   */
  public static Configuration loadConfFromFile(File file,
      boolean loadDefaults) throws IOException {
    if (!file.exists()) {
      throw new FileNotFoundException("File not found :"
                                          + file.getAbsoluteFile());
    }
    Configuration conf = new Configuration(loadDefaults);
    try {
      conf.addResource(file.toURI().toURL());
    } catch (MalformedURLException e) {
      // should never happen...
      throw new IOException(
        "File " + file.toURI() + " doesn't have a valid URL");
    }
    return conf;
  }

  /**
   * Add a configuration from a file to an existing configuration
   * @param conf existing configuration
   * @param file file to load
   * @param overwrite flag to indicate new values should overwrite the predecessor
   * @return the merged configuration
   * @throws IOException
   */
  public static Configuration addConfigurationFile(Configuration conf,
      File file, boolean overwrite)
      throws IOException {
    Configuration c2 = loadConfFromFile(file, false);
    mergeConfigurations(conf, c2, file.getAbsolutePath(), overwrite);
    return conf;
  }

  /**
   * Add the system env variables with the given prefix (by convention, env.)
   * @param conf existing configuration
   * @param prefix prefix
   */
  public static void addEnvironmentVariables(Configuration conf, String prefix) {
    Map<String, String> env = System.getenv();
    for (Map.Entry<String, String> entry : env.entrySet()) {
      conf.set(prefix + entry.getKey(),entry.getValue(), "env");
    }
  }
  
  /**
   * looks for the config under $confdir/$templateFilename; if not there
   * loads it from /conf/templateFile.
   * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
   * origin to help debug what's happening
   * @param systemConf system conf
   * @param confdir conf dir in FS
   * @param templateFilename filename in the confdir
   * @param fallbackResource resource to fall back on
   * @return loaded conf
   * @throws IOException IO problems
   */
  public static Configuration loadTemplateConfiguration(Configuration systemConf,
                                                        Path confdir,
                                                        String templateFilename,
                                                        String fallbackResource) throws
                                                                         IOException {
    FileSystem fs = FileSystem.get(confdir.toUri(), systemConf);

    Path templatePath = new Path(confdir, templateFilename);
    return loadTemplateConfiguration(fs, templatePath, fallbackResource);
  }

  /**
   * looks for the config under $confdir/$templateFilename; if not there
   * loads it from /conf/templateFile.
   * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the
   * origin to help debug what's happening.
   * @param fs Filesystem
   * @param templatePath HDFS path for template
   * @param fallbackResource resource to fall back on, or "" for no fallback
   * @return loaded conf
   * @throws IOException IO problems
   * @throws FileNotFoundException if the path doesn't have a file and there
   * was no fallback.
   */
  public static Configuration loadTemplateConfiguration(FileSystem fs,
                                                        Path templatePath,
                                                        String fallbackResource)
      throws IOException {
    Configuration conf;
    String origin;
    if (fs.exists(templatePath)) {
      log.debug("Loading template configuration {}", templatePath);
      conf = loadConfiguration(fs, templatePath);
      origin = templatePath.toString();
    } else {
      if (fallbackResource.isEmpty()) {
        throw new FileNotFoundException("No config file found at " + templatePath);
      }
      log.debug("Template {} not found" +
                " -reverting to classpath resource {}", templatePath, fallbackResource);
      conf = new Configuration(false);
      conf.addResource(fallbackResource);
      origin = "Resource " + fallbackResource;
    }
    //force a get
    conf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN);
    //now set the origin
    conf.set(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN, origin);
    return conf;
  }


  /**
   * For testing: dump a configuration
   * @param conf configuration
   * @return listing in key=value style
   */
  public static String dumpConfigToString(Configuration conf) {
    Set<String> sorted = sortedConfigKeys(conf);

    StringBuilder builder = new StringBuilder();
    for (String key : sorted) {

      builder.append(key)
             .append("=")
             .append(conf.get(key))
             .append("\n");
    }
    return builder.toString();
  }

  /**
   * Merge in one configuration above another
   * @param base base config
   * @param merge one to merge. This MUST be a non-default-load config to avoid
   * merge origin confusion
   * @param origin description of the origin for the put operation
   * @param overwrite flag to indicate new values should overwrite the predecessor
   * @return the base with the merged values
   */
  public static Configuration mergeConfigurations(Configuration base,
      Iterable<Map.Entry<String, String>> merge,
      String origin,
      boolean overwrite) {
    for (Map.Entry<String, String> entry : merge) {
      String key = entry.getKey();
      if (overwrite || base.get(key) == null) {
        base.set(key, entry.getValue(), origin);
      }
    }
    return base;
  }

  /**
   * Register a resource as a default resource.
   * Do not attempt to use this unless you understand that the
   * order in which default resources are loaded affects the outcome,
   * and that subclasses of Configuration often register new default
   * resources
   * @param resource the resource name
   * @return the URL or null
   */
  public static URL registerDefaultResource(String resource) {
    URL resURL = ConfigHelper.class.getClassLoader()
                                .getResource(resource);
    if (resURL != null) {
      Configuration.addDefaultResource(resource);
    }
    return resURL;
  }

  /**
   * Load a configuration from a resource on this classpath.
   * If the resource is not found, an empty configuration is returned
   * @param resource the resource name
   * @return the loaded configuration.
   */
  public static Configuration loadFromResource(String resource) {
    Configuration conf = new Configuration(false);
    URL resURL = ConfigHelper.class.getClassLoader()
                                .getResource(resource);
    if (resURL != null) {
      log.debug("loaded resources from {}", resURL);
      conf.addResource(resource);
    } else{
      log.debug("failed to find {} on the classpath", resource);
    }
    return conf;
    
  }

  /**
   * Load a resource that must be there
   * @param resource the resource name
   * @return the loaded configuration
   * @throws FileNotFoundException if the resource is missing
   */
  public static Configuration loadMandatoryResource(String resource)
      throws FileNotFoundException {
    Configuration conf = new Configuration(false);
    URL resURL = ConfigHelper.class.getClassLoader()
                                .getResource(resource);
    if (resURL != null) {
      log.debug("loaded resources from {}", resURL);
      conf.addResource(resource);
    } else {
      throw new FileNotFoundException(resource);
    }
    return conf;
  }

  /**
   * Propagate a property from a source to a dest config, with a best-effort
   * attempt at propagating the origin.
   * If the 
   * @param dest destination
   * @param src source
   * @param key key to try to copy
   * @return true if the key was found and propagated
   */
  public static boolean propagate(Configuration dest,
                                  Configuration src,
                                  String key) {
    String val = src.get(key);
    if (val != null) {
      String[] origin = src.getPropertySources(key);
      if (origin != null && origin.length > 0) {
        dest.set(key, val, origin[0]);
      } else {
        dest.set(key, val);
        return true;
      }
    }
    return false;
  }


  /**
   * Take a configuration, return a hash map
   * @param conf conf
   * @return hash map
   */
  public static Map<String, String> buildMapFromConfiguration(Configuration conf) {
    Map<String, String> map = new HashMap<String, String>();
    return SliderUtils.mergeEntries(map, conf);
  }

  /**
   * This goes through the keyset of one configuration and retrieves each value
   * from a value source -a different or the same configuration. This triggers
   * the property resolution process of the value, resolving any variables against
   * in-config or inherited configurations
   * @param keysource source of keys
   * @param valuesource the source of values
   * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code>
   */
  public static Configuration resolveConfiguration(
      Iterable<Map.Entry<String, String>> keysource,
      Configuration valuesource) {
    Configuration result = new Configuration(false);
    for (Map.Entry<String, String> entry : keysource) {
      String key = entry.getKey();
      String value = valuesource.get(key);
      Preconditions.checkState(value != null,
          "no reference for \"%s\" in values", key);
      result.set(key, value);
    }
    return result;
  }

  /**
   * Register anything we consider deprecated
   */
  public static void registerDeprecatedConfigItems() {
    Configuration.addDeprecation(
        SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
        RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
    Configuration.addDeprecation(
        SliderXmlConfKeys.REGISTRY_PATH,
        RegistryConstants.KEY_REGISTRY_ZK_ROOT);
    
  }
}