src/org/apache/pig/tools/DownloadResolver.java - pig - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 package org.apache.pig.tools;

 import groovy.grape.Grape;
 import groovy.lang.GroovyClassLoader;

 import java.io.File;
 import java.net.URI;
 import java.util.LinkedList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;

 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.pig.PigConfiguration;
 import org.apache.pig.PigServer;
 import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
 import org.apache.pig.parser.ParserException;

 public class DownloadResolver {

     private static final String IVY_FILE_NAME = "ivysettings.xml";
     private static final Log LOG = LogFactory.getLog(DownloadResolver.class);
     private static DownloadResolver downloadResolver = new DownloadResolver();

     private DownloadResolver() {
         if (System.getProperty("grape.config") != null) {
             LOG.info("Using ivysettings file from " + System.getProperty("grape.config"));
         } else {
             // Retrieve the ivysettings configuration file
             Map<String, String> envMap = System.getenv();
             File confFile = null;
             // Check for configuration file in PIG_CONF_DIR
             if (envMap.containsKey("PIG_CONF_DIR")) {
                 confFile = new File(new File(envMap.get("PIG_CONF_DIR")).getPath(), IVY_FILE_NAME);
             }

             // Check for configuration file in PIG_HOME if not found in PIG_CONF_DIR
             if (confFile == null || !confFile.exists()) {
                 confFile = new File(new File(envMap.get("PIG_HOME"), "conf").getPath(), IVY_FILE_NAME);
             }

             // Check for configuration file in Classloader if not found in PIG_CONF_DIR and PIG_HOME
             if (confFile == null || !confFile.exists()) {
                 ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
                 if (classLoader.getResource(IVY_FILE_NAME) != null) {
                     LOG.info("Found ivysettings file in classpath");
                     confFile = new File(classLoader.getResource(IVY_FILE_NAME).getFile());

                     if (!confFile.exists()) {
                         // ivysettings file resides inside a jar
                         try {
                             List<String> ivyLines = IOUtils.readLines(classLoader.getResourceAsStream(IVY_FILE_NAME));
                             confFile = File.createTempFile("ivysettings", ".xml");
                             confFile.deleteOnExit();
                             for(String str: ivyLines) {
                                 FileUtils.writeStringToFile(confFile, str, true);
                             }
                         } catch (Exception e) {
                             LOG.warn("Could not create an ivysettings file from resource");
                         }
                     }
                 }
             }

             // Set the Configuration file
             if (confFile != null && confFile.exists()) {
                 LOG.info("Using ivysettings file from " + confFile.toString());
                 System.setProperty("grape.config", confFile.toString());
             } else {
                 LOG.warn("Could not find custom ivysettings file in PIG_CONF_DIR or PIG_HOME or classpath.");
             }
         }
     }

     /**
      * @return Singleton Object
      */
     public static DownloadResolver getInstance() {
         return downloadResolver;
     }

     /**
      * @param uri
      * @return A map of all Query String Parameters
      */
     private Map<String, String> parseQueryString(URI uri) {
         Map<String, String> paramMap = new HashMap<String, String>();
         String queryString = uri.getQuery();
         if (queryString != null) {
             String queryParams[] = queryString.split("&");
             for (String param : queryParams) {
                 String[] parts = param.split("=");
                 if (parts.length == 2) {
                     String name = parts[0].toLowerCase();
                     String value = parts[1];
                     paramMap.put(name, value);
                 }
             }
         }
         return paramMap;
     }

     /**
      * @param uri
      * @return Returns a Map containing the organization, module, version and
      *         all the query string parameters
      * @throws ParserException
      */
     private Map<String, Object> parseUri(URI uri) throws ParserException {
         // Parse uri for artifact organization, module and version
         Map<String, Object> uriMap = new HashMap<String, Object>();
         String authority = uri.getAuthority();
         if (authority != null) {
             String[] tokens = authority.split(":", -1);
             if (tokens.length == 3) {
                 uriMap.put("org", tokens[0]);
                 if (tokens[1].isEmpty()) {
                     throw new ParserException("Please specify the artifact module.");
                 }
                 uriMap.put("module", tokens[1]);
                 uriMap.put("version", tokens[2]);
             } else {
                 throw new ParserException("Invalid Artifact. Please specify organization, module and version");
             }
         } else {
             throw new ParserException("Invalid Artifact. Please specify organization, module and version");
         }

         // Parse query string for exclude list and other parameters
         uriMap.putAll(parseQueryString(uri));
         if (uriMap.containsKey("transitive")) {
             uriMap.put("transitive", Boolean.parseBoolean(uriMap.get("transitive").toString()));
         }
         List<Map<String, Object>> excludeList = new LinkedList<Map<String, Object>>();
         if (uriMap.containsKey("exclude")) {
             for (String exclude : uriMap.get("exclude").toString().split(",")) {
                 Map<String, Object> excludeMap = new HashMap<String, Object>();
                 String parts[] = exclude.split(":", -1);
                 if (parts.length == 2) {
                     excludeMap.put("group", parts[0]);
                     excludeMap.put("module", parts[1]);
                 } else {
                     throw new ParserException("Exclude must contain organization and module separated by a colon.");
                 }
                 excludeList.add(excludeMap);
             }
         }
         uriMap.put("excludes", excludeList);

         return uriMap;
     }

     /**
      * @param uri
      * @return List of URIs of the downloaded jars
      * @throws ParserException
      */
     public URI[] downloadArtifact(URI uri, PigServer pigServer) throws ParserException {
         Configuration conf = ConfigurationUtil.toConfiguration(pigServer.getPigContext().getProperties());

         String artDownloadLocation = conf.get(PigConfiguration.PIG_ARTIFACTS_DOWNLOAD_LOCATION);

         if (artDownloadLocation != null) {
             LOG.info("Artifacts will be downloaded to " + artDownloadLocation);
             System.setProperty("grape.root", artDownloadLocation);
         } else if (System.getProperty("grape.root") != null) {
             LOG.info("Artifacts will be downloaded to " + System.getProperty("grape.root"));
         } else {
             LOG.info("Artifacts will be downloaded to default location. Please check ~/.groovy/grapes");
         }

         Map<String, Object> uriMap = parseUri(uri);
         Map<String, Object> args = new HashMap<String, Object>();
         args.put("classLoader", new GroovyClassLoader());
         args.put("excludes", uriMap.get("excludes"));
         return Grape.resolve(args, uriMap);
     }

 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/
	package org.apache.pig.tools;

	import groovy.grape.Grape;
	import groovy.lang.GroovyClassLoader;

	import java.io.File;
	import java.net.URI;
	import java.util.LinkedList;
	import java.util.HashMap;
	import java.util.List;
	import java.util.Map;

	import org.apache.commons.io.FileUtils;
	import org.apache.commons.io.IOUtils;
	import org.apache.commons.logging.Log;
	import org.apache.commons.logging.LogFactory;
	import org.apache.hadoop.conf.Configuration;
	import org.apache.pig.PigConfiguration;
	import org.apache.pig.PigServer;
	import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
	import org.apache.pig.parser.ParserException;

	public class DownloadResolver {

	private static final String IVY_FILE_NAME = "ivysettings.xml";
	private static final Log LOG = LogFactory.getLog(DownloadResolver.class);
	private static DownloadResolver downloadResolver = new DownloadResolver();

	private DownloadResolver() {
	if (System.getProperty("grape.config") != null) {
	LOG.info("Using ivysettings file from " + System.getProperty("grape.config"));
	} else {
	// Retrieve the ivysettings configuration file
	Map<String, String> envMap = System.getenv();
	File confFile = null;
	// Check for configuration file in PIG_CONF_DIR
	if (envMap.containsKey("PIG_CONF_DIR")) {
	confFile = new File(new File(envMap.get("PIG_CONF_DIR")).getPath(), IVY_FILE_NAME);
	}

	// Check for configuration file in PIG_HOME if not found in PIG_CONF_DIR
	if (confFile == null \|\| !confFile.exists()) {
	confFile = new File(new File(envMap.get("PIG_HOME"), "conf").getPath(), IVY_FILE_NAME);
	}

	// Check for configuration file in Classloader if not found in PIG_CONF_DIR and PIG_HOME
	if (confFile == null \|\| !confFile.exists()) {
	ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
	if (classLoader.getResource(IVY_FILE_NAME) != null) {
	LOG.info("Found ivysettings file in classpath");
	confFile = new File(classLoader.getResource(IVY_FILE_NAME).getFile());

	if (!confFile.exists()) {
	// ivysettings file resides inside a jar
	try {
	List<String> ivyLines = IOUtils.readLines(classLoader.getResourceAsStream(IVY_FILE_NAME));
	confFile = File.createTempFile("ivysettings", ".xml");
	confFile.deleteOnExit();
	for(String str: ivyLines) {
	FileUtils.writeStringToFile(confFile, str, true);
	}
	} catch (Exception e) {
	LOG.warn("Could not create an ivysettings file from resource");
	}
	}
	}
	}

	// Set the Configuration file
	if (confFile != null && confFile.exists()) {
	LOG.info("Using ivysettings file from " + confFile.toString());
	System.setProperty("grape.config", confFile.toString());
	} else {
	LOG.warn("Could not find custom ivysettings file in PIG_CONF_DIR or PIG_HOME or classpath.");
	}
	}
	}

	/**
	* @return Singleton Object
	*/
	public static DownloadResolver getInstance() {
	return downloadResolver;
	}

	/**
	* @param uri
	* @return A map of all Query String Parameters
	*/
	private Map<String, String> parseQueryString(URI uri) {
	Map<String, String> paramMap = new HashMap<String, String>();
	String queryString = uri.getQuery();
	if (queryString != null) {
	String queryParams[] = queryString.split("&");
	for (String param : queryParams) {
	String[] parts = param.split("=");
	if (parts.length == 2) {
	String name = parts[0].toLowerCase();
	String value = parts[1];
	paramMap.put(name, value);
	}
	}
	}
	return paramMap;
	}

	/**
	* @param uri
	* @return Returns a Map containing the organization, module, version and
	* all the query string parameters
	* @throws ParserException
	*/
	private Map<String, Object> parseUri(URI uri) throws ParserException {
	// Parse uri for artifact organization, module and version
	Map<String, Object> uriMap = new HashMap<String, Object>();
	String authority = uri.getAuthority();
	if (authority != null) {
	String[] tokens = authority.split(":", -1);
	if (tokens.length == 3) {
	uriMap.put("org", tokens[0]);
	if (tokens[1].isEmpty()) {
	throw new ParserException("Please specify the artifact module.");
	}
	uriMap.put("module", tokens[1]);
	uriMap.put("version", tokens[2]);
	} else {
	throw new ParserException("Invalid Artifact. Please specify organization, module and version");
	}
	} else {
	throw new ParserException("Invalid Artifact. Please specify organization, module and version");
	}

	// Parse query string for exclude list and other parameters
	uriMap.putAll(parseQueryString(uri));
	if (uriMap.containsKey("transitive")) {
	uriMap.put("transitive", Boolean.parseBoolean(uriMap.get("transitive").toString()));
	}
	List<Map<String, Object>> excludeList = new LinkedList<Map<String, Object>>();
	if (uriMap.containsKey("exclude")) {
	for (String exclude : uriMap.get("exclude").toString().split(",")) {
	Map<String, Object> excludeMap = new HashMap<String, Object>();
	String parts[] = exclude.split(":", -1);
	if (parts.length == 2) {
	excludeMap.put("group", parts[0]);
	excludeMap.put("module", parts[1]);
	} else {
	throw new ParserException("Exclude must contain organization and module separated by a colon.");
	}
	excludeList.add(excludeMap);
	}
	}
	uriMap.put("excludes", excludeList);

	return uriMap;
	}

	/**
	* @param uri
	* @return List of URIs of the downloaded jars
	* @throws ParserException
	*/
	public URI[] downloadArtifact(URI uri, PigServer pigServer) throws ParserException {
	Configuration conf = ConfigurationUtil.toConfiguration(pigServer.getPigContext().getProperties());

	String artDownloadLocation = conf.get(PigConfiguration.PIG_ARTIFACTS_DOWNLOAD_LOCATION);

	if (artDownloadLocation != null) {
	LOG.info("Artifacts will be downloaded to " + artDownloadLocation);
	System.setProperty("grape.root", artDownloadLocation);
	} else if (System.getProperty("grape.root") != null) {
	LOG.info("Artifacts will be downloaded to " + System.getProperty("grape.root"));
	} else {
	LOG.info("Artifacts will be downloaded to default location. Please check ~/.groovy/grapes");
	}

	Map<String, Object> uriMap = parseUri(uri);
	Map<String, Object> args = new HashMap<String, Object>();
	args.put("classLoader", new GroovyClassLoader());
	args.put("excludes", uriMap.get("excludes"));
	return Grape.resolve(args, uriMap);
	}

	}