hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java - hadoop - Git at Google

 /**
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
  * regarding copyright ownership.  The ASF licenses this file
  * to you under the Apache License, Version 2.0 (the
  * "License"); you may not use this file except in compliance
  * with the License.  You may obtain a copy of the License at
  *
  *     http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 package org.apache.hadoop.fs.aliyun.oss;

 import java.io.File;
 import java.io.IOException;
 import java.net.URI;

 import com.aliyun.oss.common.auth.CredentialsProvider;
 import com.google.common.base.Preconditions;
 import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.security.ProviderUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

 import static org.apache.hadoop.fs.aliyun.oss.Constants.*;

 /**
  * Utility methods for Aliyun OSS code.
  */
 final public class AliyunOSSUtils {
   private static final Logger LOG =
       LoggerFactory.getLogger(AliyunOSSUtils.class);
   private static LocalDirAllocator directoryAllocator;

   private AliyunOSSUtils() {
   }

   public static int intPositiveOption(
       Configuration conf, String key, int defVal) {
     int v = conf.getInt(key, defVal);
     if (v <= 0) {
       LOG.warn(key + " is configured to " + v
           + ", will use default value: " + defVal);
       v = defVal;
     }

     return v;
   }

   /**
    * Used to get password from configuration.
    *
    * @param conf configuration that contains password information
    * @param key the key of the password
    * @return the value for the key
    * @throws IOException if failed to get password from configuration
    */
   public static String getValueWithKey(Configuration conf, String key)
       throws IOException {
     try {
       final char[] pass = conf.getPassword(key);
       if (pass != null) {
         return (new String(pass)).trim();
       } else {
         return "";
       }
     } catch (IOException ioe) {
       throw new IOException("Cannot find password option " + key, ioe);
     }
   }

   /**
    * Calculate a proper size of multipart piece. If <code>minPartSize</code>
    * is too small, the number of multipart pieces may exceed the limit of
    * {@link Constants#MULTIPART_UPLOAD_PART_NUM_LIMIT}.
    *
    * @param contentLength the size of file.
    * @param minPartSize the minimum size of multipart piece.
    * @return a revisional size of multipart piece.
    */
   public static long calculatePartSize(long contentLength, long minPartSize) {
     long tmpPartSize = contentLength / MULTIPART_UPLOAD_PART_NUM_LIMIT + 1;
     return Math.max(minPartSize, tmpPartSize);
   }

   /**
    * Create credential provider specified by configuration, or create default
    * credential provider if not specified.
    *
    * @param uri uri passed by caller
    * @param conf configuration
    * @return a credential provider
    * @throws IOException on any problem. Class construction issues may be
    * nested inside the IOE.
    */
   public static CredentialsProvider getCredentialsProvider(
       URI uri, Configuration conf) throws IOException {
     CredentialsProvider credentials;

     String className = conf.getTrimmed(CREDENTIALS_PROVIDER_KEY);
     if (StringUtils.isEmpty(className)) {
       Configuration newConf =
           ProviderUtils.excludeIncompatibleCredentialProviders(conf,
               AliyunOSSFileSystem.class);
       credentials = new AliyunCredentialsProvider(newConf);
     } else {
       try {
         LOG.debug("Credential provider class is:" + className);
         Class<?> credClass = Class.forName(className);
         try {
           credentials =
               (CredentialsProvider)credClass.getDeclaredConstructor(
                   URI.class, Configuration.class).newInstance(uri, conf);
         } catch (NoSuchMethodException | SecurityException e) {
           credentials =
               (CredentialsProvider)credClass.getDeclaredConstructor()
               .newInstance();
         }
       } catch (ClassNotFoundException e) {
         throw new IOException(className + " not found.", e);
       } catch (NoSuchMethodException | SecurityException e) {
         throw new IOException(String.format("%s constructor exception.  A " +
             "class specified in %s must provide an accessible constructor " +
             "accepting URI and Configuration, or an accessible default " +
             "constructor.", className, CREDENTIALS_PROVIDER_KEY),
             e);
       } catch (ReflectiveOperationException | IllegalArgumentException e) {
         throw new IOException(className + " instantiation exception.", e);
       }
     }

     return credentials;
   }

   /**
    * Turns a path (relative or otherwise) into an OSS key, adding a trailing
    * "/" if the path is not the root <i>and</i> does not already have a "/"
    * at the end.
    *
    * @param key OSS key or ""
    * @return the with a trailing "/", or, if it is the root key, "".
    */
   public static String maybeAddTrailingSlash(String key) {
     if (StringUtils.isNotEmpty(key) && !key.endsWith("/")) {
       return key + '/';
     } else {
       return key;
     }
   }

   /**
    * Check if OSS object represents a directory.
    *
    * @param name object key
    * @param size object content length
    * @return true if object represents a directory
    */
   public static boolean objectRepresentsDirectory(final String name,
       final long size) {
     return StringUtils.isNotEmpty(name) && name.endsWith("/") && size == 0L;
   }

   /**
    * Demand create the directory allocator, then create a temporary file.
    *  @param path prefix for the temporary file
    *  @param size the size of the file that is going to be written
    *  @param conf the Configuration object
    *  @return a unique temporary file
    *  @throws IOException IO problems
    */
   public static File createTmpFileForWrite(String path, long size,
       Configuration conf) throws IOException {
     if (conf.get(BUFFER_DIR_KEY) == null) {
       conf.set(BUFFER_DIR_KEY, conf.get("hadoop.tmp.dir") + "/oss");
     }
     if (directoryAllocator == null) {
       directoryAllocator = new LocalDirAllocator(BUFFER_DIR_KEY);
     }
     return directoryAllocator.createTmpFileForWrite(path, size, conf);
   }

   /**
    * Get a integer option >= the minimum allowed value.
    * @param conf configuration
    * @param key key to look up
    * @param defVal default value
    * @param min minimum value
    * @return the value
    * @throws IllegalArgumentException if the value is below the minimum
    */
   static int intOption(Configuration conf, String key, int defVal, int min) {
     int v = conf.getInt(key, defVal);
     Preconditions.checkArgument(v >= min,
         String.format("Value of %s: %d is below the minimum value %d",
             key, v, min));
     LOG.debug("Value of {} is {}", key, v);
     return v;
   }

   /**
    * Get a long option >= the minimum allowed value.
    * @param conf configuration
    * @param key key to look up
    * @param defVal default value
    * @param min minimum value
    * @return the value
    * @throws IllegalArgumentException if the value is below the minimum
    */
   static long longOption(Configuration conf, String key, long defVal,
       long min) {
     long v = conf.getLong(key, defVal);
     Preconditions.checkArgument(v >= min,
         String.format("Value of %s: %d is below the minimum value %d",
             key, v, min));
     LOG.debug("Value of {} is {}", key, v);
     return v;
   }

   /**
    * Get a size property from the configuration: this property must
    * be at least equal to {@link Constants#MULTIPART_MIN_SIZE}.
    * If it is too small, it is rounded up to that minimum, and a warning
    * printed.
    * @param conf configuration
    * @param property property name
    * @param defVal default value
    * @return the value, guaranteed to be above the minimum size
    */
   public static long getMultipartSizeProperty(Configuration conf,
       String property, long defVal) {
     long partSize = conf.getLong(property, defVal);
     if (partSize < MULTIPART_MIN_SIZE) {
       LOG.warn("{} must be at least 100 KB; configured value is {}",
           property, partSize);
       partSize = MULTIPART_MIN_SIZE;
     } else if (partSize > Integer.MAX_VALUE) {
       LOG.warn("oss: {} capped to ~2.14GB(maximum allowed size with " +
           "current output mechanism)", MULTIPART_UPLOAD_PART_SIZE_KEY);
       partSize = Integer.MAX_VALUE;
     }
     return partSize;
   }
 }
	/**
	* Licensed to the Apache Software Foundation (ASF) under one
	* or more contributor license agreements. See the NOTICE file
	* distributed with this work for additional information
	* regarding copyright ownership. The ASF licenses this file
	* to you under the Apache License, Version 2.0 (the
	* "License"); you may not use this file except in compliance
	* with the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	package org.apache.hadoop.fs.aliyun.oss;

	import java.io.File;
	import java.io.IOException;
	import java.net.URI;

	import com.aliyun.oss.common.auth.CredentialsProvider;
	import com.google.common.base.Preconditions;
	import org.apache.commons.lang.StringUtils;
	import org.apache.hadoop.conf.Configuration;
	import org.apache.hadoop.fs.LocalDirAllocator;
	import org.apache.hadoop.security.ProviderUtils;
	import org.slf4j.Logger;
	import org.slf4j.LoggerFactory;

	import static org.apache.hadoop.fs.aliyun.oss.Constants.*;

	/**
	* Utility methods for Aliyun OSS code.
	*/
	final public class AliyunOSSUtils {
	private static final Logger LOG =
	LoggerFactory.getLogger(AliyunOSSUtils.class);
	private static LocalDirAllocator directoryAllocator;

	private AliyunOSSUtils() {
	}

	public static int intPositiveOption(
	Configuration conf, String key, int defVal) {
	int v = conf.getInt(key, defVal);
	if (v <= 0) {
	LOG.warn(key + " is configured to " + v
	+ ", will use default value: " + defVal);
	v = defVal;
	}

	return v;
	}

	/**
	* Used to get password from configuration.
	*
	* @param conf configuration that contains password information
	* @param key the key of the password
	* @return the value for the key
	* @throws IOException if failed to get password from configuration
	*/
	public static String getValueWithKey(Configuration conf, String key)
	throws IOException {
	try {
	final char[] pass = conf.getPassword(key);
	if (pass != null) {
	return (new String(pass)).trim();
	} else {
	return "";
	}
	} catch (IOException ioe) {
	throw new IOException("Cannot find password option " + key, ioe);
	}
	}

	/**
	* Calculate a proper size of multipart piece. If <code>minPartSize</code>
	* is too small, the number of multipart pieces may exceed the limit of
	* {@link Constants#MULTIPART_UPLOAD_PART_NUM_LIMIT}.
	*
	* @param contentLength the size of file.
	* @param minPartSize the minimum size of multipart piece.
	* @return a revisional size of multipart piece.
	*/
	public static long calculatePartSize(long contentLength, long minPartSize) {
	long tmpPartSize = contentLength / MULTIPART_UPLOAD_PART_NUM_LIMIT + 1;
	return Math.max(minPartSize, tmpPartSize);
	}

	/**
	* Create credential provider specified by configuration, or create default
	* credential provider if not specified.
	*
	* @param uri uri passed by caller
	* @param conf configuration
	* @return a credential provider
	* @throws IOException on any problem. Class construction issues may be
	* nested inside the IOE.
	*/
	public static CredentialsProvider getCredentialsProvider(
	URI uri, Configuration conf) throws IOException {
	CredentialsProvider credentials;

	String className = conf.getTrimmed(CREDENTIALS_PROVIDER_KEY);
	if (StringUtils.isEmpty(className)) {
	Configuration newConf =
	ProviderUtils.excludeIncompatibleCredentialProviders(conf,
	AliyunOSSFileSystem.class);
	credentials = new AliyunCredentialsProvider(newConf);
	} else {
	try {
	LOG.debug("Credential provider class is:" + className);
	Class<?> credClass = Class.forName(className);
	try {
	credentials =
	(CredentialsProvider)credClass.getDeclaredConstructor(
	URI.class, Configuration.class).newInstance(uri, conf);
	} catch (NoSuchMethodException \| SecurityException e) {
	credentials =
	(CredentialsProvider)credClass.getDeclaredConstructor()
	.newInstance();
	}
	} catch (ClassNotFoundException e) {
	throw new IOException(className + " not found.", e);
	} catch (NoSuchMethodException \| SecurityException e) {
	throw new IOException(String.format("%s constructor exception. A " +
	"class specified in %s must provide an accessible constructor " +
	"accepting URI and Configuration, or an accessible default " +
	"constructor.", className, CREDENTIALS_PROVIDER_KEY),
	e);
	} catch (ReflectiveOperationException \| IllegalArgumentException e) {
	throw new IOException(className + " instantiation exception.", e);
	}
	}

	return credentials;
	}

	/**
	* Turns a path (relative or otherwise) into an OSS key, adding a trailing
	* "/" if the path is not the root <i>and</i> does not already have a "/"
	* at the end.
	*
	* @param key OSS key or ""
	* @return the with a trailing "/", or, if it is the root key, "".
	*/
	public static String maybeAddTrailingSlash(String key) {
	if (StringUtils.isNotEmpty(key) && !key.endsWith("/")) {
	return key + '/';
	} else {
	return key;
	}
	}

	/**
	* Check if OSS object represents a directory.
	*
	* @param name object key
	* @param size object content length
	* @return true if object represents a directory
	*/
	public static boolean objectRepresentsDirectory(final String name,
	final long size) {
	return StringUtils.isNotEmpty(name) && name.endsWith("/") && size == 0L;
	}

	/**
	* Demand create the directory allocator, then create a temporary file.
	* @param path prefix for the temporary file
	* @param size the size of the file that is going to be written
	* @param conf the Configuration object
	* @return a unique temporary file
	* @throws IOException IO problems
	*/
	public static File createTmpFileForWrite(String path, long size,
	Configuration conf) throws IOException {
	if (conf.get(BUFFER_DIR_KEY) == null) {
	conf.set(BUFFER_DIR_KEY, conf.get("hadoop.tmp.dir") + "/oss");
	}
	if (directoryAllocator == null) {
	directoryAllocator = new LocalDirAllocator(BUFFER_DIR_KEY);
	}
	return directoryAllocator.createTmpFileForWrite(path, size, conf);
	}

	/**
	* Get a integer option >= the minimum allowed value.
	* @param conf configuration
	* @param key key to look up
	* @param defVal default value
	* @param min minimum value
	* @return the value
	* @throws IllegalArgumentException if the value is below the minimum
	*/
	static int intOption(Configuration conf, String key, int defVal, int min) {
	int v = conf.getInt(key, defVal);
	Preconditions.checkArgument(v >= min,
	String.format("Value of %s: %d is below the minimum value %d",
	key, v, min));
	LOG.debug("Value of {} is {}", key, v);
	return v;
	}

	/**
	* Get a long option >= the minimum allowed value.
	* @param conf configuration
	* @param key key to look up
	* @param defVal default value
	* @param min minimum value
	* @return the value
	* @throws IllegalArgumentException if the value is below the minimum
	*/
	static long longOption(Configuration conf, String key, long defVal,
	long min) {
	long v = conf.getLong(key, defVal);
	Preconditions.checkArgument(v >= min,
	String.format("Value of %s: %d is below the minimum value %d",
	key, v, min));
	LOG.debug("Value of {} is {}", key, v);
	return v;
	}

	/**
	* Get a size property from the configuration: this property must
	* be at least equal to {@link Constants#MULTIPART_MIN_SIZE}.
	* If it is too small, it is rounded up to that minimum, and a warning
	* printed.
	* @param conf configuration
	* @param property property name
	* @param defVal default value
	* @return the value, guaranteed to be above the minimum size
	*/
	public static long getMultipartSizeProperty(Configuration conf,
	String property, long defVal) {
	long partSize = conf.getLong(property, defVal);
	if (partSize < MULTIPART_MIN_SIZE) {
	LOG.warn("{} must be at least 100 KB; configured value is {}",
	property, partSize);
	partSize = MULTIPART_MIN_SIZE;
	} else if (partSize > Integer.MAX_VALUE) {
	LOG.warn("oss: {} capped to ~2.14GB(maximum allowed size with " +
	"current output mechanism)", MULTIPART_UPLOAD_PART_SIZE_KEY);
	partSize = Integer.MAX_VALUE;
	}
	return partSize;
	}
	}