Merged latest changes from trunk
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-5442@1604086 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml
index 8eeafa5..5fcb938 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -100,8 +100,25 @@
<dependency>
<groupId>org.apache.directory.server</groupId>
<artifactId>apacheds-kerberos-codec</artifactId>
- <version>2.0.0-M15</version>
<scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.directory.api</groupId>
+ <artifactId>api-asn1-ber</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.directory.api</groupId>
+ <artifactId>api-i18n</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.directory.api</groupId>
+ <artifactId>api-ldap-model</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.sf.ehcache</groupId>
+ <artifactId>ehcache-core</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e01bacc..fe0ef94 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -149,6 +149,12 @@
HADOOP-10485. Remove dead classes in hadoop-streaming. (wheat9)
+ HADOOP-10607. Create API to separate credential/password storage from
+ applications. (Larry McCay via omalley)
+
+ HADOOP-10696. Add optional attributes to KeyProvider Options and Metadata.
+ (tucu)
+
BUG FIXES
HADOOP-9451. Fault single-layer config if node group topology is enabled.
@@ -568,6 +574,11 @@
HADOOP-10699. Fix build native library on mac osx (Binglin Chang via
jlowe)
+ HADOOP-10660. GraphiteSink should implement Closeable (Chen He and Ted Yu via raviprak)
+
+ HADOOP-10716. Cannot use more than 1 har filesystem.
+ (Rushabh Shah via cnauroth)
+
BREAKDOWN OF HADOOP-10514 SUBTASKS AND RELATED JIRAS
HADOOP-10520. Extended attributes definition and FileSystem APIs for
@@ -589,6 +600,10 @@
HADOOP-10561. Copy command with preserve option should handle Xattrs.
(Yi Liu via cnauroth)
+ HADOOP-10590. ServiceAuthorizationManager is not threadsafe. (Benoy Antony via vinayakumarb)
+
+ HADOOP-10711. Cleanup some extra dependencies from hadoop-auth. (rkanter via tucu)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index 3fa414b..1185342 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -116,6 +116,8 @@
elif [ "$COMMAND" = "archive" ] ; then
CLASS=org.apache.hadoop.tools.HadoopArchives
CLASSPATH=${CLASSPATH}:${TOOL_PATH}
+ elif [ "$COMMAND" = "credential" ] ; then
+ CLASS=org.apache.hadoop.security.alias.CredentialShell
elif [[ "$COMMAND" = -* ]] ; then
# class and package names cannot begin with a -
echo "Error: No command named \`$COMMAND' was found. Perhaps you meant \`hadoop ${COMMAND#-}'"
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 1dc333b..22ccf63 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -78,6 +78,9 @@
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProvider.CredentialEntry;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
@@ -1768,6 +1771,79 @@
}
/**
+ * Get the value for a known password configuration element.
+ * In order to enable the elimination of clear text passwords in config,
+ * this method attempts to resolve the property name as an alias through
+ * the CredentialProvider API and conditionally fallsback to config.
+ * @param name property name
+ * @return password
+ */
+ public char[] getPassword(String name) throws IOException {
+ char[] pass = null;
+
+ pass = getPasswordFromCredenitalProviders(name);
+
+ if (pass == null) {
+ pass = getPasswordFromConfig(name);
+ }
+
+ return pass;
+ }
+
+ /**
+ * Try and resolve the provided element name as a credential provider
+ * alias.
+ * @param name alias of the provisioned credential
+ * @return password or null if not found
+ * @throws IOException
+ */
+ protected char[] getPasswordFromCredenitalProviders(String name)
+ throws IOException {
+ char[] pass = null;
+ try {
+ List<CredentialProvider> providers =
+ CredentialProviderFactory.getProviders(this);
+
+ if (providers != null) {
+ for (CredentialProvider provider : providers) {
+ try {
+ CredentialEntry entry = provider.getCredentialEntry(name);
+ if (entry != null) {
+ pass = entry.getCredential();
+ break;
+ }
+ }
+ catch (IOException ioe) {
+ throw new IOException("Can't get key " + name + " from key provider" +
+ "of type: " + provider.getClass().getName() + ".", ioe);
+ }
+ }
+ }
+ }
+ catch (IOException ioe) {
+ throw new IOException("Configuration problem with provider path.", ioe);
+ }
+
+ return pass;
+ }
+
+ /**
+ * Fallback to clear text passwords in configuration.
+ * @param name
+ * @return clear text password or null
+ */
+ protected char[] getPasswordFromConfig(String name) {
+ char[] pass = null;
+ if (getBoolean(CredentialProvider.CLEAR_TEXT_FALLBACK, true)) {
+ String passStr = get(name);
+ if (passStr != null) {
+ pass = passStr.toCharArray();
+ }
+ }
+ return pass;
+ }
+
+ /**
* Get the socket address for <code>name</code> property as a
* <code>InetSocketAddress</code>.
* @param name property name.
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
index a0b6a8d..0f22f63 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/JavaKeyStoreProvider.java
@@ -26,6 +26,8 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.ProviderUtils;
+
import javax.crypto.spec.SecretKeySpec;
import java.io.IOException;
import java.io.InputStream;
@@ -101,7 +103,7 @@
private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
this.uri = uri;
- path = unnestUri(uri);
+ path = ProviderUtils.unnestUri(uri);
fs = path.getFileSystem(conf);
// Get the password file from the conf, if not present from the user's
// environment var
@@ -268,7 +270,7 @@
e);
}
Metadata meta = new Metadata(options.getCipher(), options.getBitLength(),
- options.getDescription(), new Date(), 1);
+ options.getDescription(), options.getAttributes(), new Date(), 1);
if (options.getBitLength() != 8 * material.length) {
throw new IOException("Wrong key length. Required " +
options.getBitLength() + ", but got " + (8 * material.length));
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
index 0b031c0..01d7b69 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProvider.java
@@ -26,8 +26,11 @@
import java.net.URI;
import java.security.NoSuchAlgorithmException;
import java.text.MessageFormat;
+import java.util.Collections;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
@@ -107,18 +110,22 @@
private final static String CREATED_FIELD = "created";
private final static String DESCRIPTION_FIELD = "description";
private final static String VERSIONS_FIELD = "versions";
+ private final static String ATTRIBUTES_FIELD = "attributes";
private final String cipher;
private final int bitLength;
private final String description;
private final Date created;
private int versions;
+ private Map<String, String> attributes;
- protected Metadata(String cipher, int bitLength,
- String description, Date created, int versions) {
+ protected Metadata(String cipher, int bitLength, String description,
+ Map<String, String> attributes, Date created, int versions) {
this.cipher = cipher;
this.bitLength = bitLength;
this.description = description;
+ this.attributes = (attributes == null || attributes.isEmpty())
+ ? null : attributes;
this.created = created;
this.versions = versions;
}
@@ -141,6 +148,11 @@
return cipher;
}
+ @SuppressWarnings("unchecked")
+ public Map<String, String> getAttributes() {
+ return (attributes == null) ? Collections.EMPTY_MAP : attributes;
+ }
+
/**
* Get the algorithm from the cipher.
* @return the algorithm name
@@ -188,6 +200,13 @@
if (description != null) {
writer.name(DESCRIPTION_FIELD).value(description);
}
+ if (attributes != null && attributes.size() > 0) {
+ writer.name(ATTRIBUTES_FIELD).beginObject();
+ for (Map.Entry<String, String> attribute : attributes.entrySet()) {
+ writer.name(attribute.getKey()).value(attribute.getValue());
+ }
+ writer.endObject();
+ }
writer.name(VERSIONS_FIELD).value(versions);
writer.endObject();
writer.flush();
@@ -208,6 +227,7 @@
Date created = null;
int versions = 0;
String description = null;
+ Map<String, String> attributes = null;
JsonReader reader = new JsonReader(new InputStreamReader
(new ByteArrayInputStream(bytes)));
try {
@@ -224,6 +244,13 @@
versions = reader.nextInt();
} else if (DESCRIPTION_FIELD.equals(field)) {
description = reader.nextString();
+ } else if (ATTRIBUTES_FIELD.equalsIgnoreCase(field)) {
+ reader.beginObject();
+ attributes = new HashMap<String, String>();
+ while (reader.hasNext()) {
+ attributes.put(reader.nextName(), reader.nextString());
+ }
+ reader.endObject();
}
}
reader.endObject();
@@ -234,6 +261,7 @@
this.bitLength = bitLength;
this.created = created;
this.description = description;
+ this.attributes = attributes;
this.versions = versions;
}
}
@@ -245,6 +273,7 @@
private String cipher;
private int bitLength;
private String description;
+ private Map<String, String> attributes;
public Options(Configuration conf) {
cipher = conf.get(DEFAULT_CIPHER_NAME, DEFAULT_CIPHER);
@@ -266,6 +295,16 @@
return this;
}
+ public Options setAttributes(Map<String, String> attributes) {
+ if (attributes != null) {
+ if (attributes.containsKey(null)) {
+ throw new IllegalArgumentException("attributes cannot have a NULL key");
+ }
+ this.attributes = new HashMap<String, String>(attributes);
+ }
+ return this;
+ }
+
public String getCipher() {
return cipher;
}
@@ -277,6 +316,11 @@
public String getDescription() {
return description;
}
+
+ @SuppressWarnings("unchecked")
+ public Map<String, String> getAttributes() {
+ return (attributes == null) ? Collections.EMPTY_MAP : attributes;
+ }
}
/**
@@ -489,33 +533,6 @@
}
/**
- * Convert a nested URI to decode the underlying path. The translation takes
- * the authority and parses it into the underlying scheme and authority.
- * For example, "myscheme://hdfs@nn/my/path" is converted to
- * "hdfs://nn/my/path".
- * @param nestedUri the URI from the nested URI
- * @return the unnested path
- */
- public static Path unnestUri(URI nestedUri) {
- String[] parts = nestedUri.getAuthority().split("@", 2);
- StringBuilder result = new StringBuilder(parts[0]);
- result.append("://");
- if (parts.length == 2) {
- result.append(parts[1]);
- }
- result.append(nestedUri.getPath());
- if (nestedUri.getQuery() != null) {
- result.append("?");
- result.append(nestedUri.getQuery());
- }
- if (nestedUri.getFragment() != null) {
- result.append("#");
- result.append(nestedUri.getFragment());
- }
- return new Path(result.toString());
- }
-
- /**
* Find the provider with the given key.
* @param providerList the list of providers
* @param keyName the key name we are looking for
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
index 371938b..6cfb46b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/UserProvider.java
@@ -89,7 +89,7 @@
options.getBitLength() + ", but got " + (8 * material.length));
}
Metadata meta = new Metadata(options.getCipher(), options.getBitLength(),
- options.getDescription(), new Date(), 1);
+ options.getDescription(), options.getAttributes(), new Date(), 1);
cache.put(name, meta);
String versionName = buildVersionName(name, 0);
credentials.addSecretKey(nameT, meta.serialize());
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 4c87ee1..41c1f60 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.client.PseudoAuthenticator;
@@ -82,6 +83,7 @@
return keyVersion;
}
+ @SuppressWarnings("unchecked")
private static Metadata parseJSONMetadata(Map valueMap) {
Metadata metadata = null;
if (!valueMap.isEmpty()) {
@@ -89,6 +91,7 @@
(String) valueMap.get(KMSRESTConstants.CIPHER_FIELD),
(Integer) valueMap.get(KMSRESTConstants.LENGTH_FIELD),
(String) valueMap.get(KMSRESTConstants.DESCRIPTION_FIELD),
+ (Map<String, String>) valueMap.get(KMSRESTConstants.ATTRIBUTES_FIELD),
new Date((Long) valueMap.get(KMSRESTConstants.CREATED_FIELD)),
(Integer) valueMap.get(KMSRESTConstants.VERSIONS_FIELD));
}
@@ -147,7 +150,7 @@
}
public KMSClientProvider(URI uri, Configuration conf) throws IOException {
- Path path = unnestUri(uri);
+ Path path = ProviderUtils.unnestUri(uri);
URL url = path.toUri().toURL();
kmsUrl = createServiceURL(url);
if ("https".equalsIgnoreCase(url.getProtocol())) {
@@ -350,8 +353,8 @@
public static class KMSMetadata extends Metadata {
public KMSMetadata(String cipher, int bitLength, String description,
- Date created, int versions) {
- super(cipher, bitLength, description, created, versions);
+ Map<String, String> attributes, Date created, int versions) {
+ super(cipher, bitLength, description, attributes, created, versions);
}
}
@@ -415,6 +418,9 @@
jsonKey.put(KMSRESTConstants.DESCRIPTION_FIELD,
options.getDescription());
}
+ if (options.getAttributes() != null && !options.getAttributes().isEmpty()) {
+ jsonKey.put(KMSRESTConstants.ATTRIBUTES_FIELD, options.getAttributes());
+ }
URL url = createURL(KMSRESTConstants.KEYS_RESOURCE, null, null, null);
HttpURLConnection conn = createConnection(url, HTTP_POST);
conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON_MIME);
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
index 3d2ea34..807cba7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
@@ -42,6 +42,7 @@
public static final String CIPHER_FIELD = "cipher";
public static final String LENGTH_FIELD = "length";
public static final String DESCRIPTION_FIELD = "description";
+ public static final String ATTRIBUTES_FIELD = "attributes";
public static final String CREATED_FIELD = "created";
public static final String VERSIONS_FIELD = "versions";
public static final String MATERIAL_FIELD = "material";
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java
index 7fa4448..b81ed4d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/GraphiteSink.java
@@ -18,13 +18,18 @@
package org.apache.hadoop.metrics2.sink;
+import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.io.Closeable;
import java.net.Socket;
import org.apache.commons.configuration.SubsetConfiguration;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricsException;
import org.apache.hadoop.metrics2.MetricsRecord;
@@ -36,12 +41,14 @@
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public class GraphiteSink implements MetricsSink {
+public class GraphiteSink implements MetricsSink, Closeable {
+ private static final Log LOG = LogFactory.getLog(GraphiteSink.class);
private static final String SERVER_HOST_KEY = "server_host";
private static final String SERVER_PORT_KEY = "server_port";
private static final String METRICS_PREFIX = "metrics_prefix";
private Writer writer = null;
private String metricsPrefix = null;
+ private Socket socket = null;
public void setWriter(Writer writer) {
this.writer = writer;
@@ -60,7 +67,7 @@
try {
// Open an connection to Graphite server.
- Socket socket = new Socket(serverHost, serverPort);
+ socket = new Socket(serverHost, serverPort);
setWriter(new OutputStreamWriter(socket.getOutputStream()));
} catch (Exception e) {
throw new MetricsException("Error creating connection, "
@@ -99,7 +106,11 @@
}
try {
- writer.write(lines.toString());
+ if(writer != null){
+ writer.write(lines.toString());
+ } else {
+ throw new MetricsException("Writer in GraphiteSink is null!");
+ }
} catch (Exception e) {
throw new MetricsException("Error sending metrics", e);
}
@@ -113,4 +124,21 @@
throw new MetricsException("Error flushing metrics", e);
}
}
+
+ @Override
+ public void close() throws IOException {
+ try {
+ IOUtils.closeStream(writer);
+ writer = null;
+ LOG.info("writer in GraphiteSink is closed!");
+ } catch (Throwable e){
+ throw new MetricsException("Error closing writer", e);
+ } finally {
+ if (socket != null && !socket.isClosed()) {
+ socket.close();
+ socket = null;
+ LOG.info("socket in GraphiteSink is closed!");
+ }
+ }
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
index 7504991..c62caf3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
@@ -38,6 +38,7 @@
import org.apache.hadoop.metrics2.lib.Interns;
import static org.apache.hadoop.metrics2.source.JvmMetricsInfo.*;
import static org.apache.hadoop.metrics2.impl.MsInfo.*;
+import org.apache.hadoop.util.JvmPauseMonitor;
/**
* JVM and logging related metrics.
@@ -65,6 +66,7 @@
ManagementFactory.getGarbageCollectorMXBeans();
final ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean();
final String processName, sessionId;
+ private JvmPauseMonitor pauseMonitor = null;
final ConcurrentHashMap<String, MetricsInfo[]> gcInfoCache =
new ConcurrentHashMap<String, MetricsInfo[]>();
@@ -73,6 +75,10 @@
this.sessionId = sessionId;
}
+ public void setPauseMonitor(final JvmPauseMonitor pauseMonitor) {
+ this.pauseMonitor = pauseMonitor;
+ }
+
public static JvmMetrics create(String processName, String sessionId,
MetricsSystem ms) {
return ms.register(JvmMetrics.name(), JvmMetrics.description(),
@@ -120,6 +126,15 @@
}
rb.addCounter(GcCount, count)
.addCounter(GcTimeMillis, timeMillis);
+
+ if (pauseMonitor != null) {
+ rb.addCounter(GcNumWarnThresholdExceeded,
+ pauseMonitor.getNumGcWarnThreadholdExceeded());
+ rb.addCounter(GcNumInfoThresholdExceeded,
+ pauseMonitor.getNumGcInfoThresholdExceeded());
+ rb.addCounter(GcTotalExtraSleepTime,
+ pauseMonitor.getTotalGcExtraSleepTime());
+ }
}
private MetricsInfo[] getGcInfo(String gcName) {
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java
index 64a59b6..55bb417 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetricsInfo.java
@@ -48,7 +48,10 @@
LogFatal("Total number of fatal log events"),
LogError("Total number of error log events"),
LogWarn("Total number of warning log events"),
- LogInfo("Total number of info log events");
+ LogInfo("Total number of info log events"),
+ GcNumWarnThresholdExceeded("Number of times that the GC warn threshold is exceeded"),
+ GcNumInfoThresholdExceeded("Number of times that the GC info threshold is exceeded"),
+ GcTotalExtraSleepTime("Total GC extra sleep time in milliseconds");
private final String desc;
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
new file mode 100644
index 0000000..97d656d
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ProviderUtils.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security;
+
+import java.net.URI;
+
+import org.apache.hadoop.fs.Path;
+
+public class ProviderUtils {
+ /**
+ * Convert a nested URI to decode the underlying path. The translation takes
+ * the authority and parses it into the underlying scheme and authority.
+ * For example, "myscheme://hdfs@nn/my/path" is converted to
+ * "hdfs://nn/my/path".
+ * @param nestedUri the URI from the nested URI
+ * @return the unnested path
+ */
+ public static Path unnestUri(URI nestedUri) {
+ String[] parts = nestedUri.getAuthority().split("@", 2);
+ StringBuilder result = new StringBuilder(parts[0]);
+ result.append("://");
+ if (parts.length == 2) {
+ result.append(parts[1]);
+ }
+ result.append(nestedUri.getPath());
+ if (nestedUri.getQuery() != null) {
+ result.append("?");
+ result.append(nestedUri.getQuery());
+ }
+ if (nestedUri.getFragment() != null) {
+ result.append("#");
+ result.append(nestedUri.getFragment());
+ }
+ return new Path(result.toString());
+ }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProvider.java
new file mode 100644
index 0000000..bded4b97
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProvider.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.alias;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * A provider of credentials or password for Hadoop applications. Provides an
+ * abstraction to separate credential storage from users of them. It
+ * is intended to support getting or storing passwords in a variety of ways,
+ * including third party bindings.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public abstract class CredentialProvider {
+ public static final String CLEAR_TEXT_FALLBACK
+ = "hadoop.security.credential.clear-text-fallback";
+
+ /**
+ * The combination of both the alias and the actual credential value.
+ */
+ public static class CredentialEntry {
+ private final String alias;
+ private final char[] credential;
+
+ protected CredentialEntry(String alias,
+ char[] credential) {
+ this.alias = alias;
+ this.credential = credential;
+ }
+
+ public String getAlias() {
+ return alias;
+ }
+
+ public char[] getCredential() {
+ return credential;
+ }
+
+ public String toString() {
+ StringBuilder buf = new StringBuilder();
+ buf.append("alias(");
+ buf.append(alias);
+ buf.append(")=");
+ if (credential == null) {
+ buf.append("null");
+ } else {
+ for(char c: credential) {
+ buf.append(c);
+ }
+ }
+ return buf.toString();
+ }
+ }
+
+ /**
+ * Indicates whether this provider represents a store
+ * that is intended for transient use - such as the UserProvider
+ * is. These providers are generally used to provide job access to
+ * passwords rather than for long term storage.
+ * @return true if transient, false otherwise
+ */
+ public boolean isTransient() {
+ return false;
+ }
+
+ /**
+ * Ensures that any changes to the credentials are written to persistent store.
+ * @throws IOException
+ */
+ public abstract void flush() throws IOException;
+
+ /**
+ * Get the credential entry for a specific alias.
+ * @param alias the name of a specific credential
+ * @return the credentialEntry
+ * @throws IOException
+ */
+ public abstract CredentialEntry getCredentialEntry(String alias)
+ throws IOException;
+
+ /**
+ * Get the aliases for all credentials.
+ * @return the list of alias names
+ * @throws IOException
+ */
+ public abstract List<String> getAliases() throws IOException;
+
+ /**
+ * Create a new credential. The given alias must not already exist.
+ * @param name the alias of the credential
+ * @param credential the credential value for the alias.
+ * @throws IOException
+ */
+ public abstract CredentialEntry createCredentialEntry(String name,
+ char[] credential) throws IOException;
+
+ /**
+ * Delete the given credential.
+ * @param name the alias of the credential to delete
+ * @throws IOException
+ */
+ public abstract void deleteCredentialEntry(String name) throws IOException;
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProviderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProviderFactory.java
new file mode 100644
index 0000000..36b4ecb
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialProviderFactory.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.alias;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ServiceLoader;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * A factory to create a list of CredentialProvider based on the path given in a
+ * Configuration. It uses a service loader interface to find the available
+ * CredentialProviders and create them based on the list of URIs.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public abstract class CredentialProviderFactory {
+ public static final String CREDENTIAL_PROVIDER_PATH =
+ "hadoop.security.credential.provider.path";
+
+ public abstract CredentialProvider createProvider(URI providerName,
+ Configuration conf
+ ) throws IOException;
+
+ private static final ServiceLoader<CredentialProviderFactory> serviceLoader =
+ ServiceLoader.load(CredentialProviderFactory.class);
+
+ public static List<CredentialProvider> getProviders(Configuration conf
+ ) throws IOException {
+ List<CredentialProvider> result = new ArrayList<CredentialProvider>();
+ for(String path: conf.getStringCollection(CREDENTIAL_PROVIDER_PATH)) {
+ try {
+ URI uri = new URI(path);
+ boolean found = false;
+ for(CredentialProviderFactory factory: serviceLoader) {
+ CredentialProvider kp = factory.createProvider(uri, conf);
+ if (kp != null) {
+ result.add(kp);
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ throw new IOException("No CredentialProviderFactory for " + uri + " in " +
+ CREDENTIAL_PROVIDER_PATH);
+ }
+ } catch (URISyntaxException error) {
+ throw new IOException("Bad configuration of " + CREDENTIAL_PROVIDER_PATH +
+ " at " + path, error);
+ }
+ }
+ return result;
+ }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
new file mode 100644
index 0000000..02f4f75
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
@@ -0,0 +1,423 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.alias;
+
+import java.io.Console;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.security.InvalidParameterException;
+import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+/**
+ * This program is the CLI utility for the CredentialProvider facilities in
+ * Hadoop.
+ */
+public class CredentialShell extends Configured implements Tool {
+ final static private String USAGE_PREFIX = "Usage: hadoop credential " +
+ "[generic options]\n";
+ final static private String COMMANDS =
+ " [--help]\n" +
+ " [" + CreateCommand.USAGE + "]\n" +
+ " [" + DeleteCommand.USAGE + "]\n" +
+ " [" + ListCommand.USAGE + "]\n";
+
+ private boolean interactive = false;
+ private Command command = null;
+
+ /** allows stdout to be captured if necessary */
+ public PrintStream out = System.out;
+ /** allows stderr to be captured if necessary */
+ public PrintStream err = System.err;
+
+ private boolean userSuppliedProvider = false;
+ private String value = null;
+ private PasswordReader passwordReader;
+
+ @Override
+ public int run(String[] args) throws Exception {
+ int exitCode = 0;
+ try {
+ exitCode = init(args);
+ if (exitCode != 0) {
+ return exitCode;
+ }
+ if (command.validate()) {
+ command.execute();
+ } else {
+ exitCode = -1;
+ }
+ } catch (Exception e) {
+ e.printStackTrace(err);
+ return -1;
+ }
+ return exitCode;
+ }
+
+ /**
+ * Parse the command line arguments and initialize the data
+ * <pre>
+ * % hadoop alias create alias [--provider providerPath]
+ * % hadoop alias list [-provider providerPath]
+ * % hadoop alias delete alias [--provider providerPath] [-i]
+ * </pre>
+ * @param args
+ * @return
+ * @throws IOException
+ */
+ private int init(String[] args) throws IOException {
+ for (int i = 0; i < args.length; i++) { // parse command line
+ if (args[i].equals("create")) {
+ String alias = args[++i];
+ command = new CreateCommand(alias);
+ if (alias.equals("--help")) {
+ printCredShellUsage();
+ return -1;
+ }
+ } else if (args[i].equals("delete")) {
+ String alias = args[++i];
+ command = new DeleteCommand(alias);
+ if (alias.equals("--help")) {
+ printCredShellUsage();
+ return -1;
+ }
+ } else if (args[i].equals("list")) {
+ command = new ListCommand();
+ } else if (args[i].equals("--provider")) {
+ userSuppliedProvider = true;
+ getConf().set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ args[++i]);
+ } else if (args[i].equals("-i") || (args[i].equals("--interactive"))) {
+ interactive = true;
+ } else if (args[i].equals("-v") || (args[i].equals("--value"))) {
+ value = args[++i];
+ } else if (args[i].equals("--help")) {
+ printCredShellUsage();
+ return -1;
+ } else {
+ printCredShellUsage();
+ ToolRunner.printGenericCommandUsage(System.err);
+ return -1;
+ }
+ }
+ return 0;
+ }
+
+ private void printCredShellUsage() {
+ out.println(USAGE_PREFIX + COMMANDS);
+ if (command != null) {
+ out.println(command.getUsage());
+ }
+ else {
+ out.println("=========================================================" +
+ "======");
+ out.println(CreateCommand.USAGE + ":\n\n" + CreateCommand.DESC);
+ out.println("=========================================================" +
+ "======");
+ out.println(DeleteCommand.USAGE + ":\n\n" + DeleteCommand.DESC);
+ out.println("=========================================================" +
+ "======");
+ out.println(ListCommand.USAGE + ":\n\n" + ListCommand.DESC);
+ }
+ }
+
+ private abstract class Command {
+ protected CredentialProvider provider = null;
+
+ public boolean validate() {
+ return true;
+ }
+
+ protected CredentialProvider getCredentialProvider() {
+ CredentialProvider provider = null;
+ List<CredentialProvider> providers;
+ try {
+ providers = CredentialProviderFactory.getProviders(getConf());
+ if (userSuppliedProvider) {
+ provider = providers.get(0);
+ }
+ else {
+ for (CredentialProvider p : providers) {
+ if (!p.isTransient()) {
+ provider = p;
+ break;
+ }
+ }
+ }
+ } catch (IOException e) {
+ e.printStackTrace(err);
+ }
+ return provider;
+ }
+
+ protected void printProviderWritten() {
+ out.println(provider.getClass().getName() + " has been updated.");
+ }
+
+ protected void warnIfTransientProvider() {
+ if (provider.isTransient()) {
+ out.println("WARNING: you are modifying a transient provider.");
+ }
+ }
+
+ public abstract void execute() throws Exception;
+
+ public abstract String getUsage();
+ }
+
+ private class ListCommand extends Command {
+ public static final String USAGE = "list <alias> [--provider] [--help]";
+ public static final String DESC =
+ "The list subcommand displays the aliases contained within \n" +
+ "a particular provider - as configured in core-site.xml or " +
+ "indicated\nthrough the --provider argument.";
+
+ public boolean validate() {
+ boolean rc = true;
+ provider = getCredentialProvider();
+ if (provider == null) {
+ out.println("There are no non-transient CredentialProviders configured.\n"
+ + "Consider using the --provider option to indicate the provider\n"
+ + "to use. If you want to list a transient provider then you\n"
+ + "you MUST use the --provider argument.");
+ rc = false;
+ }
+ return rc;
+ }
+
+ public void execute() throws IOException {
+ List<String> aliases;
+ try {
+ aliases = provider.getAliases();
+ out.println("Listing aliases for CredentialProvider: " + provider.toString());
+ for (String alias : aliases) {
+ out.println(alias);
+ }
+ } catch (IOException e) {
+ out.println("Cannot list aliases for CredentialProvider: " + provider.toString()
+ + ": " + e.getMessage());
+ throw e;
+ }
+ }
+
+ @Override
+ public String getUsage() {
+ return USAGE + ":\n\n" + DESC;
+ }
+ }
+
+ private class DeleteCommand extends Command {
+ public static final String USAGE = "delete <alias> [--provider] [--help]";
+ public static final String DESC =
+ "The delete subcommand deletes the credenital\n" +
+ "specified as the <alias> argument from within the provider\n" +
+ "indicated through the --provider argument";
+
+ String alias = null;
+ boolean cont = true;
+
+ public DeleteCommand(String alias) {
+ this.alias = alias;
+ }
+
+ @Override
+ public boolean validate() {
+ provider = getCredentialProvider();
+ if (provider == null) {
+ out.println("There are no valid CredentialProviders configured.\n"
+ + "Nothing will be deleted.\n"
+ + "Consider using the --provider option to indicate the provider"
+ + " to use.");
+ return false;
+ }
+ if (alias == null) {
+ out.println("There is no alias specified. Please provide the" +
+ "mandatory <alias>. See the usage description with --help.");
+ return false;
+ }
+ if (interactive) {
+ try {
+ cont = ToolRunner
+ .confirmPrompt("You are about to DELETE the credential: " +
+ alias + " from CredentialProvider " + provider.toString() +
+ ". Continue?:");
+ if (!cont) {
+ out.println("Nothing has been be deleted.");
+ }
+ return cont;
+ } catch (IOException e) {
+ out.println(alias + " will not be deleted.");
+ e.printStackTrace(err);
+ }
+ }
+ return true;
+ }
+
+ public void execute() throws IOException {
+ warnIfTransientProvider();
+ out.println("Deleting credential: " + alias + " from CredentialProvider: "
+ + provider.toString());
+ if (cont) {
+ try {
+ provider.deleteCredentialEntry(alias);
+ out.println(alias + " has been successfully deleted.");
+ provider.flush();
+ printProviderWritten();
+ } catch (IOException e) {
+ out.println(alias + "has NOT been deleted.");
+ throw e;
+ }
+ }
+ }
+
+ @Override
+ public String getUsage() {
+ return USAGE + ":\n\n" + DESC;
+ }
+ }
+
+ private class CreateCommand extends Command {
+ public static final String USAGE = "create <alias> [--provider] [--help]";
+ public static final String DESC =
+ "The create subcommand creates a new credential for the name specified\n" +
+ "as the <alias> argument within the provider indicated through\n" +
+ "the --provider argument.";
+
+ String alias = null;
+
+ public CreateCommand(String alias) {
+ this.alias = alias;
+ }
+
+ public boolean validate() {
+ boolean rc = true;
+ provider = getCredentialProvider();
+ if (provider == null) {
+ out.println("There are no valid CredentialProviders configured." +
+ "\nCredential will not be created.\n"
+ + "Consider using the --provider option to indicate the provider" +
+ " to use.");
+ rc = false;
+ }
+ if (alias == null) {
+ out.println("There is no alias specified. Please provide the" +
+ "mandatory <alias>. See the usage description with --help.");
+ rc = false;
+ }
+ return rc;
+ }
+
+ public void execute() throws IOException, NoSuchAlgorithmException {
+ warnIfTransientProvider();
+ try {
+ char[] credential = null;
+ if (value != null) {
+ // testing only
+ credential = value.toCharArray();
+ }
+ else {
+ credential = promptForCredential();
+ }
+ provider.createCredentialEntry(alias, credential);
+ out.println(alias + " has been successfully created.");
+ provider.flush();
+ printProviderWritten();
+ } catch (InvalidParameterException e) {
+ out.println(alias + " has NOT been created. " + e.getMessage());
+ throw e;
+ } catch (IOException e) {
+ out.println(alias + " has NOT been created. " + e.getMessage());
+ throw e;
+ }
+ }
+
+ @Override
+ public String getUsage() {
+ return USAGE + ":\n\n" + DESC;
+ }
+ }
+
+ protected char[] promptForCredential() throws IOException {
+ PasswordReader c = getPasswordReader();
+ if (c == null) {
+ throw new IOException("No console available for prompting user.");
+ }
+
+ char[] cred = null;
+
+ boolean noMatch;
+ do {
+ char[] newPassword1 = c.readPassword("Enter password: ");
+ char[] newPassword2 = c.readPassword("Enter password again: ");
+ noMatch = !Arrays.equals(newPassword1, newPassword2);
+ if (noMatch) {
+ Arrays.fill(newPassword1, ' ');
+ c.format("Passwords don't match. Try again.%n");
+ } else {
+ cred = newPassword1;
+ }
+ Arrays.fill(newPassword2, ' ');
+ } while (noMatch);
+ return cred;
+ }
+
+ public PasswordReader getPasswordReader() {
+ if (passwordReader == null) {
+ passwordReader = new PasswordReader();
+ }
+ return passwordReader;
+ }
+
+ public void setPasswordReader(PasswordReader reader) {
+ passwordReader = reader;
+ }
+
+ // to facilitate testing since Console is a final class...
+ public static class PasswordReader {
+ public char[] readPassword(String prompt) {
+ Console console = System.console();
+ char[] pass = console.readPassword(prompt);
+ return pass;
+ }
+
+ public void format(String message) {
+ Console console = System.console();
+ console.format(message);
+ }
+ }
+
+
+ /**
+ * Main program.
+ *
+ * @param args
+ * Command line arguments
+ * @throws Exception
+ */
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(new Configuration(), new CredentialShell(), args);
+ System.exit(res);
+ }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
new file mode 100644
index 0000000..61958fe
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
@@ -0,0 +1,290 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.alias;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.ProviderUtils;
+
+import javax.crypto.spec.SecretKeySpec;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.net.URL;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * CredentialProvider based on Java's KeyStore file format. The file may be
+ * stored in any Hadoop FileSystem using the following name mangling:
+ * jceks://hdfs@nn1.example.com/my/creds.jceks -> hdfs://nn1.example.com/my/creds.jceks
+ * jceks://file/home/larry/creds.jceks -> file:///home/larry/creds.jceks
+ *
+ * The password for the keystore is taken from the HADOOP_CREDSTORE_PASSWORD
+ * environment variable with a default of 'none'.
+ *
+ * It is expected that for access to credential protected resource to copy the
+ * creds from the original provider into the job's Credentials object, which is
+ * accessed via the UserProvider. Therefore, this provider won't be directly
+ * used by MapReduce tasks.
+ */
+@InterfaceAudience.Private
+public class JavaKeyStoreProvider extends CredentialProvider {
+ public static final String SCHEME_NAME = "jceks";
+ public static final String CREDENTIAL_PASSWORD_NAME =
+ "HADOOP_CREDSTORE_PASSWORD";
+ public static final String KEYSTORE_PASSWORD_FILE_KEY =
+ "hadoop.security.credstore.java-keystore-provider.password-file";
+ public static final String KEYSTORE_PASSWORD_DEFAULT = "none";
+
+ private final URI uri;
+ private final Path path;
+ private final FileSystem fs;
+ private final FsPermission permissions;
+ private final KeyStore keyStore;
+ private char[] password = null;
+ private boolean changed = false;
+ private Lock readLock;
+ private Lock writeLock;
+
+ private final Map<String, CredentialEntry> cache = new HashMap<String, CredentialEntry>();
+
+ private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
+ this.uri = uri;
+ path = ProviderUtils.unnestUri(uri);
+ fs = path.getFileSystem(conf);
+ // Get the password from the user's environment
+ if (System.getenv().containsKey(CREDENTIAL_PASSWORD_NAME)) {
+ password = System.getenv(CREDENTIAL_PASSWORD_NAME).toCharArray();
+ }
+ // if not in ENV get check for file
+ if (password == null) {
+ String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
+ if (pwFile != null) {
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ URL pwdFile = cl.getResource(pwFile);
+ if (pwdFile != null) {
+ InputStream is = pwdFile.openStream();
+ try {
+ password = IOUtils.toCharArray(is);
+ } finally {
+ is.close();
+ }
+ }
+ }
+ }
+ if (password == null) {
+ password = KEYSTORE_PASSWORD_DEFAULT.toCharArray();
+ }
+ try {
+ keyStore = KeyStore.getInstance(SCHEME_NAME);
+ if (fs.exists(path)) {
+ // save off permissions in case we need to
+ // rewrite the keystore in flush()
+ FileStatus s = fs.getFileStatus(path);
+ permissions = s.getPermission();
+
+ keyStore.load(fs.open(path), password);
+ } else {
+ permissions = new FsPermission("700");
+ // required to create an empty keystore. *sigh*
+ keyStore.load(null, password);
+ }
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't create keystore", e);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IOException("Can't load keystore " + path, e);
+ } catch (CertificateException e) {
+ throw new IOException("Can't load keystore " + path, e);
+ }
+ ReadWriteLock lock = new ReentrantReadWriteLock(true);
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+ }
+
+ @Override
+ public CredentialEntry getCredentialEntry(String alias) throws IOException {
+ readLock.lock();
+ try {
+ SecretKeySpec key = null;
+ try {
+ if (cache.containsKey(alias)) {
+ return cache.get(alias);
+ }
+ if (!keyStore.containsAlias(alias)) {
+ return null;
+ }
+ key = (SecretKeySpec) keyStore.getKey(alias, password);
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't get credential " + alias + " from " +
+ path, e);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IOException("Can't get algorithm for credential " + alias + " from " +
+ path, e);
+ } catch (UnrecoverableKeyException e) {
+ throw new IOException("Can't recover credential " + alias + " from " + path, e);
+ }
+ return new CredentialEntry(alias, bytesToChars(key.getEncoded()));
+ }
+ finally {
+ readLock.unlock();
+ }
+ }
+
+ public static char[] bytesToChars(byte[] bytes) {
+ String pass = new String(bytes);
+ return pass.toCharArray();
+ }
+
+ @Override
+ public List<String> getAliases() throws IOException {
+ readLock.lock();
+ try {
+ ArrayList<String> list = new ArrayList<String>();
+ String alias = null;
+ try {
+ Enumeration<String> e = keyStore.aliases();
+ while (e.hasMoreElements()) {
+ alias = e.nextElement();
+ list.add(alias);
+ }
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't get alias " + alias + " from " + path, e);
+ }
+ return list;
+ }
+ finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public CredentialEntry createCredentialEntry(String alias, char[] credential)
+ throws IOException {
+ try {
+ if (keyStore.containsAlias(alias) || cache.containsKey(alias)) {
+ throw new IOException("Credential " + alias + " already exists in " + this);
+ }
+ } catch (KeyStoreException e) {
+ throw new IOException("Problem looking up credential " + alias + " in " + this,
+ e);
+ }
+ return innerSetCredential(alias, credential);
+ }
+
+ @Override
+ public void deleteCredentialEntry(String name) throws IOException {
+ writeLock.lock();
+ try {
+ try {
+ if (keyStore.containsAlias(name)) {
+ keyStore.deleteEntry(name);
+ }
+ else {
+ throw new IOException("Credential " + name + " does not exist in " + this);
+ }
+ } catch (KeyStoreException e) {
+ throw new IOException("Problem removing " + name + " from " +
+ this, e);
+ }
+ cache.remove(name);
+ changed = true;
+ }
+ finally {
+ writeLock.unlock();
+ }
+ }
+
+ CredentialEntry innerSetCredential(String alias, char[] material)
+ throws IOException {
+ try {
+ keyStore.setKeyEntry(alias, new SecretKeySpec(
+ new String(material).getBytes("UTF-8"), "AES"),
+ password, null);
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't store credential " + alias + " in " + this,
+ e);
+ }
+ changed = true;
+ return new CredentialEntry(alias, material);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ writeLock.lock();
+ try {
+ if (!changed) {
+ return;
+ }
+ // write out the keystore
+ FSDataOutputStream out = FileSystem.create(fs, path, permissions);
+ try {
+ keyStore.store(out, password);
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't store keystore " + this, e);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IOException("No such algorithm storing keystore " + this, e);
+ } catch (CertificateException e) {
+ throw new IOException("Certificate exception storing keystore " + this,
+ e);
+ }
+ out.close();
+ changed = false;
+ }
+ finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return uri.toString();
+ }
+
+ /**
+ * The factory to create JksProviders, which is used by the ServiceLoader.
+ */
+ public static class Factory extends CredentialProviderFactory {
+ @Override
+ public CredentialProvider createProvider(URI providerName,
+ Configuration conf) throws IOException {
+ if (SCHEME_NAME.equals(providerName.getScheme())) {
+ return new JavaKeyStoreProvider(providerName, conf);
+ }
+ return null;
+ }
+ }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java
new file mode 100644
index 0000000..9e724c0
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/UserProvider.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.alias;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+
+/**
+ * A CredentialProvider for UGIs. It uses the credentials object associated
+ * with the current user to find credentials. This provider is created using a
+ * URI of "user:///".
+ */
+@InterfaceAudience.Private
+public class UserProvider extends CredentialProvider {
+ public static final String SCHEME_NAME = "user";
+ private final UserGroupInformation user;
+ private final Credentials credentials;
+ private final Map<String, CredentialEntry> cache = new HashMap<String,
+ CredentialEntry>();
+
+ private UserProvider() throws IOException {
+ user = UserGroupInformation.getCurrentUser();
+ credentials = user.getCredentials();
+ }
+
+ @Override
+ public boolean isTransient() {
+ return true;
+ }
+
+ @Override
+ public CredentialEntry getCredentialEntry(String alias) {
+ byte[] bytes = credentials.getSecretKey(new Text(alias));
+ if (bytes == null) {
+ return null;
+ }
+ return new CredentialEntry(alias, new String(bytes).toCharArray());
+ }
+
+ @Override
+ public CredentialEntry createCredentialEntry(String name, char[] credential)
+ throws IOException {
+ Text nameT = new Text(name);
+ if (credentials.getSecretKey(nameT) != null) {
+ throw new IOException("Credential " + name +
+ " already exists in " + this);
+ }
+ credentials.addSecretKey(new Text(name),
+ new String(credential).getBytes("UTF-8"));
+ return new CredentialEntry(name, credential);
+ }
+
+ @Override
+ public void deleteCredentialEntry(String name) throws IOException {
+ byte[] cred = credentials.getSecretKey(new Text(name));
+ if (cred != null) {
+ credentials.removeSecretKey(new Text(name));
+ }
+ else {
+ throw new IOException("Credential " + name +
+ " does not exist in " + this);
+ }
+ cache.remove(name);
+ }
+
+ @Override
+ public String toString() {
+ return SCHEME_NAME + ":///";
+ }
+
+ @Override
+ public void flush() {
+ user.addCredentials(credentials);
+ }
+
+ public static class Factory extends CredentialProviderFactory {
+
+ @Override
+ public CredentialProvider createProvider(URI providerName,
+ Configuration conf) throws IOException {
+ if (SCHEME_NAME.equals(providerName.getScheme())) {
+ return new UserProvider();
+ }
+ return null;
+ }
+ }
+
+ @Override
+ public List<String> getAliases() throws IOException {
+ List<String> list = new ArrayList<String>();
+ List<Text> aliases = credentials.getAllSecretKeys();
+ for (Text key : aliases) {
+ list.add(key.toString());
+ }
+ return list;
+ }
+}
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java
index 66ab50c..29d4a6a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/authorize/ServiceAuthorizationManager.java
@@ -45,7 +45,7 @@
public class ServiceAuthorizationManager {
private static final String HADOOP_POLICY_FILE = "hadoop-policy.xml";
- private Map<Class<?>, AccessControlList> protocolToAcl =
+ private volatile Map<Class<?>, AccessControlList> protocolToAcl =
new IdentityHashMap<Class<?>, AccessControlList>();
/**
@@ -114,7 +114,7 @@
AUDITLOG.info(AUTHZ_SUCCESSFUL_FOR + user + " for protocol="+protocol);
}
- public synchronized void refresh(Configuration conf,
+ public void refresh(Configuration conf,
PolicyProvider provider) {
// Get the system property 'hadoop.policy.file'
String policyFile =
@@ -127,7 +127,7 @@
}
@Private
- public synchronized void refreshWithLoadedConfiguration(Configuration conf,
+ public void refreshWithLoadedConfiguration(Configuration conf,
PolicyProvider provider) {
final Map<Class<?>, AccessControlList> newAcls =
new IdentityHashMap<Class<?>, AccessControlList>();
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
index f7932a6..e8af45e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
@@ -62,10 +62,13 @@
"jvm.pause.info-threshold.ms";
private static final long INFO_THRESHOLD_DEFAULT = 1000;
-
+ private long numGcWarnThresholdExceeded = 0;
+ private long numGcInfoThresholdExceeded = 0;
+ private long totalGcExtraSleepTime = 0;
+
private Thread monitorThread;
private volatile boolean shouldRun = true;
-
+
public JvmPauseMonitor(Configuration conf) {
this.warnThresholdMs = conf.getLong(WARN_THRESHOLD_KEY, WARN_THRESHOLD_DEFAULT);
this.infoThresholdMs = conf.getLong(INFO_THRESHOLD_KEY, INFO_THRESHOLD_DEFAULT);
@@ -87,6 +90,22 @@
Thread.currentThread().interrupt();
}
}
+
+ public boolean isStarted() {
+ return monitorThread != null;
+ }
+
+ public long getNumGcWarnThreadholdExceeded() {
+ return numGcWarnThresholdExceeded;
+ }
+
+ public long getNumGcInfoThresholdExceeded() {
+ return numGcInfoThresholdExceeded;
+ }
+
+ public long getTotalGcExtraSleepTime() {
+ return totalGcExtraSleepTime;
+ }
private String formatMessage(long extraSleepTime,
Map<String, GcTimes> gcTimesAfterSleep,
@@ -166,13 +185,15 @@
Map<String, GcTimes> gcTimesAfterSleep = getGcTimes();
if (extraSleepTime > warnThresholdMs) {
+ ++numGcWarnThresholdExceeded;
LOG.warn(formatMessage(
extraSleepTime, gcTimesAfterSleep, gcTimesBeforeSleep));
} else if (extraSleepTime > infoThresholdMs) {
+ ++numGcInfoThresholdExceeded;
LOG.info(formatMessage(
extraSleepTime, gcTimesAfterSleep, gcTimesBeforeSleep));
}
-
+ totalGcExtraSleepTime += extraSleepTime;
gcTimesBeforeSleep = gcTimesAfterSleep;
}
}
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
new file mode 100644
index 0000000..bb7366e
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.security.alias.JavaKeyStoreProvider$Factory
+org.apache.hadoop.security.alias.UserProvider$Factory
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 0dbb53f..aef6582 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1358,4 +1358,9 @@
true.
</description>
</property>
+<property>
+ <name>fs.har.impl.disable.cache</name>
+ <value>true</value>
+ <description>Don't cache 'har' filesystem instances.</description>
+</property>
</configuration>
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
index 18114bb..f4fabab 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
@@ -306,9 +306,9 @@
Runs a cluster balancing utility. An administrator can simply press Ctrl-C
to stop the rebalancing process. See
- {{{../hadoop-hdfs/HdfsUserGuide.html#Rebalancer}Rebalancer}} for more details.
+ {{{../hadoop-hdfs/HdfsUserGuide.html#Balancer}Balancer}} for more details.
- Usage: <<<hadoop balancer [-threshold <threshold>]>>>
+ Usage: <<<hadoop balancer [-threshold <threshold>] [-policy <policy>]>>>
*------------------------+-----------------------------------------------------------+
|| COMMAND_OPTION | Description
@@ -316,6 +316,11 @@
| -threshold <threshold> | Percentage of disk capacity. This overwrites the
| default threshold.
*------------------------+-----------------------------------------------------------+
+| -policy <policy> | <<<datanode>>> (default): Cluster is balanced if each datanode is balanced. \
+ | <<<blockpool>>>: Cluster is balanced if each block pool in each datanode is balanced.
+*------------------------+-----------------------------------------------------------+
+
+ Note that the <<<blockpool>>> policy is more strict than the <<<datanode>>> policy.
* <<<daemonlog>>>
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/Metrics.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/Metrics.apt.vm
index 55e532d..14cc712 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/Metrics.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/Metrics.apt.vm
@@ -86,6 +86,14 @@
*-------------------------------------+--------------------------------------+
|<<<LogInfo>>> | Total number of INFO logs
*-------------------------------------+--------------------------------------+
+|<<<GcNumWarnThresholdExceeded>>> | Number of times that the GC warn
+ | threshold is exceeded
+*-------------------------------------+--------------------------------------+
+|<<<GcNumInfoThresholdExceeded>>> | Number of times that the GC info
+ | threshold is exceeded
+*-------------------------------------+--------------------------------------+
+|<<<GcTotalExtraSleepTime>>> | Total GC extra sleep time in msec
+*-------------------------------------+--------------------------------------+
rpc context
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
index 42069bc..7da1675 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProvider.java
@@ -21,6 +21,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.ProviderUtils;
import org.junit.Test;
import java.io.IOException;
@@ -29,7 +30,9 @@
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
@@ -72,7 +75,7 @@
DateFormat format = new SimpleDateFormat("y/m/d");
Date date = format.parse("2013/12/25");
KeyProvider.Metadata meta = new KeyProvider.Metadata("myCipher", 100, null,
- date, 123);
+ null, date, 123);
assertEquals("myCipher", meta.getCipher());
assertEquals(100, meta.getBitLength());
assertNull(meta.getDescription());
@@ -82,6 +85,7 @@
assertEquals(meta.getCipher(), second.getCipher());
assertEquals(meta.getBitLength(), second.getBitLength());
assertNull(second.getDescription());
+ assertTrue(second.getAttributes().isEmpty());
assertEquals(meta.getCreated(), second.getCreated());
assertEquals(meta.getVersions(), second.getVersions());
int newVersion = second.addVersion();
@@ -92,17 +96,21 @@
//Metadata with description
format = new SimpleDateFormat("y/m/d");
date = format.parse("2013/12/25");
+ Map<String, String> attributes = new HashMap<String, String>();
+ attributes.put("a", "A");
meta = new KeyProvider.Metadata("myCipher", 100,
- "description", date, 123);
+ "description", attributes, date, 123);
assertEquals("myCipher", meta.getCipher());
assertEquals(100, meta.getBitLength());
assertEquals("description", meta.getDescription());
+ assertEquals(attributes, meta.getAttributes());
assertEquals(date, meta.getCreated());
assertEquals(123, meta.getVersions());
second = new KeyProvider.Metadata(meta.serialize());
assertEquals(meta.getCipher(), second.getCipher());
assertEquals(meta.getBitLength(), second.getBitLength());
assertEquals(meta.getDescription(), second.getDescription());
+ assertEquals(meta.getAttributes(), second.getAttributes());
assertEquals(meta.getCreated(), second.getCreated());
assertEquals(meta.getVersions(), second.getVersions());
newVersion = second.addVersion();
@@ -116,15 +124,19 @@
Configuration conf = new Configuration();
conf.set(KeyProvider.DEFAULT_CIPHER_NAME, "myCipher");
conf.setInt(KeyProvider.DEFAULT_BITLENGTH_NAME, 512);
+ Map<String, String> attributes = new HashMap<String, String>();
+ attributes.put("a", "A");
KeyProvider.Options options = KeyProvider.options(conf);
assertEquals("myCipher", options.getCipher());
assertEquals(512, options.getBitLength());
options.setCipher("yourCipher");
options.setDescription("description");
+ options.setAttributes(attributes);
options.setBitLength(128);
assertEquals("yourCipher", options.getCipher());
assertEquals(128, options.getBitLength());
assertEquals("description", options.getDescription());
+ assertEquals(attributes, options.getAttributes());
options = KeyProvider.options(new Configuration());
assertEquals(KeyProvider.DEFAULT_CIPHER, options.getCipher());
assertEquals(KeyProvider.DEFAULT_BITLENGTH, options.getBitLength());
@@ -133,13 +145,13 @@
@Test
public void testUnnestUri() throws Exception {
assertEquals(new Path("hdfs://nn.example.com/my/path"),
- KeyProvider.unnestUri(new URI("myscheme://hdfs@nn.example.com/my/path")));
+ ProviderUtils.unnestUri(new URI("myscheme://hdfs@nn.example.com/my/path")));
assertEquals(new Path("hdfs://nn/my/path?foo=bar&baz=bat#yyy"),
- KeyProvider.unnestUri(new URI("myscheme://hdfs@nn/my/path?foo=bar&baz=bat#yyy")));
+ ProviderUtils.unnestUri(new URI("myscheme://hdfs@nn/my/path?foo=bar&baz=bat#yyy")));
assertEquals(new Path("inner://hdfs@nn1.example.com/my/path"),
- KeyProvider.unnestUri(new URI("outer://inner@hdfs@nn1.example.com/my/path")));
+ ProviderUtils.unnestUri(new URI("outer://inner@hdfs@nn1.example.com/my/path")));
assertEquals(new Path("user:///"),
- KeyProvider.unnestUri(new URI("outer://user/")));
+ ProviderUtils.unnestUri(new URI("outer://user/")));
}
private static class MyKeyProvider extends KeyProvider {
@@ -166,7 +178,7 @@
@Override
public Metadata getMetadata(String name) throws IOException {
- return new Metadata(CIPHER, 128, "description", new Date(), 0);
+ return new Metadata(CIPHER, 128, "description", null, new Date(), 0);
}
@Override
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
index 89c5aeb..d33247b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java
@@ -31,6 +31,7 @@
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Assert;
import org.junit.Before;
@@ -213,7 +214,7 @@
file.delete();
conf.set(KeyProviderFactory.KEY_PROVIDER_PATH, ourUrl);
checkSpecificProvider(conf, ourUrl);
- Path path = KeyProvider.unnestUri(new URI(ourUrl));
+ Path path = ProviderUtils.unnestUri(new URI(ourUrl));
FileSystem fs = path.getFileSystem(conf);
FileStatus s = fs.getFileStatus(path);
assertTrue(s.getPermission().toString().equals("rwx------"));
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java
index b41ea09..ab89bb8 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/impl/TestGraphiteMetrics.java
@@ -22,6 +22,7 @@
import java.io.IOException;
import java.io.OutputStreamWriter;
+import java.io.Writer;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -30,6 +31,7 @@
import static org.mockito.Mockito.*;
import org.apache.hadoop.metrics2.AbstractMetric;
+import org.apache.hadoop.metrics2.MetricsException;
import org.apache.hadoop.metrics2.MetricsRecord;
import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.hadoop.metrics2.sink.GraphiteSink;
@@ -107,4 +109,39 @@
result.equals("null.all.Context.Context=all.foo2 2 10\n" +
"null.all.Context.Context=all.foo1 1 10\n"));
}
+ @Test(expected=MetricsException.class)
+ public void testCloseAndWrite() throws IOException {
+ GraphiteSink sink = new GraphiteSink();
+ List<MetricsTag> tags = new ArrayList<MetricsTag>();
+ tags.add(new MetricsTag(MsInfo.Context, "all"));
+ tags.add(new MetricsTag(MsInfo.Hostname, "host"));
+ Set<AbstractMetric> metrics = new HashSet<AbstractMetric>();
+ metrics.add(makeMetric("foo1", 1.25));
+ metrics.add(makeMetric("foo2", 2.25));
+ MetricsRecord record = new MetricsRecordImpl(MsInfo.Context, (long) 10000, tags, metrics);
+
+ OutputStreamWriter writer = mock(OutputStreamWriter.class);
+
+ sink.setWriter(writer);
+ sink.close();
+ sink.putMetrics(record);
+ }
+
+ @Test
+ public void testClose(){
+ GraphiteSink sink = new GraphiteSink();
+ Writer mockWriter = mock(Writer.class);
+ sink.setWriter(mockWriter);
+ try {
+ sink.close();
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+
+ try {
+ verify(mockWriter).close();
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+ }
}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
index 1fc91c3..3cb3384 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
@@ -19,18 +19,25 @@
package org.apache.hadoop.metrics2.source;
import org.junit.Test;
+
import static org.mockito.Mockito.*;
import static org.apache.hadoop.test.MetricsAsserts.*;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.MetricsCollector;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.util.JvmPauseMonitor;
+
import static org.apache.hadoop.metrics2.source.JvmMetricsInfo.*;
import static org.apache.hadoop.metrics2.impl.MsInfo.*;
public class TestJvmMetrics {
@Test public void testPresence() {
- MetricsRecordBuilder rb = getMetrics(new JvmMetrics("test", "test"));
+ JvmPauseMonitor pauseMonitor = new JvmPauseMonitor(new Configuration());
+ JvmMetrics jvmMetrics = new JvmMetrics("test", "test");
+ jvmMetrics.setPauseMonitor(pauseMonitor);
+ MetricsRecordBuilder rb = getMetrics(jvmMetrics);
MetricsCollector mc = rb.parent();
verify(mc).addRecord(JvmMetrics);
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
new file mode 100644
index 0000000..34758be
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.security.alias;
+
+import static org.junit.Assert.*;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.alias.CredentialShell.PasswordReader;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestCredShell {
+ private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+ private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
+ private static final File tmpDir =
+ new File(System.getProperty("test.build.data", "/tmp"), "creds");
+
+ @Before
+ public void setup() throws Exception {
+ System.setOut(new PrintStream(outContent));
+ System.setErr(new PrintStream(errContent));
+ }
+
+ @Test
+ public void testCredentialSuccessfulLifecycle() throws Exception {
+ outContent.reset();
+ String[] args1 = {"create", "credential1", "--value", "p@ssw0rd", "--provider",
+ "jceks://file" + tmpDir + "/credstore.jceks"};
+ int rc = 0;
+ CredentialShell cs = new CredentialShell();
+ cs.setConf(new Configuration());
+ rc = cs.run(args1);
+ assertEquals(outContent.toString(), 0, rc);
+ assertTrue(outContent.toString().contains("credential1 has been successfully " +
+ "created."));
+
+ outContent.reset();
+ String[] args2 = {"list", "--provider",
+ "jceks://file" + tmpDir + "/credstore.jceks"};
+ rc = cs.run(args2);
+ assertEquals(0, rc);
+ assertTrue(outContent.toString().contains("credential1"));
+
+ outContent.reset();
+ String[] args4 = {"delete", "credential1", "--provider",
+ "jceks://file" + tmpDir + "/credstore.jceks"};
+ rc = cs.run(args4);
+ assertEquals(0, rc);
+ assertTrue(outContent.toString().contains("credential1 has been successfully " +
+ "deleted."));
+
+ outContent.reset();
+ String[] args5 = {"list", "--provider",
+ "jceks://file" + tmpDir + "/credstore.jceks"};
+ rc = cs.run(args5);
+ assertEquals(0, rc);
+ assertFalse(outContent.toString(), outContent.toString().contains("credential1"));
+ }
+
+ @Test
+ public void testInvalidProvider() throws Exception {
+ String[] args1 = {"create", "credential1", "--value", "p@ssw0rd", "--provider",
+ "sdff://file/tmp/credstore.jceks"};
+
+ int rc = 0;
+ CredentialShell cs = new CredentialShell();
+ cs.setConf(new Configuration());
+ rc = cs.run(args1);
+ assertEquals(-1, rc);
+ assertTrue(outContent.toString().contains("There are no valid " +
+ "CredentialProviders configured."));
+ }
+
+ @Test
+ public void testTransientProviderWarning() throws Exception {
+ String[] args1 = {"create", "credential1", "--value", "p@ssw0rd", "--provider",
+ "user:///"};
+
+ int rc = 0;
+ CredentialShell cs = new CredentialShell();
+ cs.setConf(new Configuration());
+ rc = cs.run(args1);
+ assertEquals(outContent.toString(), 0, rc);
+ assertTrue(outContent.toString().contains("WARNING: you are modifying a " +
+ "transient provider."));
+
+ String[] args2 = {"delete", "credential1", "--provider", "user:///"};
+ rc = cs.run(args2);
+ assertEquals(outContent.toString(), 0, rc);
+ assertTrue(outContent.toString().contains("credential1 has been successfully " +
+ "deleted."));
+ }
+
+ @Test
+ public void testTransientProviderOnlyConfig() throws Exception {
+ String[] args1 = {"create", "credential1"};
+
+ int rc = 0;
+ CredentialShell cs = new CredentialShell();
+ Configuration config = new Configuration();
+ config.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, "user:///");
+ cs.setConf(config);
+ rc = cs.run(args1);
+ assertEquals(-1, rc);
+ assertTrue(outContent.toString().contains("There are no valid " +
+ "CredentialProviders configured."));
+ }
+
+ @Test
+ public void testPromptForCredential() throws Exception {
+ String[] args1 = {"create", "credential1", "--provider",
+ "jceks://file" + tmpDir + "/credstore.jceks"};
+ ArrayList<String> passwords = new ArrayList<String>();
+ passwords.add("p@ssw0rd");
+ passwords.add("p@ssw0rd");
+ int rc = 0;
+ CredentialShell shell = new CredentialShell();
+ shell.setConf(new Configuration());
+ shell.setPasswordReader(new MockPasswordReader(passwords));
+ rc = shell.run(args1);
+ assertEquals(0, rc);
+ assertTrue(outContent.toString().contains("credential1 has been successfully " +
+ "created."));
+
+ String[] args2 = {"delete", "credential1", "--provider",
+ "jceks://file" + tmpDir + "/credstore.jceks"};
+ rc = shell.run(args2);
+ assertEquals(0, rc);
+ assertTrue(outContent.toString().contains("credential1 has been successfully " +
+ "deleted."));
+ }
+
+ public class MockPasswordReader extends CredentialShell.PasswordReader {
+ List<String> passwords = null;
+
+ public MockPasswordReader(List<String> passwds) {
+ passwords = passwds;
+ }
+
+ @Override
+ public char[] readPassword(String prompt) {
+ if (passwords.size() == 0) return null;
+ String pass = passwords.remove(0);
+ return pass.toCharArray();
+ }
+
+ @Override
+ public void format(String message) {
+ System.out.println(message);
+ }
+ }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProvider.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProvider.java
new file mode 100644
index 0000000..0d83974
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProvider.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.security.alias;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.ProviderUtils;
+import org.junit.Test;
+
+import java.net.URI;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertArrayEquals;
+
+public class TestCredentialProvider {
+
+ @Test
+ public void testCredentialEntry() throws Exception {
+ char[] key1 = new char[]{1,2,3,4};
+ CredentialProvider.CredentialEntry obj =
+ new CredentialProvider.CredentialEntry("cred1", key1);
+ assertEquals("cred1", obj.getAlias());
+ assertArrayEquals(new char[]{1,2,3,4}, obj.getCredential());
+ }
+
+ @Test
+ public void testUnnestUri() throws Exception {
+ assertEquals(new Path("hdfs://nn.example.com/my/path"),
+ ProviderUtils.unnestUri(new URI("myscheme://hdfs@nn.example.com/my/path")));
+ assertEquals(new Path("hdfs://nn/my/path?foo=bar&baz=bat#yyy"),
+ ProviderUtils.unnestUri(new URI("myscheme://hdfs@nn/my/path?foo=bar&baz=bat#yyy")));
+ assertEquals(new Path("inner://hdfs@nn1.example.com/my/path"),
+ ProviderUtils.unnestUri(new URI("outer://inner@hdfs@nn1.example.com/my/path")));
+ assertEquals(new Path("user:///"),
+ ProviderUtils.unnestUri(new URI("outer://user/")));
+ }
+}
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
new file mode 100644
index 0000000..3d8794f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
@@ -0,0 +1,234 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.security.alias;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.ProviderUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class TestCredentialProviderFactory {
+
+ private static char[] chars = { 'a', 'b', 'c', 'd', 'e', 'f', 'g',
+ 'h', 'j', 'k', 'm', 'n', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w',
+ 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K',
+ 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+ '2', '3', '4', '5', '6', '7', '8', '9',};
+
+ private static final File tmpDir =
+ new File(System.getProperty("test.build.data", "/tmp"), "creds");
+
+ @Test
+ public void testFactory() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ UserProvider.SCHEME_NAME + ":///," +
+ JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks");
+ List<CredentialProvider> providers =
+ CredentialProviderFactory.getProviders(conf);
+ assertEquals(2, providers.size());
+ assertEquals(UserProvider.class, providers.get(0).getClass());
+ assertEquals(JavaKeyStoreProvider.class, providers.get(1).getClass());
+ assertEquals(UserProvider.SCHEME_NAME +
+ ":///", providers.get(0).toString());
+ assertEquals(JavaKeyStoreProvider.SCHEME_NAME +
+ "://file" + tmpDir + "/test.jks",
+ providers.get(1).toString());
+ }
+
+ @Test
+ public void testFactoryErrors() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, "unknown:///");
+ try {
+ List<CredentialProvider> providers =
+ CredentialProviderFactory.getProviders(conf);
+ assertTrue("should throw!", false);
+ } catch (IOException e) {
+ assertEquals("No CredentialProviderFactory for unknown:/// in " +
+ CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
+ e.getMessage());
+ }
+ }
+
+ @Test
+ public void testUriErrors() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, "unkn@own:/x/y");
+ try {
+ List<CredentialProvider> providers =
+ CredentialProviderFactory.getProviders(conf);
+ assertTrue("should throw!", false);
+ } catch (IOException e) {
+ assertEquals("Bad configuration of " +
+ CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH +
+ " at unkn@own:/x/y", e.getMessage());
+ }
+ }
+
+ private static char[] generatePassword(int length) {
+ StringBuffer sb = new StringBuffer();
+ Random r = new Random();
+ for (int i = 0; i < length; i++) {
+ sb.append(chars[r.nextInt(chars.length)]);
+ }
+ return sb.toString().toCharArray();
+ }
+
+ static void checkSpecificProvider(Configuration conf,
+ String ourUrl) throws Exception {
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+ char[] passwd = generatePassword(16);
+
+ // ensure that we get nulls when the key isn't there
+ assertEquals(null, provider.getCredentialEntry("no-such-key"));
+ assertEquals(null, provider.getCredentialEntry("key"));
+ // create a new key
+ try {
+ provider.createCredentialEntry("pass", passwd);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ // make sure we get back the right key
+ assertArrayEquals(passwd, provider.getCredentialEntry("pass").getCredential());
+ // try recreating pass
+ try {
+ provider.createCredentialEntry("pass", passwd);
+ assertTrue("should throw", false);
+ } catch (IOException e) {
+ assertEquals("Credential pass already exists in " + ourUrl, e.getMessage());
+ }
+ provider.deleteCredentialEntry("pass");
+ try {
+ provider.deleteCredentialEntry("pass");
+ assertTrue("should throw", false);
+ } catch (IOException e) {
+ assertEquals("Credential pass does not exist in " + ourUrl, e.getMessage());
+ }
+ char[] passTwo = new char[]{'1', '2', '3'};
+ provider.createCredentialEntry("pass", passwd);
+ provider.createCredentialEntry("pass2", passTwo);
+ assertArrayEquals(passTwo,
+ provider.getCredentialEntry("pass2").getCredential());
+
+ // write them to disk so that configuration.getPassword will find them
+ provider.flush();
+
+ // configuration.getPassword should get this from provider
+ assertArrayEquals(passTwo, conf.getPassword("pass2"));
+
+ // configuration.getPassword should get this from config
+ conf.set("onetwothree", "123");
+ assertArrayEquals(passTwo, conf.getPassword("onetwothree"));
+
+ // configuration.getPassword should NOT get this from config since
+ // we are disabling the fallback to clear text config
+ conf.set(CredentialProvider.CLEAR_TEXT_FALLBACK, "false");
+ assertArrayEquals(null, conf.getPassword("onetwothree"));
+
+ // get a new instance of the provider to ensure it was saved correctly
+ provider = CredentialProviderFactory.getProviders(conf).get(0);
+ assertTrue(provider != null);
+ assertArrayEquals(new char[]{'1', '2', '3'},
+ provider.getCredentialEntry("pass2").getCredential());
+ assertArrayEquals(passwd, provider.getCredentialEntry("pass").getCredential());
+
+ List<String> creds = provider.getAliases();
+ assertTrue("Credentials should have been returned.", creds.size() == 2);
+ assertTrue("Returned Credentials should have included pass.", creds.contains("pass"));
+ assertTrue("Returned Credentials should have included pass2.", creds.contains("pass2"));
+ }
+
+ @Test
+ public void testUserProvider() throws Exception {
+ Configuration conf = new Configuration();
+ final String ourUrl = UserProvider.SCHEME_NAME + ":///";
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
+ checkSpecificProvider(conf, ourUrl);
+ // see if the credentials are actually in the UGI
+ Credentials credentials =
+ UserGroupInformation.getCurrentUser().getCredentials();
+ assertArrayEquals(new byte[]{'1', '2', '3'},
+ credentials.getSecretKey(new Text("pass2")));
+ }
+
+ @Test
+ public void testJksProvider() throws Exception {
+ Configuration conf = new Configuration();
+ final String ourUrl =
+ JavaKeyStoreProvider.SCHEME_NAME + "://file" + tmpDir + "/test.jks";
+
+ File file = new File(tmpDir, "test.jks");
+ file.delete();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
+ checkSpecificProvider(conf, ourUrl);
+ Path path = ProviderUtils.unnestUri(new URI(ourUrl));
+ FileSystem fs = path.getFileSystem(conf);
+ FileStatus s = fs.getFileStatus(path);
+ assertTrue(s.getPermission().toString().equals("rwx------"));
+ assertTrue(file + " should exist", file.isFile());
+
+ // check permission retention after explicit change
+ fs.setPermission(path, new FsPermission("777"));
+ checkPermissionRetention(conf, ourUrl, path);
+ }
+
+ public void checkPermissionRetention(Configuration conf, String ourUrl,
+ Path path) throws Exception {
+ CredentialProvider provider = CredentialProviderFactory.getProviders(conf).get(0);
+ // let's add a new credential and flush and check that permissions are still set to 777
+ char[] cred = new char[32];
+ for(int i =0; i < cred.length; ++i) {
+ cred[i] = (char) i;
+ }
+ // create a new key
+ try {
+ provider.createCredentialEntry("key5", cred);
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ provider.flush();
+ // get a new instance of the provider to ensure it was saved correctly
+ provider = CredentialProviderFactory.getProviders(conf).get(0);
+ assertArrayEquals(cred, provider.getCredentialEntry("key5").getCredential());
+
+ FileSystem fs = path.getFileSystem(conf);
+ FileStatus s = fs.getFileStatus(path);
+ assertTrue("Permissions should have been retained from the preexisting " +
+ "keystore.", s.getPermission().toString().equals("rwxrwxrwx"));
+ }
+}
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
index de8d844..3446c78 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
@@ -103,6 +103,7 @@
@Path(KMSRESTConstants.KEYS_RESOURCE)
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
+ @SuppressWarnings("unchecked")
public Response createKey(@Context SecurityContext securityContext,
Map jsonKey) throws Exception {
KMSWebApp.getAdminCallsMeter().mark();
@@ -116,7 +117,8 @@
? (Integer) jsonKey.get(KMSRESTConstants.LENGTH_FIELD) : 0;
String description = (String)
jsonKey.get(KMSRESTConstants.DESCRIPTION_FIELD);
-
+ Map<String, String> attributes = (Map<String, String>)
+ jsonKey.get(KMSRESTConstants.ATTRIBUTES_FIELD);
if (material != null) {
assertAccess(KMSACLs.Type.SET_KEY_MATERIAL, user,
CREATE_KEY + " with user provided material", name);
@@ -130,6 +132,7 @@
options.setBitLength(length);
}
options.setDescription(description);
+ options.setAttributes(attributes);
KeyProvider.KeyVersion keyVersion = (material != null)
? provider.createKey(name, Base64.decodeBase64(material), options)
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
index cc995cd..9131a18 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
@@ -61,6 +61,7 @@
json.put(KMSRESTConstants.CIPHER_FIELD, meta.getCipher());
json.put(KMSRESTConstants.LENGTH_FIELD, meta.getBitLength());
json.put(KMSRESTConstants.DESCRIPTION_FIELD, meta.getDescription());
+ json.put(KMSRESTConstants.ATTRIBUTES_FIELD, meta.getAttributes());
json.put(KMSRESTConstants.CREATED_FIELD,
meta.getCreated().getTime());
json.put(KMSRESTConstants.VERSIONS_FIELD,
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 70aa5989..0959dce 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -490,6 +490,49 @@
// getKeysMetadata() empty
Assert.assertEquals(0, kp.getKeysMetadata().length);
+ // createKey() no description, no tags
+ options = new KeyProvider.Options(conf);
+ options.setCipher("AES/CTR/NoPadding");
+ options.setBitLength(128);
+ kp.createKey("k2", options);
+ KeyProvider.Metadata meta = kp.getMetadata("k2");
+ Assert.assertNull(meta.getDescription());
+ Assert.assertTrue(meta.getAttributes().isEmpty());
+
+ // createKey() description, no tags
+ options = new KeyProvider.Options(conf);
+ options.setCipher("AES/CTR/NoPadding");
+ options.setBitLength(128);
+ options.setDescription("d");
+ kp.createKey("k3", options);
+ meta = kp.getMetadata("k3");
+ Assert.assertEquals("d", meta.getDescription());
+ Assert.assertTrue(meta.getAttributes().isEmpty());
+
+ Map<String, String> attributes = new HashMap<String, String>();
+ attributes.put("a", "A");
+
+ // createKey() no description, tags
+ options = new KeyProvider.Options(conf);
+ options.setCipher("AES/CTR/NoPadding");
+ options.setBitLength(128);
+ options.setAttributes(attributes);
+ kp.createKey("k4", options);
+ meta = kp.getMetadata("k4");
+ Assert.assertNull(meta.getDescription());
+ Assert.assertEquals(attributes, meta.getAttributes());
+
+ // createKey() description, tags
+ options = new KeyProvider.Options(conf);
+ options.setCipher("AES/CTR/NoPadding");
+ options.setBitLength(128);
+ options.setDescription("d");
+ options.setAttributes(attributes);
+ kp.createKey("k5", options);
+ meta = kp.getMetadata("k5");
+ Assert.assertEquals("d", meta.getDescription());
+ Assert.assertEquals(attributes, meta.getAttributes());
+
return null;
}
});
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSCacheKeyProvider.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSCacheKeyProvider.java
index 110b0c9..72b2191 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSCacheKeyProvider.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSCacheKeyProvider.java
@@ -102,7 +102,7 @@
Mockito.when(mockProv.getCurrentKey(Mockito.eq("k1"))).thenReturn(mockKey);
Mockito.when(mockProv.getKeyVersion(Mockito.eq("k1@0"))).thenReturn(mockKey);
Mockito.when(mockProv.getMetadata(Mockito.eq("k1"))).thenReturn(
- new KMSClientProvider.KMSMetadata("c", 0, "l", new Date(), 1));
+ new KMSClientProvider.KMSMetadata("c", 0, "l", null, new Date(), 1));
KeyProvider cache = new KMSCacheKeyProvider(mockProv, 100);
Assert.assertEquals(mockKey, cache.getCurrentKey("k1"));
Mockito.verify(mockProv, Mockito.times(1)).getCurrentKey(Mockito.eq("k1"));
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Interface.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Interface.java
index 24aa8f0..9bca9e7 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Interface.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/nfs/nfs3/Nfs3Interface.java
@@ -17,12 +17,9 @@
*/
package org.apache.hadoop.nfs.nfs3;
-import java.net.InetAddress;
-
import org.apache.hadoop.nfs.nfs3.response.NFS3Response;
+import org.apache.hadoop.oncrpc.RpcInfo;
import org.apache.hadoop.oncrpc.XDR;
-import org.apache.hadoop.oncrpc.security.SecurityHandler;
-import org.jboss.netty.channel.Channel;
/**
* RPC procedures as defined in RFC 1813.
@@ -33,70 +30,65 @@
public NFS3Response nullProcedure();
/** GETATTR: Get file attributes */
- public NFS3Response getattr(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response getattr(XDR xdr, RpcInfo info);
/** SETATTR: Set file attributes */
- public NFS3Response setattr(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response setattr(XDR xdr, RpcInfo info);
/** LOOKUP: Lookup filename */
- public NFS3Response lookup(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response lookup(XDR xdr, RpcInfo info);
/** ACCESS: Check access permission */
- public NFS3Response access(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response access(XDR xdr, RpcInfo info);
+
+ /** READLINK: Read from symbolic link */
+ public NFS3Response readlink(XDR xdr, RpcInfo info);
/** READ: Read from file */
- public NFS3Response read(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response read(XDR xdr, RpcInfo info);
/** WRITE: Write to file */
- public NFS3Response write(XDR xdr, Channel channel, int xid,
- SecurityHandler securityHandler, InetAddress client);
+ public NFS3Response write(XDR xdr, RpcInfo info);
/** CREATE: Create a file */
- public NFS3Response create(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response create(XDR xdr, RpcInfo info);
/** MKDIR: Create a directory */
- public NFS3Response mkdir(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
-
- /** REMOVE: Remove a file */
- public NFS3Response remove(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
-
- /** RMDIR: Remove a directory */
- public NFS3Response rmdir(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
-
- /** RENAME: Rename a file or directory */
- public NFS3Response rename(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response mkdir(XDR xdr, RpcInfo info);
/** SYMLINK: Create a symbolic link */
- public NFS3Response symlink(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response symlink(XDR xdr, RpcInfo info);
+
+ /** MKNOD: Create a special device */
+ public NFS3Response mknod(XDR xdr, RpcInfo info);
+
+ /** REMOVE: Remove a file */
+ public NFS3Response remove(XDR xdr, RpcInfo info);
+
+ /** RMDIR: Remove a directory */
+ public NFS3Response rmdir(XDR xdr, RpcInfo info);
+
+ /** RENAME: Rename a file or directory */
+ public NFS3Response rename(XDR xdr, RpcInfo info);
+
+ /** LINK: create link to an object */
+ public NFS3Response link(XDR xdr, RpcInfo info);
/** READDIR: Read From directory */
- public NFS3Response readdir(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response readdir(XDR xdr, RpcInfo info);
+ /** READDIRPLUS: Extended read from directory */
+ public NFS3Response readdirplus(XDR xdr, RpcInfo info);
+
/** FSSTAT: Get dynamic file system information */
- public NFS3Response fsstat(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response fsstat(XDR xdr, RpcInfo info);
/** FSINFO: Get static file system information */
- public NFS3Response fsinfo(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response fsinfo(XDR xdr, RpcInfo info);
/** PATHCONF: Retrieve POSIX information */
- public NFS3Response pathconf(XDR xdr, SecurityHandler securityHandler,
- InetAddress client);
+ public NFS3Response pathconf(XDR xdr, RpcInfo info);
/** COMMIT: Commit cached data on a server to stable storage */
- public NFS3Response commit(XDR xdr, Channel channel, int xid,
- SecurityHandler securityHandler, InetAddress client);
+ public NFS3Response commit(XDR xdr, RpcInfo info);
}
diff --git a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java
index d828f88..89e7173 100644
--- a/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java
+++ b/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/RpcProgram.java
@@ -48,7 +48,7 @@
private final int progNumber;
private final int lowProgVersion;
private final int highProgVersion;
- private final boolean allowInsecurePorts;
+ protected final boolean allowInsecurePorts;
/**
* If not null, this will be used as the socket to use to connect to the
@@ -146,31 +146,6 @@
RpcCall call = (RpcCall) info.header();
SocketAddress remoteAddress = info.remoteAddress();
- if (!allowInsecurePorts) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Will not allow connections from unprivileged ports. " +
- "Checking for valid client port...");
- }
- if (remoteAddress instanceof InetSocketAddress) {
- InetSocketAddress inetRemoteAddress = (InetSocketAddress) remoteAddress;
- if (inetRemoteAddress.getPort() > 1023) {
- LOG.warn("Connection attempted from '" + inetRemoteAddress + "' "
- + "which is an unprivileged port. Rejecting connection.");
- sendRejectedReply(call, remoteAddress, ctx);
- return;
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Accepting connection from '" + remoteAddress + "'");
- }
- }
- } else {
- LOG.warn("Could not determine remote port of socket address '" +
- remoteAddress + "'. Rejecting connection.");
- sendRejectedReply(call, remoteAddress, ctx);
- return;
- }
- }
-
if (LOG.isTraceEnabled()) {
LOG.trace(program + " procedure #" + call.getProcedure());
}
@@ -191,6 +166,29 @@
handleInternal(ctx, info);
}
+ public boolean doPortMonitoring(SocketAddress remoteAddress) {
+ if (!allowInsecurePorts) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Will not allow connections from unprivileged ports. "
+ + "Checking for valid client port...");
+ }
+
+ if (remoteAddress instanceof InetSocketAddress) {
+ InetSocketAddress inetRemoteAddress = (InetSocketAddress) remoteAddress;
+ if (inetRemoteAddress.getPort() > 1023) {
+ LOG.warn("Connection attempted from '" + inetRemoteAddress + "' "
+ + "which is an unprivileged port. Rejecting connection.");
+ return false;
+ }
+ } else {
+ LOG.warn("Could not determine remote port of socket address '"
+ + remoteAddress + "'. Rejecting connection.");
+ return false;
+ }
+ }
+ return true;
+ }
+
private void sendAcceptedReply(RpcCall call, SocketAddress remoteAddress,
AcceptState acceptState, ChannelHandlerContext ctx) {
RpcAcceptedReply reply = RpcAcceptedReply.getInstance(call.getXid(),
@@ -208,7 +206,7 @@
RpcUtil.sendRpcResponse(ctx, rsp);
}
- private static void sendRejectedReply(RpcCall call,
+ protected static void sendRejectedReply(RpcCall call,
SocketAddress remoteAddress, ChannelHandlerContext ctx) {
XDR out = new XDR();
RpcDeniedReply reply = new RpcDeniedReply(call.getXid(),
diff --git a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java
index 9c649bc..9d0fe0f 100644
--- a/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java
+++ b/hadoop-common-project/hadoop-nfs/src/test/java/org/apache/hadoop/oncrpc/TestFrameDecoder.java
@@ -66,6 +66,18 @@
@Override
protected void handleInternal(ChannelHandlerContext ctx, RpcInfo info) {
+ // This is just like what's done in RpcProgramMountd#handleInternal and
+ // RpcProgramNfs3#handleInternal.
+ RpcCall rpcCall = (RpcCall) info.header();
+ final int procedure = rpcCall.getProcedure();
+ if (procedure != 0) {
+ boolean portMonitorSuccess = doPortMonitoring(info.remoteAddress());
+ if (!portMonitorSuccess) {
+ sendRejectedReply(rpcCall, info.remoteAddress(), ctx);
+ return;
+ }
+ }
+
resultSize = info.data().readableBytes();
RpcAcceptedReply reply = RpcAcceptedReply.getAcceptInstance(1234,
new VerifierNone());
@@ -190,6 +202,20 @@
// Verify the server rejected the request.
assertEquals(0, resultSize);
+
+ // Ensure that the NULL procedure does in fact succeed.
+ xdrOut = new XDR();
+ createPortmapXDRheader(xdrOut, 0);
+ int headerSize = xdrOut.size();
+ buffer = new byte[bufsize];
+ xdrOut.writeFixedOpaque(buffer);
+ int requestSize = xdrOut.size() - headerSize;
+
+ // Send the request to the server
+ testRequest(xdrOut, serverPort);
+
+ // Verify the server did not reject the request.
+ assertEquals(requestSize, resultSize);
}
private static int startRpcServer(boolean allowInsecurePorts) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java
index 616d5121..2f65ce4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfigKeys.java
@@ -51,7 +51,10 @@
public static final String DFS_NFS_KEYTAB_FILE_KEY = "nfs.keytab.file";
public static final String DFS_NFS_KERBEROS_PRINCIPAL_KEY = "nfs.kerberos.principal";
public static final String DFS_NFS_REGISTRATION_PORT_KEY = "nfs.registration.port";
- public static final int DFS_NFS_REGISTRATION_PORT_DEFAULT = 40; // Currently unassigned.
- public static final String DFS_NFS_ALLOW_INSECURE_PORTS_KEY = "nfs.allow.insecure.ports";
- public static final boolean DFS_NFS_ALLOW_INSECURE_PORTS_DEFAULT = true;
-}
\ No newline at end of file
+ public static final int DFS_NFS_REGISTRATION_PORT_DEFAULT = 40; // Currently unassigned.
+ public static final String DFS_NFS_PORT_MONITORING_DISABLED_KEY = "nfs.port.monitoring.disabled";
+ public static final boolean DFS_NFS_PORT_MONITORING_DISABLED_DEFAULT = true;
+
+ public static final String AIX_COMPAT_MODE_KEY = "nfs.aix.compatibility.mode.enabled";
+ public static final boolean AIX_COMPAT_MODE_DEFAULT = false;
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfiguration.java
index c831c14..ff92794 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfiguration.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/conf/NfsConfiguration.java
@@ -36,6 +36,8 @@
NfsConfigKeys.DFS_NFS_SERVER_PORT_KEY),
new DeprecationDelta("nfs3.mountd.port",
NfsConfigKeys.DFS_NFS_MOUNTD_PORT_KEY),
+ new DeprecationDelta("dfs.nfs.exports.cache.size",
+ Nfs3Constant.NFS_EXPORTS_CACHE_SIZE_KEY),
new DeprecationDelta("dfs.nfs.exports.cache.expirytime.millis",
Nfs3Constant.NFS_EXPORTS_CACHE_EXPIRYTIME_MILLIS_KEY),
new DeprecationDelta("hadoop.nfs.userupdate.milly",
@@ -49,6 +51,18 @@
new DeprecationDelta("dfs.nfs3.stream.timeout",
NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_KEY),
new DeprecationDelta("dfs.nfs3.export.point",
- NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY) });
+ NfsConfigKeys.DFS_NFS_EXPORT_POINT_KEY),
+ new DeprecationDelta("nfs.allow.insecure.ports",
+ NfsConfigKeys.DFS_NFS_PORT_MONITORING_DISABLED_KEY),
+ new DeprecationDelta("dfs.nfs.keytab.file",
+ NfsConfigKeys.DFS_NFS_KEYTAB_FILE_KEY),
+ new DeprecationDelta("dfs.nfs.kerberos.principal",
+ NfsConfigKeys.DFS_NFS_KERBEROS_PRINCIPAL_KEY),
+ new DeprecationDelta("dfs.nfs.rtmax",
+ NfsConfigKeys.DFS_NFS_MAX_READ_TRANSFER_SIZE_KEY),
+ new DeprecationDelta("dfs.nfs.wtmax",
+ NfsConfigKeys.DFS_NFS_MAX_WRITE_TRANSFER_SIZE_KEY),
+ new DeprecationDelta("dfs.nfs.dtmax",
+ NfsConfigKeys.DFS_NFS_MAX_READDIR_TRANSFER_SIZE_KEY) });
}
}
\ No newline at end of file
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java
index b89fc03..9fbab24 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/mount/RpcProgramMountd.java
@@ -194,7 +194,13 @@
if (mntproc == MNTPROC.NULL) {
out = nullOp(out, xid, client);
} else if (mntproc == MNTPROC.MNT) {
- out = mnt(xdr, out, xid, client);
+ // Only do port monitoring for MNT
+ if (!doPortMonitoring(info.remoteAddress())) {
+ out = MountResponse.writeMNTResponse(Nfs3Status.NFS3ERR_ACCES, out,
+ xid, null);
+ } else {
+ out = mnt(xdr, out, xid, client);
+ }
} else if (mntproc == MNTPROC.DUMP) {
out = dump(out, xid, client);
} else if (mntproc == MNTPROC.UMNT) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java
index 0eb641a..3daf7bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/Nfs3.java
@@ -61,8 +61,8 @@
StringUtils.startupShutdownMessage(Nfs3.class, args, LOG);
NfsConfiguration conf = new NfsConfiguration();
boolean allowInsecurePorts = conf.getBoolean(
- NfsConfigKeys.DFS_NFS_ALLOW_INSECURE_PORTS_KEY,
- NfsConfigKeys.DFS_NFS_ALLOW_INSECURE_PORTS_DEFAULT);
+ NfsConfigKeys.DFS_NFS_PORT_MONITORING_DISABLED_KEY,
+ NfsConfigKeys.DFS_NFS_PORT_MONITORING_DISABLED_DEFAULT);
final Nfs3 nfsServer = new Nfs3(conf, registrationSocket,
allowInsecurePorts);
nfsServer.startServiceInternal(true);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
index e2ab317..cf44af5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/OpenFileCtx.java
@@ -95,6 +95,7 @@
*/
private AtomicLong nextOffset;
private final HdfsDataOutputStream fos;
+ private final boolean aixCompatMode;
// It's updated after each sync to HDFS
private Nfs3FileAttributes latestAttr;
@@ -199,8 +200,15 @@
OpenFileCtx(HdfsDataOutputStream fos, Nfs3FileAttributes latestAttr,
String dumpFilePath, DFSClient client, IdUserGroup iug) {
+ this(fos, latestAttr, dumpFilePath, client, iug, false);
+ }
+
+ OpenFileCtx(HdfsDataOutputStream fos, Nfs3FileAttributes latestAttr,
+ String dumpFilePath, DFSClient client, IdUserGroup iug,
+ boolean aixCompatMode) {
this.fos = fos;
this.latestAttr = latestAttr;
+ this.aixCompatMode = aixCompatMode;
// We use the ReverseComparatorOnMin as the comparator of the map. In this
// way, we first dump the data with larger offset. In the meanwhile, we
// retrieve the last element to write back to HDFS.
@@ -780,15 +788,29 @@
}
if (commitOffset > 0) {
- if (commitOffset > flushed) {
- if (!fromRead) {
- CommitCtx commitCtx = new CommitCtx(commitOffset, channel, xid,
- preOpAttr);
- pendingCommits.put(commitOffset, commitCtx);
+ if (aixCompatMode) {
+ // The AIX NFS client misinterprets RFC-1813 and will always send 4096
+ // for the commitOffset even if fewer bytes than that have ever (or will
+ // ever) be sent by the client. So, if in AIX compatibility mode, we
+ // will always DO_SYNC if the number of bytes to commit have already all
+ // been flushed, else we will fall through to the logic below which
+ // checks for pending writes in the case that we're being asked to
+ // commit more bytes than have so far been flushed. See HDFS-6549 for
+ // more info.
+ if (commitOffset <= flushed) {
+ return COMMIT_STATUS.COMMIT_DO_SYNC;
}
- return COMMIT_STATUS.COMMIT_WAIT;
} else {
- return COMMIT_STATUS.COMMIT_DO_SYNC;
+ if (commitOffset > flushed) {
+ if (!fromRead) {
+ CommitCtx commitCtx = new CommitCtx(commitOffset, channel, xid,
+ preOpAttr);
+ pendingCommits.put(commitOffset, commitCtx);
+ }
+ return COMMIT_STATUS.COMMIT_WAIT;
+ } else {
+ return COMMIT_STATUS.COMMIT_DO_SYNC;
+ }
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
index 7fd92d1..446e722 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java
@@ -23,6 +23,7 @@
import java.net.DatagramSocket;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.net.SocketAddress;
import java.nio.ByteBuffer;
import java.util.EnumSet;
@@ -152,6 +153,7 @@
private final short replication;
private final long blockSize;
private final int bufferSize;
+ private final boolean aixCompatMode;
private Statistics statistics;
private String writeDumpDir; // The dir save dump files
@@ -169,8 +171,11 @@
config.set(FsPermission.UMASK_LABEL, "000");
iug = new IdUserGroup(config);
+ aixCompatMode = config.getBoolean(
+ NfsConfigKeys.AIX_COMPAT_MODE_KEY,
+ NfsConfigKeys.AIX_COMPAT_MODE_DEFAULT);
exports = NfsExports.getInstance(config);
- writeManager = new WriteManager(iug, config);
+ writeManager = new WriteManager(iug, config, aixCompatMode);
clientCache = new DFSClientCache(config);
replication = (short) config.getInt(DFSConfigKeys.DFS_REPLICATION_KEY,
DFSConfigKeys.DFS_REPLICATION_DEFAULT);
@@ -230,15 +235,15 @@
}
@Override
- public GETATTR3Response getattr(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ public GETATTR3Response getattr(XDR xdr, RpcInfo info) {
GETATTR3Response response = new GETATTR3Response(Nfs3Status.NFS3_OK);
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_ONLY)) {
+ if (!checkAccessPrivilege(info, AccessPrivilege.READ_ONLY)) {
response.setStatus(Nfs3Status.NFS3ERR_ACCES);
return response;
}
+ SecurityHandler securityHandler = getSecurityHandler(info);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
@@ -322,9 +327,9 @@
}
@Override
- public SETATTR3Response setattr(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ public SETATTR3Response setattr(XDR xdr, RpcInfo info) {
SETATTR3Response response = new SETATTR3Response(Nfs3Status.NFS3_OK);
+ SecurityHandler securityHandler = getSecurityHandler(info);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
@@ -370,7 +375,7 @@
}
// check the write access privilege
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_WRITE)) {
+ if (!checkAccessPrivilege(info, AccessPrivilege.READ_WRITE)) {
return new SETATTR3Response(Nfs3Status.NFS3ERR_ACCES, new WccData(
preOpWcc, preOpAttr));
}
@@ -398,15 +403,15 @@
}
@Override
- public LOOKUP3Response lookup(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ public LOOKUP3Response lookup(XDR xdr, RpcInfo info) {
LOOKUP3Response response = new LOOKUP3Response(Nfs3Status.NFS3_OK);
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_ONLY)) {
+ if (!checkAccessPrivilege(info, AccessPrivilege.READ_ONLY)) {
response.setStatus(Nfs3Status.NFS3ERR_ACCES);
return response;
}
+ SecurityHandler securityHandler = getSecurityHandler(info);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
@@ -460,15 +465,15 @@
}
@Override
- public ACCESS3Response access(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ public ACCESS3Response access(XDR xdr, RpcInfo info) {
ACCESS3Response response = new ACCESS3Response(Nfs3Status.NFS3_OK);
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_ONLY)) {
+ if (!checkAccessPrivilege(info, AccessPrivilege.READ_ONLY)) {
response.setStatus(Nfs3Status.NFS3ERR_ACCES);
return response;
}
+ SecurityHandler securityHandler = getSecurityHandler(info);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
@@ -519,15 +524,16 @@
}
}
- public READLINK3Response readlink(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ @Override
+ public READLINK3Response readlink(XDR xdr, RpcInfo info) {
READLINK3Response response = new READLINK3Response(Nfs3Status.NFS3_OK);
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_ONLY)) {
+ if (!checkAccessPrivilege(info, AccessPrivilege.READ_ONLY)) {
response.setStatus(Nfs3Status.NFS3ERR_ACCES);
return response;
}
+ SecurityHandler securityHandler = getSecurityHandler(info);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
@@ -591,12 +597,19 @@
}
@Override
- public READ3Response read(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ public READ3Response read(XDR xdr, RpcInfo info) {
+ SecurityHandler securityHandler = getSecurityHandler(info);
+ SocketAddress remoteAddress = info.remoteAddress();
+ return read(xdr, securityHandler, remoteAddress);
+ }
+
+ @VisibleForTesting
+ READ3Response read(XDR xdr, SecurityHandler securityHandler,
+ SocketAddress remoteAddress) {
READ3Response response = new READ3Response(Nfs3Status.NFS3_OK);
final String userName = securityHandler.getUser();
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_ONLY)) {
+ if (!checkAccessPrivilege(remoteAddress, AccessPrivilege.READ_ONLY)) {
response.setStatus(Nfs3Status.NFS3ERR_ACCES);
return response;
}
@@ -715,8 +728,17 @@
}
@Override
- public WRITE3Response write(XDR xdr, Channel channel, int xid,
- SecurityHandler securityHandler, InetAddress client) {
+ public WRITE3Response write(XDR xdr, RpcInfo info) {
+ SecurityHandler securityHandler = getSecurityHandler(info);
+ RpcCall rpcCall = (RpcCall) info.header();
+ int xid = rpcCall.getXid();
+ SocketAddress remoteAddress = info.remoteAddress();
+ return write(xdr, info.channel(), xid, securityHandler, remoteAddress);
+ }
+
+ @VisibleForTesting
+ WRITE3Response write(XDR xdr, Channel channel, int xid,
+ SecurityHandler securityHandler, SocketAddress remoteAddress) {
WRITE3Response response = new WRITE3Response(Nfs3Status.NFS3_OK);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
@@ -758,7 +780,7 @@
return new WRITE3Response(Nfs3Status.NFS3ERR_STALE);
}
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_WRITE)) {
+ if (!checkAccessPrivilege(remoteAddress, AccessPrivilege.READ_WRITE)) {
return new WRITE3Response(Nfs3Status.NFS3ERR_ACCES, new WccData(
Nfs3Utils.getWccAttr(preOpAttr), preOpAttr), 0, stableHow,
Nfs3Constant.WRITE_COMMIT_VERF);
@@ -791,8 +813,15 @@
}
@Override
- public CREATE3Response create(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ public CREATE3Response create(XDR xdr, RpcInfo info) {
+ SecurityHandler securityHandler = getSecurityHandler(info);
+ SocketAddress remoteAddress = info.remoteAddress();
+ return create(xdr, securityHandler, remoteAddress);
+ }
+
+ @VisibleForTesting
+ CREATE3Response create(XDR xdr, SecurityHandler securityHandler,
+ SocketAddress remoteAddress) {
CREATE3Response response = new CREATE3Response(Nfs3Status.NFS3_OK);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
@@ -838,7 +867,7 @@
return new CREATE3Response(Nfs3Status.NFS3ERR_STALE);
}
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_WRITE)) {
+ if (!checkAccessPrivilege(remoteAddress, AccessPrivilege.READ_WRITE)) {
return new CREATE3Response(Nfs3Status.NFS3ERR_ACCES, null,
preOpDirAttr, new WccData(Nfs3Utils.getWccAttr(preOpDirAttr),
preOpDirAttr));
@@ -875,7 +904,8 @@
// Add open stream
OpenFileCtx openFileCtx = new OpenFileCtx(fos, postOpObjAttr,
- writeDumpDir + "/" + postOpObjAttr.getFileId(), dfsClient, iug);
+ writeDumpDir + "/" + postOpObjAttr.getFileId(), dfsClient, iug,
+ aixCompatMode);
fileHandle = new FileHandle(postOpObjAttr.getFileId());
if (!writeManager.addOpenFileStream(fileHandle, openFileCtx)) {
LOG.warn("Can't add more stream, close it."
@@ -922,9 +952,9 @@
}
@Override
- public MKDIR3Response mkdir(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ public MKDIR3Response mkdir(XDR xdr, RpcInfo info) {
MKDIR3Response response = new MKDIR3Response(Nfs3Status.NFS3_OK);
+ SecurityHandler securityHandler = getSecurityHandler(info);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
@@ -960,7 +990,7 @@
return new MKDIR3Response(Nfs3Status.NFS3ERR_STALE);
}
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_WRITE)) {
+ if (!checkAccessPrivilege(info, AccessPrivilege.READ_WRITE)) {
return new MKDIR3Response(Nfs3Status.NFS3ERR_ACCES, null, preOpDirAttr,
new WccData(Nfs3Utils.getWccAttr(preOpDirAttr), preOpDirAttr));
}
@@ -1012,15 +1042,15 @@
}
}
- public READDIR3Response mknod(XDR xdr,
- SecurityHandler securityHandler, InetAddress client) {
+ @Override
+ public READDIR3Response mknod(XDR xdr, RpcInfo info) {
return new READDIR3Response(Nfs3Status.NFS3ERR_NOTSUPP);
}
@Override
- public REMOVE3Response remove(XDR xdr,
- SecurityHandler securityHandler, InetAddress client) {
+ public REMOVE3Response remove(XDR xdr, RpcInfo info) {
REMOVE3Response response = new REMOVE3Response(Nfs3Status.NFS3_OK);
+ SecurityHandler securityHandler = getSecurityHandler(info);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
@@ -1093,9 +1123,9 @@
}
@Override
- public RMDIR3Response rmdir(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ public RMDIR3Response rmdir(XDR xdr, RpcInfo info) {
RMDIR3Response response = new RMDIR3Response(Nfs3Status.NFS3_OK);
+ SecurityHandler securityHandler = getSecurityHandler(info);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
@@ -1129,7 +1159,7 @@
WccData errWcc = new WccData(Nfs3Utils.getWccAttr(preOpDirAttr),
preOpDirAttr);
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_WRITE)) {
+ if (!checkAccessPrivilege(info, AccessPrivilege.READ_WRITE)) {
return new RMDIR3Response(Nfs3Status.NFS3ERR_ACCES, errWcc);
}
@@ -1175,9 +1205,9 @@
}
@Override
- public RENAME3Response rename(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ public RENAME3Response rename(XDR xdr, RpcInfo info) {
RENAME3Response response = new RENAME3Response(Nfs3Status.NFS3_OK);
+ SecurityHandler securityHandler = getSecurityHandler(info);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
@@ -1221,7 +1251,7 @@
return new RENAME3Response(Nfs3Status.NFS3ERR_STALE);
}
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_WRITE)) {
+ if (!checkAccessPrivilege(info, AccessPrivilege.READ_WRITE)) {
WccData fromWcc = new WccData(Nfs3Utils.getWccAttr(fromPreOpAttr),
fromPreOpAttr);
WccData toWcc = new WccData(Nfs3Utils.getWccAttr(toPreOpAttr),
@@ -1263,15 +1293,15 @@
}
@Override
- public SYMLINK3Response symlink(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ public SYMLINK3Response symlink(XDR xdr, RpcInfo info) {
SYMLINK3Response response = new SYMLINK3Response(Nfs3Status.NFS3_OK);
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_WRITE)) {
+ if (!checkAccessPrivilege(info, AccessPrivilege.READ_WRITE)) {
response.setStatus(Nfs3Status.NFS3ERR_ACCES);
return response;
}
+ SecurityHandler securityHandler = getSecurityHandler(info);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
@@ -1322,8 +1352,8 @@
}
}
- public READDIR3Response link(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ @Override
+ public READDIR3Response link(XDR xdr, RpcInfo info) {
return new READDIR3Response(Nfs3Status.NFS3ERR_NOTSUPP);
}
@@ -1351,11 +1381,16 @@
}
@Override
+ public READDIR3Response readdir(XDR xdr, RpcInfo info) {
+ SecurityHandler securityHandler = getSecurityHandler(info);
+ SocketAddress remoteAddress = info.remoteAddress();
+ return readdir(xdr, securityHandler, remoteAddress);
+ }
public READDIR3Response readdir(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ SocketAddress remoteAddress) {
READDIR3Response response = new READDIR3Response(Nfs3Status.NFS3_OK);
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_ONLY)) {
+ if (!checkAccessPrivilege(remoteAddress, AccessPrivilege.READ_ONLY)) {
response.setStatus(Nfs3Status.NFS3ERR_ACCES);
return response;
}
@@ -1408,9 +1443,24 @@
}
long cookieVerf = request.getCookieVerf();
if ((cookieVerf != 0) && (cookieVerf != dirStatus.getModificationTime())) {
- LOG.error("CookierVerf mismatch. request cookierVerf:" + cookieVerf
- + " dir cookieVerf:" + dirStatus.getModificationTime());
- return new READDIR3Response(Nfs3Status.NFS3ERR_BAD_COOKIE);
+ if (aixCompatMode) {
+ // The AIX NFS client misinterprets RFC-1813 and will repeatedly send
+ // the same cookieverf value even across VFS-level readdir calls,
+ // instead of getting a new cookieverf for every VFS-level readdir
+ // call, and reusing the cookieverf only in the event that multiple
+ // incremental NFS-level readdir calls must be made to fetch all of
+ // the directory entries. This means that whenever a readdir call is
+ // made by an AIX NFS client for a given directory, and that directory
+ // is subsequently modified, thus changing its mtime, no later readdir
+ // calls will succeed from AIX for that directory until the FS is
+ // unmounted/remounted. See HDFS-6549 for more info.
+ LOG.warn("AIX compatibility mode enabled, ignoring cookieverf " +
+ "mismatches.");
+ } else {
+ LOG.error("CookieVerf mismatch. request cookieVerf: " + cookieVerf
+ + " dir cookieVerf: " + dirStatus.getModificationTime());
+ return new READDIR3Response(Nfs3Status.NFS3ERR_BAD_COOKIE);
+ }
}
if (cookie == 0) {
@@ -1491,9 +1541,17 @@
dirStatus.getModificationTime(), dirList);
}
- public READDIRPLUS3Response readdirplus(XDR xdr,
- SecurityHandler securityHandler, InetAddress client) {
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_ONLY)) {
+ @Override
+ public READDIRPLUS3Response readdirplus(XDR xdr, RpcInfo info) {
+ SecurityHandler securityHandler = getSecurityHandler(info);
+ SocketAddress remoteAddress = info.remoteAddress();
+ return readdirplus(xdr, securityHandler, remoteAddress);
+ }
+
+ @VisibleForTesting
+ READDIRPLUS3Response readdirplus(XDR xdr, SecurityHandler securityHandler,
+ SocketAddress remoteAddress) {
+ if (!checkAccessPrivilege(remoteAddress, AccessPrivilege.READ_ONLY)) {
return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_ACCES);
}
@@ -1550,9 +1608,22 @@
}
long cookieVerf = request.getCookieVerf();
if ((cookieVerf != 0) && (cookieVerf != dirStatus.getModificationTime())) {
- LOG.error("CookierVerf mismatch. request cookierVerf:" + cookieVerf
- + " dir cookieVerf:" + dirStatus.getModificationTime());
- return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_BAD_COOKIE);
+ if (aixCompatMode) {
+ // The AIX NFS client misinterprets RFC-1813 and will repeatedly send
+ // the same cookieverf value even across VFS-level readdir calls,
+ // instead of getting a new cookieverf for every VFS-level readdir
+ // call. This means that whenever a readdir call is made by an AIX NFS
+ // client for a given directory, and that directory is subsequently
+ // modified, thus changing its mtime, no later readdir calls will
+ // succeed for that directory from AIX until the FS is
+ // unmounted/remounted. See HDFS-6549 for more info.
+ LOG.warn("AIX compatibility mode enabled, ignoring cookieverf " +
+ "mismatches.");
+ } else {
+ LOG.error("cookieverf mismatch. request cookieverf: " + cookieVerf
+ + " dir cookieverf: " + dirStatus.getModificationTime());
+ return new READDIRPLUS3Response(Nfs3Status.NFS3ERR_BAD_COOKIE);
+ }
}
if (cookie == 0) {
@@ -1643,15 +1714,15 @@
}
@Override
- public FSSTAT3Response fsstat(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ public FSSTAT3Response fsstat(XDR xdr, RpcInfo info) {
FSSTAT3Response response = new FSSTAT3Response(Nfs3Status.NFS3_OK);
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_ONLY)) {
+ if (!checkAccessPrivilege(info, AccessPrivilege.READ_ONLY)) {
response.setStatus(Nfs3Status.NFS3ERR_ACCES);
return response;
}
+ SecurityHandler securityHandler = getSecurityHandler(info);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
@@ -1711,15 +1782,15 @@
}
@Override
- public FSINFO3Response fsinfo(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ public FSINFO3Response fsinfo(XDR xdr, RpcInfo info) {
FSINFO3Response response = new FSINFO3Response(Nfs3Status.NFS3_OK);
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_ONLY)) {
+ if (!checkAccessPrivilege(info, AccessPrivilege.READ_ONLY)) {
response.setStatus(Nfs3Status.NFS3ERR_ACCES);
return response;
}
+ SecurityHandler securityHandler = getSecurityHandler(info);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
@@ -1769,15 +1840,15 @@
}
@Override
- public PATHCONF3Response pathconf(XDR xdr, SecurityHandler securityHandler,
- InetAddress client) {
+ public PATHCONF3Response pathconf(XDR xdr, RpcInfo info) {
PATHCONF3Response response = new PATHCONF3Response(Nfs3Status.NFS3_OK);
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_ONLY)) {
+ if (!checkAccessPrivilege(info, AccessPrivilege.READ_ONLY)) {
response.setStatus(Nfs3Status.NFS3ERR_ACCES);
return response;
}
+ SecurityHandler securityHandler = getSecurityHandler(info);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
@@ -1816,9 +1887,11 @@
}
@Override
- public COMMIT3Response commit(XDR xdr, Channel channel, int xid,
- SecurityHandler securityHandler, InetAddress client) {
+ public COMMIT3Response commit(XDR xdr, RpcInfo info) {
+ //Channel channel, int xid,
+ // SecurityHandler securityHandler, InetAddress client) {
COMMIT3Response response = new COMMIT3Response(Nfs3Status.NFS3_OK);
+ SecurityHandler securityHandler = getSecurityHandler(info);
DFSClient dfsClient = clientCache.getDfsClient(securityHandler.getUser());
if (dfsClient == null) {
response.setStatus(Nfs3Status.NFS3ERR_SERVERFAULT);
@@ -1849,7 +1922,7 @@
return new COMMIT3Response(Nfs3Status.NFS3ERR_STALE);
}
- if (!checkAccessPrivilege(client, AccessPrivilege.READ_WRITE)) {
+ if (!checkAccessPrivilege(info, AccessPrivilege.READ_WRITE)) {
return new COMMIT3Response(Nfs3Status.NFS3ERR_ACCES, new WccData(
Nfs3Utils.getWccAttr(preOpAttr), preOpAttr),
Nfs3Constant.WRITE_COMMIT_VERF);
@@ -1859,8 +1932,10 @@
: (request.getOffset() + request.getCount());
// Insert commit as an async request
- writeManager.handleCommit(dfsClient, handle, commitOffset, channel, xid,
- preOpAttr);
+ RpcCall rpcCall = (RpcCall) info.header();
+ int xid = rpcCall.getXid();
+ writeManager.handleCommit(dfsClient, handle, commitOffset,
+ info.channel(), xid, preOpAttr);
return null;
} catch (IOException e) {
LOG.warn("Exception ", e);
@@ -1885,11 +1960,16 @@
return null;
}
}
+
+ private SecurityHandler getSecurityHandler(RpcInfo info) {
+ RpcCall rpcCall = (RpcCall) info.header();
+ return getSecurityHandler(rpcCall.getCredential(), rpcCall.getVerifier());
+ }
@Override
public void handleInternal(ChannelHandlerContext ctx, RpcInfo info) {
RpcCall rpcCall = (RpcCall) info.header();
- final NFSPROC3 nfsproc3 = NFSPROC3.fromValue(rpcCall.getProcedure());
+ final NFSPROC3 nfsproc3 = NFSPROC3.fromValue(rpcCall.getProcedure());
int xid = rpcCall.getXid();
byte[] data = new byte[info.data().readableBytes()];
info.data().readBytes(data);
@@ -1897,9 +1977,8 @@
XDR out = new XDR();
InetAddress client = ((InetSocketAddress) info.remoteAddress())
.getAddress();
- Channel channel = info.channel();
-
Credentials credentials = rpcCall.getCredential();
+
// Ignore auth only for NFSPROC3_NULL, especially for Linux clients.
if (nfsproc3 != NFSPROC3.NULL) {
if (credentials.getFlavor() != AuthFlavor.AUTH_SYS
@@ -1937,27 +2016,24 @@
}
}
- SecurityHandler securityHandler = getSecurityHandler(credentials,
- rpcCall.getVerifier());
-
NFS3Response response = null;
if (nfsproc3 == NFSPROC3.NULL) {
response = nullProcedure();
} else if (nfsproc3 == NFSPROC3.GETATTR) {
- response = getattr(xdr, securityHandler, client);
+ response = getattr(xdr, info);
} else if (nfsproc3 == NFSPROC3.SETATTR) {
- response = setattr(xdr, securityHandler, client);
+ response = setattr(xdr, info);
} else if (nfsproc3 == NFSPROC3.LOOKUP) {
- response = lookup(xdr, securityHandler, client);
+ response = lookup(xdr, info);
} else if (nfsproc3 == NFSPROC3.ACCESS) {
- response = access(xdr, securityHandler, client);
+ response = access(xdr, info);
} else if (nfsproc3 == NFSPROC3.READLINK) {
- response = readlink(xdr, securityHandler, client);
+ response = readlink(xdr, info);
} else if (nfsproc3 == NFSPROC3.READ) {
if (LOG.isDebugEnabled()) {
LOG.debug(Nfs3Utils.READ_RPC_START + xid);
}
- response = read(xdr, securityHandler, client);
+ response = read(xdr, info);
if (LOG.isDebugEnabled() && (nfsproc3 == NFSPROC3.READ)) {
LOG.debug(Nfs3Utils.READ_RPC_END + xid);
}
@@ -1965,36 +2041,36 @@
if (LOG.isDebugEnabled()) {
LOG.debug(Nfs3Utils.WRITE_RPC_START + xid);
}
- response = write(xdr, channel, xid, securityHandler, client);
+ response = write(xdr, info);
// Write end debug trace is in Nfs3Utils.writeChannel
} else if (nfsproc3 == NFSPROC3.CREATE) {
- response = create(xdr, securityHandler, client);
+ response = create(xdr, info);
} else if (nfsproc3 == NFSPROC3.MKDIR) {
- response = mkdir(xdr, securityHandler, client);
+ response = mkdir(xdr, info);
} else if (nfsproc3 == NFSPROC3.SYMLINK) {
- response = symlink(xdr, securityHandler, client);
+ response = symlink(xdr, info);
} else if (nfsproc3 == NFSPROC3.MKNOD) {
- response = mknod(xdr, securityHandler, client);
+ response = mknod(xdr, info);
} else if (nfsproc3 == NFSPROC3.REMOVE) {
- response = remove(xdr, securityHandler, client);
+ response = remove(xdr, info);
} else if (nfsproc3 == NFSPROC3.RMDIR) {
- response = rmdir(xdr, securityHandler, client);
+ response = rmdir(xdr, info);
} else if (nfsproc3 == NFSPROC3.RENAME) {
- response = rename(xdr, securityHandler, client);
+ response = rename(xdr, info);
} else if (nfsproc3 == NFSPROC3.LINK) {
- response = link(xdr, securityHandler, client);
+ response = link(xdr, info);
} else if (nfsproc3 == NFSPROC3.READDIR) {
- response = readdir(xdr, securityHandler, client);
+ response = readdir(xdr, info);
} else if (nfsproc3 == NFSPROC3.READDIRPLUS) {
- response = readdirplus(xdr, securityHandler, client);
+ response = readdirplus(xdr, info);
} else if (nfsproc3 == NFSPROC3.FSSTAT) {
- response = fsstat(xdr, securityHandler, client);
+ response = fsstat(xdr, info);
} else if (nfsproc3 == NFSPROC3.FSINFO) {
- response = fsinfo(xdr, securityHandler, client);
+ response = fsinfo(xdr, info);
} else if (nfsproc3 == NFSPROC3.PATHCONF) {
- response = pathconf(xdr, securityHandler, client);
+ response = pathconf(xdr,info);
} else if (nfsproc3 == NFSPROC3.COMMIT) {
- response = commit(xdr, channel, xid, securityHandler, client);
+ response = commit(xdr, info);
} else {
// Invalid procedure
RpcAcceptedReply.getInstance(xid,
@@ -2027,8 +2103,21 @@
return nfsproc3 == null || nfsproc3.isIdempotent();
}
- private boolean checkAccessPrivilege(final InetAddress client,
+ private boolean checkAccessPrivilege(RpcInfo info,
final AccessPrivilege expected) {
+ SocketAddress remoteAddress = info.remoteAddress();
+ return checkAccessPrivilege(remoteAddress, expected);
+ }
+
+ private boolean checkAccessPrivilege(SocketAddress remoteAddress,
+ final AccessPrivilege expected) {
+ // Port monitoring
+ if (!doPortMonitoring(remoteAddress)) {
+ return false;
+ }
+
+ // Check export table
+ InetAddress client = ((InetSocketAddress) remoteAddress).getAddress();
AccessPrivilege access = exports.getAccessPrivilege(client);
if (access == AccessPrivilege.NONE) {
return false;
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
index 2cd8b22..5f2ded7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/main/java/org/apache/hadoop/hdfs/nfs/nfs3/WriteManager.java
@@ -58,6 +58,7 @@
private boolean asyncDataServiceStarted = false;
private final int maxStreams;
+ private final boolean aixCompatMode;
/**
* The time limit to wait for accumulate reordered sequential writes to the
@@ -79,9 +80,11 @@
return fileContextCache.put(h, ctx);
}
- WriteManager(IdUserGroup iug, final NfsConfiguration config) {
+ WriteManager(IdUserGroup iug, final NfsConfiguration config,
+ boolean aixCompatMode) {
this.iug = iug;
this.config = config;
+ this.aixCompatMode = aixCompatMode;
streamTimeout = config.getLong(NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_KEY,
NfsConfigKeys.DFS_NFS_STREAM_TIMEOUT_DEFAULT);
LOG.info("Stream timeout is " + streamTimeout + "ms.");
@@ -175,7 +178,7 @@
String writeDumpDir = config.get(NfsConfigKeys.DFS_NFS_FILE_DUMP_DIR_KEY,
NfsConfigKeys.DFS_NFS_FILE_DUMP_DIR_DEFAULT);
openFileCtx = new OpenFileCtx(fos, latestAttr, writeDumpDir + "/"
- + fileHandle.getFileId(), dfsClient, iug);
+ + fileHandle.getFileId(), dfsClient, iug, aixCompatMode);
if (!addOpenFileStream(fileHandle, openFileCtx)) {
LOG.info("Can't add new stream. Close it. Tell client to retry.");
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestReaddir.java
similarity index 90%
rename from hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java
rename to hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestReaddir.java
index 33c89e9..617c31d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/TestReaddir.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestReaddir.java
@@ -16,12 +16,14 @@
* limitations under the License.
*/
-package org.apache.hadoop.hdfs.nfs;
+package org.apache.hadoop.hdfs.nfs.nfs3;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
import java.util.List;
import org.apache.hadoop.fs.Path;
@@ -38,10 +40,15 @@
import org.apache.hadoop.nfs.nfs3.response.READDIR3Response.Entry3;
import org.apache.hadoop.nfs.nfs3.response.READDIRPLUS3Response;
import org.apache.hadoop.nfs.nfs3.response.READDIRPLUS3Response.EntryPlus3;
+import org.apache.hadoop.oncrpc.RpcInfo;
+import org.apache.hadoop.oncrpc.RpcMessage;
import org.apache.hadoop.oncrpc.XDR;
import org.apache.hadoop.oncrpc.security.SecurityHandler;
import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelHandlerContext;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -122,7 +129,7 @@
xdr_req.writeInt(100); // count
READDIR3Response response = nfsd.readdir(xdr_req.asReadOnlyWrap(),
- securityHandler, InetAddress.getLocalHost());
+ securityHandler, new InetSocketAddress("localhost", 1234));
List<Entry3> dirents = response.getDirList().getEntries();
assertTrue(dirents.size() == 5); // inculding dot, dotdot
@@ -139,7 +146,7 @@
xdr_req.writeInt(100); // count
response = nfsd.readdir(xdr_req.asReadOnlyWrap(), securityHandler,
- InetAddress.getLocalHost());
+ new InetSocketAddress("localhost", 1234));
dirents = response.getDirList().getEntries();
assertTrue(dirents.size() == 1);
Entry3 entry = dirents.get(0);
@@ -149,7 +156,7 @@
hdfs.delete(new Path(testdir + "/f2"), false);
response = nfsd.readdir(xdr_req.asReadOnlyWrap(), securityHandler,
- InetAddress.getLocalHost());
+ new InetSocketAddress("localhost", 1234));
dirents = response.getDirList().getEntries();
assertTrue(dirents.size() == 2); // No dot, dotdot
}
@@ -170,8 +177,9 @@
xdr_req.writeInt(100); // dirCount
xdr_req.writeInt(1000); // maxCount
- READDIRPLUS3Response responsePlus = nfsd.readdirplus(
- xdr_req.asReadOnlyWrap(), securityHandler, InetAddress.getLocalHost());
+ READDIRPLUS3Response responsePlus = nfsd.readdirplus(xdr_req
+ .asReadOnlyWrap(), securityHandler, new InetSocketAddress("localhost",
+ 1234));
List<EntryPlus3> direntPlus = responsePlus.getDirListPlus().getEntries();
assertTrue(direntPlus.size() == 5); // including dot, dotdot
@@ -189,7 +197,7 @@
xdr_req.writeInt(1000); // maxCount
responsePlus = nfsd.readdirplus(xdr_req.asReadOnlyWrap(), securityHandler,
- InetAddress.getLocalHost());
+ new InetSocketAddress("localhost", 1234));
direntPlus = responsePlus.getDirListPlus().getEntries();
assertTrue(direntPlus.size() == 1);
EntryPlus3 entryPlus = direntPlus.get(0);
@@ -199,7 +207,7 @@
hdfs.delete(new Path(testdir + "/f2"), false);
responsePlus = nfsd.readdirplus(xdr_req.asReadOnlyWrap(), securityHandler,
- InetAddress.getLocalHost());
+ new InetSocketAddress("localhost", 1234));
direntPlus = responsePlus.getDirListPlus().getEntries();
assertTrue(direntPlus.size() == 2); // No dot, dotdot
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
index 674265d..3945b29 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestWrites.java
@@ -22,7 +22,7 @@
import static org.junit.Assert.fail;
import java.io.IOException;
-import java.net.InetAddress;
+import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.concurrent.ConcurrentNavigableMap;
@@ -190,6 +190,29 @@
ret = ctx.checkCommit(dfsClient, 0, ch, 1, attr, false);
Assert.assertTrue(ret == COMMIT_STATUS.COMMIT_FINISHED);
}
+
+ @Test
+ public void testCheckCommitAixCompatMode() throws IOException {
+ DFSClient dfsClient = Mockito.mock(DFSClient.class);
+ Nfs3FileAttributes attr = new Nfs3FileAttributes();
+ HdfsDataOutputStream fos = Mockito.mock(HdfsDataOutputStream.class);
+
+ // Last argument "true" here to enable AIX compatibility mode.
+ OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient,
+ new IdUserGroup(new NfsConfiguration()), true);
+
+ // Test fall-through to pendingWrites check in the event that commitOffset
+ // is greater than the number of bytes we've so far flushed.
+ Mockito.when(fos.getPos()).thenReturn((long) 2);
+ COMMIT_STATUS status = ctx.checkCommitInternal(5, null, 1, attr, false);
+ Assert.assertTrue(status == COMMIT_STATUS.COMMIT_FINISHED);
+
+ // Test the case when we actually have received more bytes than we're trying
+ // to commit.
+ Mockito.when(fos.getPos()).thenReturn((long) 10);
+ status = ctx.checkCommitInternal(5, null, 1, attr, false);
+ Assert.assertTrue(status == COMMIT_STATUS.COMMIT_DO_SYNC);
+ }
@Test
// Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS, which
@@ -207,7 +230,7 @@
FileHandle h = new FileHandle(1); // fake handle for "/dumpFilePath"
COMMIT_STATUS ret;
- WriteManager wm = new WriteManager(new IdUserGroup(config), config);
+ WriteManager wm = new WriteManager(new IdUserGroup(config), config, false);
assertTrue(wm.addOpenFileStream(h, ctx));
// Test inactive open file context
@@ -318,7 +341,7 @@
XDR createXdr = new XDR();
createReq.serialize(createXdr);
CREATE3Response createRsp = nfsd.create(createXdr.asReadOnlyWrap(),
- securityHandler, InetAddress.getLocalHost());
+ securityHandler, new InetSocketAddress("localhost", 1234));
FileHandle handle = createRsp.getObjHandle();
// Test DATA_SYNC
@@ -331,7 +354,7 @@
XDR writeXdr = new XDR();
writeReq.serialize(writeXdr);
nfsd.write(writeXdr.asReadOnlyWrap(), null, 1, securityHandler,
- InetAddress.getLocalHost());
+ new InetSocketAddress("localhost", 1234));
waitWrite(nfsd, handle, 60000);
@@ -340,7 +363,7 @@
XDR readXdr = new XDR();
readReq.serialize(readXdr);
READ3Response readRsp = nfsd.read(readXdr.asReadOnlyWrap(),
- securityHandler, InetAddress.getLocalHost());
+ securityHandler, new InetSocketAddress("localhost", 1234));
assertTrue(Arrays.equals(buffer, readRsp.getData().array()));
@@ -352,7 +375,7 @@
XDR createXdr2 = new XDR();
createReq2.serialize(createXdr2);
CREATE3Response createRsp2 = nfsd.create(createXdr2.asReadOnlyWrap(),
- securityHandler, InetAddress.getLocalHost());
+ securityHandler, new InetSocketAddress("localhost", 1234));
FileHandle handle2 = createRsp2.getObjHandle();
WRITE3Request writeReq2 = new WRITE3Request(handle2, 0, 10,
@@ -360,7 +383,7 @@
XDR writeXdr2 = new XDR();
writeReq2.serialize(writeXdr2);
nfsd.write(writeXdr2.asReadOnlyWrap(), null, 1, securityHandler,
- InetAddress.getLocalHost());
+ new InetSocketAddress("localhost", 1234));
waitWrite(nfsd, handle2, 60000);
@@ -369,7 +392,7 @@
XDR readXdr2 = new XDR();
readReq2.serialize(readXdr2);
READ3Response readRsp2 = nfsd.read(readXdr2.asReadOnlyWrap(),
- securityHandler, InetAddress.getLocalHost());
+ securityHandler, new InetSocketAddress("localhost", 1234));
assertTrue(Arrays.equals(buffer, readRsp2.getData().array()));
// FILE_SYNC should sync the file size
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 93adae6..dae38c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -254,9 +254,6 @@
HDFS-5794. Fix the inconsistency of layout version number of
ADD_DATANODE_AND_STORAGE_UUIDS between trunk and branch-2. (jing9)
- HDFS-6375. Listing extended attributes with the search permission.
- (Charles Lamb via wang)
-
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -452,6 +449,13 @@
HDFS-6545. Finalizing rolling upgrade can make NN unavailable for a long
duration. (kihwal)
+ HDFS-6530. Fix Balancer documentation. (szetszwo)
+
+ HDFS-6480. Move waitForReady() from FSDirectory to FSNamesystem. (wheat9)
+
+ HDFS-6403. Add metrics for log warnings reported by JVM pauses. (Yongjun
+ Zhang via atm)
+
OPTIMIZATIONS
HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
@@ -651,6 +655,30 @@
HDFS-6527. Edit log corruption due to defered INode removal. (kihwal and
jing9 via jing9)
+ HDFS-6552. add DN storage to a BlockInfo will not replace the different
+ storage from same DN. (Amir Langer via Arpit Agarwal)
+
+ HDFS-6551. Rename with OVERWRITE option may throw NPE when the target
+ file/directory is a reference INode. (jing9)
+
+ HDFS-6439. NFS should not reject NFS requests to the NULL procedure whether
+ port monitoring is enabled or not. (brandonli)
+
+ HDFS-6559. Fix wrong option "dfsadmin -rollingUpgrade start" in the
+ document. (Akira Ajisaka via Arpit Agarwal)
+
+ HDFS-6553. Add missing DeprecationDeltas for NFS Kerberos configurations
+ (Stephen Chu via brandonli)
+
+ HDFS-6563. NameNode cannot save fsimage in certain circumstances when
+ snapshots are in use. (atm)
+
+ HDFS-3848. A Bug in recoverLeaseInternal method of FSNameSystem class
+ (Hooman Peiro Sajjad and Chen He via kihwal)
+
+ HDFS-6549. Add support for accessing the NFS gateway from the AIX NFS
+ client. (atm)
+
BREAKDOWN OF HDFS-2006 SUBTASKS AND RELATED JIRAS
HDFS-6299. Protobuf for XAttr and client-side implementation. (Yi Liu via umamahesh)
@@ -714,6 +742,15 @@
HDFS-6374. setXAttr should require the user to be the owner of the file
or directory (Charles Lamb via wang)
+ HDFS-6375. Listing extended attributes with the search permission.
+ (Charles Lamb via wang)
+
+ HDFS-6492. Support create-time xattrs and atomically setting multiple
+ xattrs. (wang)
+
+ HDFS-6312. WebHdfs HA failover is broken on secure clusters.
+ (daryn via tucu)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java
index 35a7680..82272e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/XAttr.java
@@ -19,6 +19,8 @@
import java.util.Arrays;
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
/**
@@ -105,42 +107,47 @@
@Override
public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + ((name == null) ? 0 : name.hashCode());
- result = prime * result + ((ns == null) ? 0 : ns.hashCode());
- result = prime * result + Arrays.hashCode(value);
- return result;
+ return new HashCodeBuilder(811, 67)
+ .append(name)
+ .append(ns)
+ .append(value)
+ .toHashCode();
}
@Override
public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
+ if (obj == null) { return false; }
+ if (obj == this) { return true; }
+ if (obj.getClass() != getClass()) {
return false;
}
- if (getClass() != obj.getClass()) {
- return false;
- }
- XAttr other = (XAttr) obj;
- if (name == null) {
- if (other.name != null) {
- return false;
- }
- } else if (!name.equals(other.name)) {
- return false;
- }
- if (ns != other.ns) {
- return false;
- }
- if (!Arrays.equals(value, other.value)) {
- return false;
- }
- return true;
+ XAttr rhs = (XAttr) obj;
+ return new EqualsBuilder()
+ .append(ns, rhs.ns)
+ .append(name, rhs.name)
+ .append(value, rhs.value)
+ .isEquals();
}
-
+
+ /**
+ * Similar to {@link #equals(Object)}, except ignores the XAttr value.
+ *
+ * @param obj to compare equality
+ * @return if the XAttrs are equal, ignoring the XAttr value
+ */
+ public boolean equalsIgnoreValue(Object obj) {
+ if (obj == null) { return false; }
+ if (obj == this) { return true; }
+ if (obj.getClass() != getClass()) {
+ return false;
+ }
+ XAttr rhs = (XAttr) obj;
+ return new EqualsBuilder()
+ .append(ns, rhs.ns)
+ .append(name, rhs.name)
+ .isEquals();
+ }
+
@Override
public String toString() {
return "XAttr [ns=" + ns + ", name=" + name + ", value="
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index a897577..f3b62c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -2093,6 +2093,9 @@
public static List<XAttrProto> convertXAttrProto(
List<XAttr> xAttrSpec) {
+ if (xAttrSpec == null) {
+ return Lists.newArrayListWithCapacity(0);
+ }
ArrayList<XAttrProto> xAttrs = Lists.newArrayListWithCapacity(
xAttrSpec.size());
for (XAttr a : xAttrSpec) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
index 983f60b..c6650bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockInfo.java
@@ -203,7 +203,7 @@
} else {
// The block is on the DN but belongs to a different storage.
// Update our state.
- removeStorage(storage);
+ removeStorage(getStorageInfo(idx));
added = false; // Just updating storage. Return false.
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 292259a..5797c00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -778,7 +778,8 @@
initIpcServer(conf);
metrics = DataNodeMetrics.create(conf, getDisplayName());
-
+ metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
+
blockPoolManager = new BlockPoolManager(this);
blockPoolManager.refreshNamenodes(conf);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
index 9601bcf..b536e7e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
@@ -90,13 +90,15 @@
final MutableQuantiles[] sendDataPacketBlockedOnNetworkNanosQuantiles;
@Metric MutableRate sendDataPacketTransferNanos;
final MutableQuantiles[] sendDataPacketTransferNanosQuantiles;
-
final MetricsRegistry registry = new MetricsRegistry("datanode");
final String name;
-
- public DataNodeMetrics(String name, String sessionId, int[] intervals) {
+ JvmMetrics jvmMetrics = null;
+
+ public DataNodeMetrics(String name, String sessionId, int[] intervals,
+ final JvmMetrics jvmMetrics) {
this.name = name;
+ this.jvmMetrics = jvmMetrics;
registry.tag(SessionId, sessionId);
final int len = intervals.length;
@@ -131,7 +133,7 @@
public static DataNodeMetrics create(Configuration conf, String dnName) {
String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
MetricsSystem ms = DefaultMetricsSystem.instance();
- JvmMetrics.create("DataNode", sessionId, ms);
+ JvmMetrics jm = JvmMetrics.create("DataNode", sessionId, ms);
String name = "DataNodeActivity-"+ (dnName.isEmpty()
? "UndefinedDataNodeName"+ DFSUtil.getRandom().nextInt()
: dnName.replace(':', '-'));
@@ -141,11 +143,15 @@
conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
return ms.register(name, null, new DataNodeMetrics(name, sessionId,
- intervals));
+ intervals, jm));
}
public String name() { return name; }
+ public JvmMetrics getJvmMetrics() {
+ return jvmMetrics;
+ }
+
public void addHeartbeat(long latency) {
heartbeats.add(latency);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
index 62aefb9..9327f43 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
@@ -252,7 +252,7 @@
backupNode.namesystem.writeLock();
try {
- backupNode.namesystem.dir.setReady();
+ backupNode.namesystem.setImageLoaded();
if(backupNode.namesystem.getBlocksTotal() > 0) {
backupNode.namesystem.setBlockTotal();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 34a579f..da8d11e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -26,11 +26,11 @@
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Condition;
+import java.util.ListIterator;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileAlreadyExistsException;
@@ -44,7 +44,6 @@
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
-import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -84,15 +83,14 @@
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-/*************************************************
- * FSDirectory stores the filesystem directory state.
- * It handles writing/loading values to disk, and logging
- * changes as we go.
- *
- * It keeps the filename->blockset mapping always-current
- * and logged to disk.
- *
- *************************************************/
+/**
+ * Both FSDirectory and FSNamesystem manage the state of the namespace.
+ * FSDirectory is a pure in-memory data structure, all of whose operations
+ * happen entirely in memory. In contrast, FSNamesystem persists the operations
+ * to the disk.
+ * @see org.apache.hadoop.hdfs.server.namenode.FSNamesystem
+ **/
+@InterfaceAudience.Private
public class FSDirectory implements Closeable {
private static INodeDirectorySnapshottable createRoot(FSNamesystem namesystem) {
final INodeDirectory r = new INodeDirectory(
@@ -121,7 +119,6 @@
INodeDirectory rootDir;
FSImage fsImage;
private final FSNamesystem namesystem;
- private volatile boolean ready = false;
private volatile boolean skipQuotaCheck = false; //skip while consuming edits
private final int maxComponentLength;
private final int maxDirItems;
@@ -133,7 +130,6 @@
// lock to protect the directory and BlockMap
private final ReentrantReadWriteLock dirLock;
- private final Condition cond;
// utility methods to acquire and release read lock and write lock
void readLock() {
@@ -176,7 +172,6 @@
FSDirectory(FSImage fsImage, FSNamesystem ns, Configuration conf) {
this.dirLock = new ReentrantReadWriteLock(true); // fair
- this.cond = dirLock.writeLock().newCondition();
rootDir = createRoot(ns);
inodeMap = INodeMap.newInstance(rootDir);
this.fsImage = fsImage;
@@ -233,38 +228,6 @@
}
/**
- * Notify that loading of this FSDirectory is complete, and
- * it is ready for use
- */
- void imageLoadComplete() {
- Preconditions.checkState(!ready, "FSDirectory already loaded");
- setReady();
- }
-
- void setReady() {
- if(ready) return;
- writeLock();
- try {
- setReady(true);
- this.nameCache.initialized();
- cond.signalAll();
- } finally {
- writeUnlock();
- }
- }
-
- //This is for testing purposes only
- @VisibleForTesting
- boolean isReady() {
- return ready;
- }
-
- // exposed for unit tests
- protected void setReady(boolean flag) {
- ready = flag;
- }
-
- /**
* Shutdown the filestore
*/
@Override
@@ -272,22 +235,12 @@
fsImage.close();
}
- /**
- * Block until the object is ready to be used.
- */
- void waitForReady() {
- if (!ready) {
- writeLock();
- try {
- while (!ready) {
- try {
- cond.await(5000, TimeUnit.MILLISECONDS);
- } catch (InterruptedException ignored) {
- }
- }
- } finally {
- writeUnlock();
- }
+ void markNameCacheInitialized() {
+ writeLock();
+ try {
+ nameCache.initialized();
+ } finally {
+ writeUnlock();
}
}
@@ -313,7 +266,6 @@
String clientMachine, DatanodeDescriptor clientNode)
throws FileAlreadyExistsException, QuotaExceededException,
UnresolvedLinkException, SnapshotAccessControlException, AclException {
- waitForReady();
long modTime = now();
INodeFile newNode = new INodeFile(namesystem.allocateNewInodeId(), null,
@@ -343,6 +295,7 @@
String path,
PermissionStatus permissions,
List<AclEntry> aclEntries,
+ List<XAttr> xAttrs,
short replication,
long modificationTime,
long atime,
@@ -369,6 +322,10 @@
AclStorage.updateINodeAcl(newNode, aclEntries,
Snapshot.CURRENT_STATE_ID);
}
+ if (xAttrs != null) {
+ XAttrStorage.updateINodeXAttrs(newNode, xAttrs,
+ Snapshot.CURRENT_STATE_ID);
+ }
return newNode;
}
} catch (IOException e) {
@@ -386,8 +343,6 @@
*/
BlockInfo addBlock(String path, INodesInPath inodesInPath, Block block,
DatanodeStorageInfo[] targets) throws IOException {
- waitForReady();
-
writeLock();
try {
final INodeFile fileINode = inodesInPath.getLastINode().asFile();
@@ -425,8 +380,6 @@
boolean removeBlock(String path, INodeFile fileNode, Block block)
throws IOException {
Preconditions.checkArgument(fileNode.isUnderConstruction());
- waitForReady();
-
writeLock();
try {
return unprotectedRemoveBlock(path, fileNode, block);
@@ -470,7 +423,6 @@
NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: "
+src+" to "+dst);
}
- waitForReady();
writeLock();
try {
if (!unprotectedRenameTo(src, dst, mtime))
@@ -493,7 +445,6 @@
NameNode.stateChangeLog.debug("DIR* FSDirectory.renameTo: " + src
+ " to " + dst);
}
- waitForReady();
writeLock();
try {
if (unprotectedRenameTo(src, dst, mtime, options)) {
@@ -891,9 +842,10 @@
boolean undoRemoveDst = false;
INode removedDst = null;
+ long removedNum = 0;
try {
if (dstInode != null) { // dst exists remove it
- if (removeLastINode(dstIIP) != -1) {
+ if ((removedNum = removeLastINode(dstIIP)) != -1) {
removedDst = dstIIP.getLastINode();
undoRemoveDst = true;
}
@@ -933,13 +885,15 @@
long filesDeleted = -1;
if (removedDst != null) {
undoRemoveDst = false;
- BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
- List<INode> removedINodes = new ChunkedArrayList<INode>();
- filesDeleted = removedDst.cleanSubtree(Snapshot.CURRENT_STATE_ID,
- dstIIP.getLatestSnapshotId(), collectedBlocks, removedINodes, true)
- .get(Quota.NAMESPACE);
- getFSNamesystem().removePathAndBlocks(src, collectedBlocks,
- removedINodes);
+ if (removedNum > 0) {
+ BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
+ List<INode> removedINodes = new ChunkedArrayList<INode>();
+ filesDeleted = removedDst.cleanSubtree(Snapshot.CURRENT_STATE_ID,
+ dstIIP.getLatestSnapshotId(), collectedBlocks, removedINodes,
+ true).get(Quota.NAMESPACE);
+ getFSNamesystem().removePathAndBlocks(src, collectedBlocks,
+ removedINodes);
+ }
}
if (snapshottableDirs.size() > 0) {
@@ -1022,7 +976,6 @@
Block[] setReplication(String src, short replication, short[] blockRepls)
throws QuotaExceededException, UnresolvedLinkException,
SnapshotAccessControlException {
- waitForReady();
writeLock();
try {
return unprotectedSetReplication(src, replication, blockRepls);
@@ -1145,7 +1098,6 @@
writeLock();
try {
// actual move
- waitForReady();
unprotectedConcat(target, srcs, timestamp);
} finally {
writeUnlock();
@@ -1228,7 +1180,6 @@
if (NameNode.stateChangeLog.isDebugEnabled()) {
NameNode.stateChangeLog.debug("DIR* FSDirectory.delete: " + src);
}
- waitForReady();
final long filesRemoved;
writeLock();
try {
@@ -1701,7 +1652,7 @@
long nsDelta, long dsDelta, boolean checkQuota)
throws QuotaExceededException {
assert hasWriteLock();
- if (!ready) {
+ if (!namesystem.isImageLoaded()) {
//still initializing. do not check or update quotas.
return;
}
@@ -1894,7 +1845,7 @@
*/
private void verifyQuotaForRename(INode[] src, INode[] dst)
throws QuotaExceededException {
- if (!ready || skipQuotaCheck) {
+ if (!namesystem.isImageLoaded() || skipQuotaCheck) {
// Do not check quota if edits log is still being processed
return;
}
@@ -1950,7 +1901,7 @@
void verifyINodeName(byte[] childName) throws HadoopIllegalArgumentException {
if (Arrays.equals(HdfsConstants.DOT_SNAPSHOT_DIR_BYTES, childName)) {
String s = "\"" + HdfsConstants.DOT_SNAPSHOT_DIR + "\" is a reserved name.";
- if (!ready) {
+ if (!namesystem.isImageLoaded()) {
s += " Please rename it before upgrade.";
}
throw new HadoopIllegalArgumentException(s);
@@ -1977,7 +1928,7 @@
getFullPathName((INode[])parentPath, pos - 1): (String)parentPath;
final PathComponentTooLongException e = new PathComponentTooLongException(
maxComponentLength, length, p, DFSUtil.bytes2String(childName));
- if (ready) {
+ if (namesystem.isImageLoaded()) {
throw e;
} else {
// Do not throw if edits log is still being processed
@@ -2001,7 +1952,7 @@
if (count >= maxDirItems) {
final MaxDirectoryItemsExceededException e
= new MaxDirectoryItemsExceededException(maxDirItems, count);
- if (ready) {
+ if (namesystem.isImageLoaded()) {
e.setPathName(getFullPathName(pathComponents, pos - 1));
throw e;
} else {
@@ -2337,7 +2288,6 @@
void reset() {
writeLock();
try {
- setReady(false);
rootDir = createRoot(getFSNamesystem());
inodeMap.clear();
addToInodeMap(rootDir);
@@ -2619,101 +2569,171 @@
}
}
- XAttr removeXAttr(String src, XAttr xAttr) throws IOException {
+ /**
+ * Removes a list of XAttrs from an inode at a path.
+ *
+ * @param src path of inode
+ * @param toRemove XAttrs to be removed
+ * @return List of XAttrs that were removed
+ * @throws IOException if the inode does not exist, if quota is exceeded
+ */
+ List<XAttr> removeXAttrs(final String src, final List<XAttr> toRemove)
+ throws IOException {
writeLock();
try {
- return unprotectedRemoveXAttr(src, xAttr);
+ return unprotectedRemoveXAttrs(src, toRemove);
} finally {
writeUnlock();
}
}
-
- XAttr unprotectedRemoveXAttr(String src,
- XAttr xAttr) throws IOException {
+
+ List<XAttr> unprotectedRemoveXAttrs(final String src,
+ final List<XAttr> toRemove) throws IOException {
assert hasWriteLock();
INodesInPath iip = getINodesInPath4Write(normalizePath(src), true);
INode inode = resolveLastINode(src, iip);
int snapshotId = iip.getLatestSnapshotId();
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
- List<XAttr> newXAttrs = filterINodeXAttr(existingXAttrs, xAttr);
+ List<XAttr> removedXAttrs = Lists.newArrayListWithCapacity(toRemove.size());
+ List<XAttr> newXAttrs = filterINodeXAttrs(existingXAttrs, toRemove,
+ removedXAttrs);
if (existingXAttrs.size() != newXAttrs.size()) {
XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId);
- return xAttr;
+ return removedXAttrs;
}
return null;
}
-
- List<XAttr> filterINodeXAttr(List<XAttr> existingXAttrs,
- XAttr xAttr) throws QuotaExceededException {
- if (existingXAttrs == null || existingXAttrs.isEmpty()) {
+
+ /**
+ * Filter XAttrs from a list of existing XAttrs. Removes matched XAttrs from
+ * toFilter and puts them into filtered. Upon completion,
+ * toFilter contains the filter XAttrs that were not found, while
+ * fitleredXAttrs contains the XAttrs that were found.
+ *
+ * @param existingXAttrs Existing XAttrs to be filtered
+ * @param toFilter XAttrs to filter from the existing XAttrs
+ * @param filtered Return parameter, XAttrs that were filtered
+ * @return List of XAttrs that does not contain filtered XAttrs
+ */
+ @VisibleForTesting
+ List<XAttr> filterINodeXAttrs(final List<XAttr> existingXAttrs,
+ final List<XAttr> toFilter, final List<XAttr> filtered) {
+ if (existingXAttrs == null || existingXAttrs.isEmpty() ||
+ toFilter == null || toFilter.isEmpty()) {
return existingXAttrs;
}
-
- List<XAttr> xAttrs = Lists.newArrayListWithCapacity(existingXAttrs.size());
+
+ // Populate a new list with XAttrs that pass the filter
+ List<XAttr> newXAttrs =
+ Lists.newArrayListWithCapacity(existingXAttrs.size());
for (XAttr a : existingXAttrs) {
- if (!(a.getNameSpace() == xAttr.getNameSpace()
- && a.getName().equals(xAttr.getName()))) {
- xAttrs.add(a);
+ boolean add = true;
+ for (ListIterator<XAttr> it = toFilter.listIterator(); it.hasNext()
+ ;) {
+ XAttr filter = it.next();
+ if (a.equalsIgnoreValue(filter)) {
+ add = false;
+ it.remove();
+ filtered.add(filter);
+ break;
+ }
+ }
+ if (add) {
+ newXAttrs.add(a);
}
}
-
- return xAttrs;
+
+ return newXAttrs;
}
- void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
- throws IOException {
+ void setXAttrs(final String src, final List<XAttr> xAttrs,
+ final EnumSet<XAttrSetFlag> flag) throws IOException {
writeLock();
try {
- unprotectedSetXAttr(src, xAttr, flag);
+ unprotectedSetXAttrs(src, xAttrs, flag);
} finally {
writeUnlock();
}
}
- void unprotectedSetXAttr(String src, XAttr xAttr,
- EnumSet<XAttrSetFlag> flag) throws IOException {
+ void unprotectedSetXAttrs(final String src, final List<XAttr> xAttrs,
+ final EnumSet<XAttrSetFlag> flag)
+ throws QuotaExceededException, IOException {
assert hasWriteLock();
INodesInPath iip = getINodesInPath4Write(normalizePath(src), true);
INode inode = resolveLastINode(src, iip);
int snapshotId = iip.getLatestSnapshotId();
List<XAttr> existingXAttrs = XAttrStorage.readINodeXAttrs(inode);
- List<XAttr> newXAttrs = setINodeXAttr(existingXAttrs, xAttr, flag);
+ List<XAttr> newXAttrs = setINodeXAttrs(existingXAttrs, xAttrs, flag);
XAttrStorage.updateINodeXAttrs(inode, newXAttrs, snapshotId);
}
-
- List<XAttr> setINodeXAttr(List<XAttr> existingXAttrs, XAttr xAttr,
- EnumSet<XAttrSetFlag> flag) throws QuotaExceededException, IOException {
- List<XAttr> xAttrs = Lists.newArrayListWithCapacity(
- existingXAttrs != null ? existingXAttrs.size() + 1 : 1);
+
+ List<XAttr> setINodeXAttrs(final List<XAttr> existingXAttrs,
+ final List<XAttr> toSet, final EnumSet<XAttrSetFlag> flag)
+ throws IOException {
+ // Check for duplicate XAttrs in toSet
+ // We need to use a custom comparator, so using a HashSet is not suitable
+ for (int i = 0; i < toSet.size(); i++) {
+ for (int j = i + 1; j < toSet.size(); j++) {
+ if (toSet.get(i).equalsIgnoreValue(toSet.get(j))) {
+ throw new IOException("Cannot specify the same XAttr to be set " +
+ "more than once");
+ }
+ }
+ }
+
+ // Count the current number of user-visible XAttrs for limit checking
int userVisibleXAttrsNum = 0; // Number of user visible xAttrs
- boolean exist = false;
+
+ // The XAttr list is copied to an exactly-sized array when it's stored,
+ // so there's no need to size it precisely here.
+ int newSize = (existingXAttrs != null) ? existingXAttrs.size() : 0;
+ newSize += toSet.size();
+ List<XAttr> xAttrs = Lists.newArrayListWithCapacity(newSize);
+
+ // Check if the XAttr already exists to validate with the provided flag
+ for (XAttr xAttr: toSet) {
+ boolean exist = false;
+ if (existingXAttrs != null) {
+ for (XAttr a : existingXAttrs) {
+ if (a.equalsIgnoreValue(xAttr)) {
+ exist = true;
+ break;
+ }
+ }
+ }
+ XAttrSetFlag.validate(xAttr.getName(), exist, flag);
+ // add the new XAttr since it passed validation
+ xAttrs.add(xAttr);
+ if (isUserVisible(xAttr)) {
+ userVisibleXAttrsNum++;
+ }
+ }
+
+ // Add the existing xattrs back in, if they weren't already set
if (existingXAttrs != null) {
- for (XAttr a: existingXAttrs) {
- if ((a.getNameSpace() == xAttr.getNameSpace()
- && a.getName().equals(xAttr.getName()))) {
- exist = true;
- } else {
- xAttrs.add(a);
-
- if (isUserVisible(a)) {
+ for (XAttr existing : existingXAttrs) {
+ boolean alreadySet = false;
+ for (XAttr set : toSet) {
+ if (set.equalsIgnoreValue(existing)) {
+ alreadySet = true;
+ break;
+ }
+ }
+ if (!alreadySet) {
+ xAttrs.add(existing);
+ if (isUserVisible(existing)) {
userVisibleXAttrsNum++;
}
}
}
}
-
- XAttrSetFlag.validate(xAttr.getName(), exist, flag);
- xAttrs.add(xAttr);
-
- if (isUserVisible(xAttr)) {
- userVisibleXAttrsNum++;
- }
-
+
if (userVisibleXAttrsNum > inodeXAttrsLimit) {
throw new IOException("Cannot add additional XAttr to inode, "
+ "would exceed limit of " + inodeXAttrsLimit);
}
-
+
return xAttrs;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index e88cc53..85cfc1c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -700,12 +700,19 @@
.setBlocks(newNode.getBlocks())
.setPermissionStatus(permissions)
.setClientName(newNode.getFileUnderConstructionFeature().getClientName())
- .setClientMachine(newNode.getFileUnderConstructionFeature().getClientMachine());
+ .setClientMachine(
+ newNode.getFileUnderConstructionFeature().getClientMachine());
AclFeature f = newNode.getAclFeature();
if (f != null) {
op.setAclEntries(AclStorage.readINodeLogicalAcl(newNode));
}
+
+ XAttrFeature x = newNode.getXAttrFeature();
+ if (x != null) {
+ op.setXAttrs(x.getXAttrs());
+ }
+
logRpcIds(op, toLogRpcIds);
logEdit(op);
}
@@ -761,6 +768,11 @@
if (f != null) {
op.setAclEntries(AclStorage.readINodeLogicalAcl(newNode));
}
+
+ XAttrFeature x = newNode.getXAttrFeature();
+ if (x != null) {
+ op.setXAttrs(x.getXAttrs());
+ }
logEdit(op);
}
@@ -1054,18 +1066,18 @@
logEdit(op);
}
- void logSetXAttr(String src, XAttr xAttr, boolean toLogRpcIds) {
+ void logSetXAttrs(String src, List<XAttr> xAttrs, boolean toLogRpcIds) {
final SetXAttrOp op = SetXAttrOp.getInstance();
op.src = src;
- op.xAttr = xAttr;
+ op.xAttrs = xAttrs;
logRpcIds(op, toLogRpcIds);
logEdit(op);
}
- void logRemoveXAttr(String src, XAttr xAttr) {
+ void logRemoveXAttrs(String src, List<XAttr> xAttrs) {
final RemoveXAttrOp op = RemoveXAttrOp.getInstance();
op.src = src;
- op.xAttr = xAttr;
+ op.xAttrs = xAttrs;
logEdit(op);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 262fec0..04785c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -355,6 +355,7 @@
lastInodeId);
newFile = fsDir.unprotectedAddFile(inodeId,
path, addCloseOp.permissions, addCloseOp.aclEntries,
+ addCloseOp.xAttrs,
replication, addCloseOp.mtime, addCloseOp.atime,
addCloseOp.blockSize, true, addCloseOp.clientName,
addCloseOp.clientMachine);
@@ -804,7 +805,7 @@
}
case OP_SET_XATTR: {
SetXAttrOp setXAttrOp = (SetXAttrOp) op;
- fsDir.unprotectedSetXAttr(setXAttrOp.src, setXAttrOp.xAttr,
+ fsDir.unprotectedSetXAttrs(setXAttrOp.src, setXAttrOp.xAttrs,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
if (toAddRetryCache) {
fsNamesys.addCacheEntry(setXAttrOp.rpcClientId, setXAttrOp.rpcCallId);
@@ -813,7 +814,8 @@
}
case OP_REMOVE_XATTR: {
RemoveXAttrOp removeXAttrOp = (RemoveXAttrOp) op;
- fsDir.unprotectedRemoveXAttr(removeXAttrOp.src, removeXAttrOp.xAttr);
+ fsDir.unprotectedRemoveXAttrs(removeXAttrOp.src,
+ removeXAttrOp.xAttrs);
break;
}
default:
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index 39e9ca2..14cf7ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -382,6 +382,16 @@
}
}
+ private static List<XAttr> readXAttrsFromEditLog(DataInputStream in,
+ int logVersion) throws IOException {
+ if (!NameNodeLayoutVersion.supports(NameNodeLayoutVersion.Feature.XATTRS,
+ logVersion)) {
+ return null;
+ }
+ XAttrEditLogProto proto = XAttrEditLogProto.parseDelimitedFrom(in);
+ return PBHelper.convertXAttrs(proto.getXAttrsList());
+ }
+
@SuppressWarnings("unchecked")
static abstract class AddCloseOp extends FSEditLogOp implements BlockListUpdatingOp {
int length;
@@ -394,6 +404,7 @@
Block[] blocks;
PermissionStatus permissions;
List<AclEntry> aclEntries;
+ List<XAttr> xAttrs;
String clientName;
String clientMachine;
@@ -461,6 +472,11 @@
return (T)this;
}
+ <T extends AddCloseOp> T setXAttrs(List<XAttr> xAttrs) {
+ this.xAttrs = xAttrs;
+ return (T)this;
+ }
+
<T extends AddCloseOp> T setClientName(String clientName) {
this.clientName = clientName;
return (T)this;
@@ -484,6 +500,9 @@
if (this.opCode == OP_ADD) {
AclEditLogUtil.write(aclEntries, out);
+ XAttrEditLogProto.Builder b = XAttrEditLogProto.newBuilder();
+ b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
+ b.build().writeDelimitedTo(out);
FSImageSerialization.writeString(clientName,out);
FSImageSerialization.writeString(clientMachine,out);
// write clientId and callId
@@ -546,9 +565,9 @@
this.blocks = readBlocks(in, logVersion);
this.permissions = PermissionStatus.read(in);
- // clientname, clientMachine and block locations of last block.
if (this.opCode == OP_ADD) {
aclEntries = AclEditLogUtil.read(in, logVersion);
+ this.xAttrs = readXAttrsFromEditLog(in, logVersion);
this.clientName = FSImageSerialization.readString(in);
this.clientMachine = FSImageSerialization.readString(in);
// read clientId and callId
@@ -1343,6 +1362,7 @@
long timestamp;
PermissionStatus permissions;
List<AclEntry> aclEntries;
+ List<XAttr> xAttrs;
private MkdirOp() {
super(OP_MKDIR);
@@ -1377,6 +1397,11 @@
return this;
}
+ MkdirOp setXAttrs(List<XAttr> xAttrs) {
+ this.xAttrs = xAttrs;
+ return this;
+ }
+
@Override
public
void writeFields(DataOutputStream out) throws IOException {
@@ -1386,6 +1411,9 @@
FSImageSerialization.writeLong(timestamp, out); // atime, unused at this
permissions.write(out);
AclEditLogUtil.write(aclEntries, out);
+ XAttrEditLogProto.Builder b = XAttrEditLogProto.newBuilder();
+ b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
+ b.build().writeDelimitedTo(out);
}
@Override
@@ -1430,6 +1458,8 @@
this.permissions = PermissionStatus.read(in);
aclEntries = AclEditLogUtil.read(in, logVersion);
+
+ xAttrs = readXAttrsFromEditLog(in, logVersion);
}
@Override
@@ -1451,6 +1481,8 @@
builder.append(opCode);
builder.append(", txid=");
builder.append(txid);
+ builder.append(", xAttrs=");
+ builder.append(xAttrs);
builder.append("]");
return builder.toString();
}
@@ -1468,6 +1500,9 @@
if (aclEntries != null) {
appendAclEntriesToXml(contentHandler, aclEntries);
}
+ if (xAttrs != null) {
+ appendXAttrsToXml(contentHandler, xAttrs);
+ }
}
@Override void fromXml(Stanza st) throws InvalidXmlException {
@@ -1477,6 +1512,7 @@
this.timestamp = Long.parseLong(st.getValue("TIMESTAMP"));
this.permissions = permissionStatusFromXml(st);
aclEntries = readAclEntriesFromXml(st);
+ xAttrs = readXAttrsFromXml(st);
}
}
@@ -3499,7 +3535,7 @@
}
static class RemoveXAttrOp extends FSEditLogOp {
- XAttr xAttr;
+ List<XAttr> xAttrs;
String src;
private RemoveXAttrOp() {
@@ -3514,7 +3550,7 @@
void readFields(DataInputStream in, int logVersion) throws IOException {
XAttrEditLogProto p = XAttrEditLogProto.parseDelimitedFrom(in);
src = p.getSrc();
- xAttr = PBHelper.convertXAttr(p.getXAttr());
+ xAttrs = PBHelper.convertXAttrs(p.getXAttrsList());
}
@Override
@@ -3523,25 +3559,25 @@
if (src != null) {
b.setSrc(src);
}
- b.setXAttr(PBHelper.convertXAttrProto(xAttr));
+ b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
b.build().writeDelimitedTo(out);
}
@Override
protected void toXml(ContentHandler contentHandler) throws SAXException {
XMLUtils.addSaxString(contentHandler, "SRC", src);
- appendXAttrToXml(contentHandler, xAttr);
+ appendXAttrsToXml(contentHandler, xAttrs);
}
@Override
void fromXml(Stanza st) throws InvalidXmlException {
src = st.getValue("SRC");
- xAttr = readXAttrFromXml(st);
+ xAttrs = readXAttrsFromXml(st);
}
}
static class SetXAttrOp extends FSEditLogOp {
- XAttr xAttr;
+ List<XAttr> xAttrs;
String src;
private SetXAttrOp() {
@@ -3556,7 +3592,7 @@
void readFields(DataInputStream in, int logVersion) throws IOException {
XAttrEditLogProto p = XAttrEditLogProto.parseDelimitedFrom(in);
src = p.getSrc();
- xAttr = PBHelper.convertXAttr(p.getXAttr());
+ xAttrs = PBHelper.convertXAttrs(p.getXAttrsList());
readRpcIds(in, logVersion);
}
@@ -3566,7 +3602,7 @@
if (src != null) {
b.setSrc(src);
}
- b.setXAttr(PBHelper.convertXAttrProto(xAttr));
+ b.addAllXAttrs(PBHelper.convertXAttrProto(xAttrs));
b.build().writeDelimitedTo(out);
// clientId and callId
writeRpcIds(rpcClientId, rpcCallId, out);
@@ -3575,14 +3611,14 @@
@Override
protected void toXml(ContentHandler contentHandler) throws SAXException {
XMLUtils.addSaxString(contentHandler, "SRC", src);
- appendXAttrToXml(contentHandler, xAttr);
+ appendXAttrsToXml(contentHandler, xAttrs);
appendRpcIdsToXml(contentHandler, rpcClientId, rpcCallId);
}
@Override
void fromXml(Stanza st) throws InvalidXmlException {
src = st.getValue("SRC");
- xAttr = readXAttrFromXml(st);
+ xAttrs = readXAttrsFromXml(st);
readRpcIdsFromXml(st);
}
}
@@ -4202,42 +4238,48 @@
}
return aclEntries;
}
-
- private static void appendXAttrToXml(ContentHandler contentHandler,
- XAttr xAttr) throws SAXException {
- contentHandler.startElement("", "", "XATTR", new AttributesImpl());
- XMLUtils.addSaxString(contentHandler, "NAMESPACE",
- xAttr.getNameSpace().toString());
- XMLUtils.addSaxString(contentHandler, "NAME", xAttr.getName());
- if (xAttr.getValue() != null) {
- try {
- XMLUtils.addSaxString(contentHandler, "VALUE",
- XAttrCodec.encodeValue(xAttr.getValue(), XAttrCodec.HEX));
- } catch (IOException e) {
- throw new SAXException(e);
+
+ private static void appendXAttrsToXml(ContentHandler contentHandler,
+ List<XAttr> xAttrs) throws SAXException {
+ for (XAttr xAttr: xAttrs) {
+ contentHandler.startElement("", "", "XATTR", new AttributesImpl());
+ XMLUtils.addSaxString(contentHandler, "NAMESPACE",
+ xAttr.getNameSpace().toString());
+ XMLUtils.addSaxString(contentHandler, "NAME", xAttr.getName());
+ if (xAttr.getValue() != null) {
+ try {
+ XMLUtils.addSaxString(contentHandler, "VALUE",
+ XAttrCodec.encodeValue(xAttr.getValue(), XAttrCodec.HEX));
+ } catch (IOException e) {
+ throw new SAXException(e);
+ }
}
+ contentHandler.endElement("", "", "XATTR");
}
- contentHandler.endElement("", "", "XATTR");
}
-
- private static XAttr readXAttrFromXml(Stanza st)
+
+ private static List<XAttr> readXAttrsFromXml(Stanza st)
throws InvalidXmlException {
if (!st.hasChildren("XATTR")) {
return null;
}
-
- Stanza a = st.getChildren("XATTR").get(0);
- XAttr.Builder builder = new XAttr.Builder();
- builder.setNameSpace(XAttr.NameSpace.valueOf(a.getValue("NAMESPACE"))).
- setName(a.getValue("NAME"));
- String v = a.getValueOrNull("VALUE");
- if (v != null) {
- try {
- builder.setValue(XAttrCodec.decodeValue(v));
- } catch (IOException e) {
- throw new InvalidXmlException(e.toString());
+
+ List<Stanza> stanzas = st.getChildren("XATTR");
+ List<XAttr> xattrs = Lists.newArrayListWithCapacity(stanzas.size());
+ for (Stanza a: stanzas) {
+ XAttr.Builder builder = new XAttr.Builder();
+ builder.setNameSpace(XAttr.NameSpace.valueOf(a.getValue("NAMESPACE"))).
+ setName(a.getValue("NAME"));
+ String v = a.getValueOrNull("VALUE");
+ if (v != null) {
+ try {
+ builder.setValue(XAttrCodec.decodeValue(v));
+ } catch (IOException e) {
+ throw new InvalidXmlException(e.toString());
+ }
}
+ xattrs.add(builder.build());
}
- return builder.build();
+ return xattrs;
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index fb8e3f6..077570e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -533,8 +533,10 @@
INodeSection.INodeFile.Builder b = buildINodeFile(n,
parent.getSaverContext());
- for (Block block : n.getBlocks()) {
- b.addBlocks(PBHelper.convert(block));
+ if (n.getBlocks() != null) {
+ for (Block block : n.getBlocks()) {
+ b.addBlocks(PBHelper.convert(block));
+ }
}
FileUnderConstructionFeature uc = n.getFileUnderConstructionFeature();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index b7698b6..ad7b4d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -103,6 +103,7 @@
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -514,6 +515,59 @@
private final NNConf nnConf;
+ private volatile boolean imageLoaded = false;
+ private final Condition cond;
+ /**
+ * Notify that loading of this FSDirectory is complete, and
+ * it is imageLoaded for use
+ */
+ void imageLoadComplete() {
+ Preconditions.checkState(!imageLoaded, "FSDirectory already loaded");
+ setImageLoaded();
+ }
+
+ void setImageLoaded() {
+ if(imageLoaded) return;
+ writeLock();
+ try {
+ setImageLoaded(true);
+ dir.markNameCacheInitialized();
+ cond.signalAll();
+ } finally {
+ writeUnlock();
+ }
+ }
+
+ //This is for testing purposes only
+ @VisibleForTesting
+ boolean isImageLoaded() {
+ return imageLoaded;
+ }
+
+ // exposed for unit tests
+ protected void setImageLoaded(boolean flag) {
+ imageLoaded = flag;
+ }
+
+ /**
+ * Block until the object is imageLoaded to be used.
+ */
+ void waitForLoadingFSImage() {
+ if (!imageLoaded) {
+ writeLock();
+ try {
+ while (!imageLoaded) {
+ try {
+ cond.await(5000, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException ignored) {
+ }
+ }
+ } finally {
+ writeUnlock();
+ }
+ }
+ }
+
/**
* Set the last allocated inode id when fsimage or editlog is loaded.
*/
@@ -555,6 +609,7 @@
inodeId.setCurrentValue(INodeId.LAST_RESERVED_ID);
snapshotManager.clearSnapshottableDirs();
cacheManager.clear();
+ setImageLoaded(false);
}
@VisibleForTesting
@@ -682,6 +737,7 @@
boolean fair = conf.getBoolean("dfs.namenode.fslock.fair", true);
LOG.info("fsLock is fair:" + fair);
fsLock = new FSNamesystemLock(fair);
+ cond = fsLock.writeLock().newCondition();
try {
resourceRecheckInterval = conf.getLong(
DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY,
@@ -921,7 +977,7 @@
}
writeUnlock();
}
- dir.imageLoadComplete();
+ imageLoadComplete();
}
private void startSecretManager() {
@@ -1840,6 +1896,7 @@
HdfsFileStatus resultingStat = null;
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -2115,6 +2172,7 @@
FSPermissionChecker pc = getPermissionChecker();
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -2242,6 +2300,8 @@
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
boolean create = flag.contains(CreateFlag.CREATE);
boolean overwrite = flag.contains(CreateFlag.OVERWRITE);
+
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -2524,10 +2584,10 @@
// We found the lease for this file. And surprisingly the original
// holder is trying to recreate this file. This should never occur.
//
+
if (!force && lease != null) {
Lease leaseFile = leaseManager.getLeaseByPath(src);
- if ((leaseFile != null && leaseFile.equals(lease)) ||
- lease.getHolder().equals(holder)) {
+ if (leaseFile != null && leaseFile.equals(lease)) {
throw new AlreadyBeingCreatedException(
"failed to create file " + src + " for " + holder +
" for client " + clientMachine +
@@ -2730,6 +2790,7 @@
Block newBlock = null;
long offset;
checkOperation(OperationCategory.WRITE);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -2952,6 +3013,7 @@
}
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -3050,6 +3112,7 @@
boolean success = false;
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -3249,6 +3312,7 @@
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot rename " + src);
+ waitForLoadingFSImage();
src = FSDirectory.resolvePath(src, srcComponents, dir);
dst = FSDirectory.resolvePath(dst, dstComponents, dir);
checkOperation(OperationCategory.WRITE);
@@ -3356,6 +3420,7 @@
false);
}
+ waitForLoadingFSImage();
long mtime = now();
dir.renameTo(src, dst, mtime, options);
getEditLog().logRename(src, dst, mtime, logRetryCache, options);
@@ -3429,6 +3494,8 @@
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
boolean ret = false;
+
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -3902,6 +3969,8 @@
NameNode.stateChangeLog.info("BLOCK* fsync: " + src + " for " + clientName);
checkOperation(OperationCategory.WRITE);
byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
+
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -4103,6 +4172,7 @@
INodeFile pendingFile, int latestSnapshot) throws IOException,
UnresolvedLinkException {
assert hasWriteLock();
+
FileUnderConstructionFeature uc = pendingFile.getFileUnderConstructionFeature();
Preconditions.checkArgument(uc != null);
leaseManager.removeLease(uc.getClientName(), src);
@@ -4114,6 +4184,7 @@
// since we just remove the uc feature from pendingFile
final INodeFile newFile = pendingFile.toCompleteFile(now());
+ waitForLoadingFSImage();
// close file and persist block allocations for this file
closeFile(src, newFile);
@@ -4172,6 +4243,7 @@
+ ")");
checkOperation(OperationCategory.WRITE);
String src = "";
+ waitForLoadingFSImage();
writeLock();
try {
checkOperation(OperationCategory.WRITE);
@@ -4517,7 +4589,7 @@
*/
private void closeFile(String path, INodeFile file) {
assert hasWriteLock();
- dir.waitForReady();
+ waitForLoadingFSImage();
// file is closed
getEditLog().logCloseFile(path, file);
if (NameNode.stateChangeLog.isDebugEnabled()) {
@@ -4541,7 +4613,7 @@
boolean createParent, boolean logRetryCache)
throws UnresolvedLinkException, FileAlreadyExistsException,
QuotaExceededException, SnapshotAccessControlException, AclException {
- dir.waitForReady();
+ waitForLoadingFSImage();
final long modTime = now();
if (createParent) {
@@ -5804,7 +5876,7 @@
boolean ignoreEmptyDir, boolean resolveLink)
throws AccessControlException, UnresolvedLinkException {
if (!pc.isSuperUser()) {
- dir.waitForReady();
+ waitForLoadingFSImage();
readLock();
try {
pc.checkPermission(path, dir, doCheckOwner, ancestorAccess,
@@ -6271,6 +6343,7 @@
+ ", newNodes=" + Arrays.asList(newNodes)
+ ", clientName=" + clientName
+ ")");
+ waitForLoadingFSImage();
writeLock();
boolean success = false;
try {
@@ -8121,8 +8194,10 @@
checkOwner(pc, src);
checkPathAccess(pc, src, FsAction.WRITE);
}
- dir.setXAttr(src, xAttr, flag);
- getEditLog().logSetXAttr(src, xAttr, logRetryCache);
+ List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
+ xAttrs.add(xAttr);
+ dir.setXAttrs(src, xAttrs, flag);
+ getEditLog().logSetXAttrs(src, xAttrs, logRetryCache);
resultingStat = getAuditFileInfo(src, false);
} finally {
writeUnlock();
@@ -8242,10 +8317,12 @@
checkOwner(pc, src);
checkPathAccess(pc, src, FsAction.WRITE);
}
-
- XAttr removedXAttr = dir.removeXAttr(src, xAttr);
- if (removedXAttr != null) {
- getEditLog().logRemoveXAttr(src, removedXAttr);
+
+ List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
+ xAttrs.add(xAttr);
+ List<XAttr> removedXAttrs = dir.removeXAttrs(src, xAttrs);
+ if (removedXAttrs != null && !removedXAttrs.isEmpty()) {
+ getEditLog().logRemoveXAttrs(src, removedXAttrs);
}
resultingStat = getAuditFileInfo(src, false);
} catch (AccessControlException e) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
index 9cdad26..9747f9c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
@@ -598,7 +598,8 @@
pauseMonitor = new JvmPauseMonitor(conf);
pauseMonitor.start();
-
+ metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
+
startCommonServices(conf);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
index 2f3e290..10f1720 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/SecondaryNameNode.java
@@ -1064,7 +1064,7 @@
} finally {
dstNamesystem.writeUnlock();
}
- dstNamesystem.dir.imageLoadComplete();
+ dstNamesystem.imageLoadComplete();
}
// error simulation code for junit test
CheckpointFaultInjector.getInstance().duringMerge();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
index 52ad185..42942dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/NameNodeMetrics.java
@@ -98,7 +98,11 @@
@Metric("GetImageServlet putImage")
MutableRate putImage;
- NameNodeMetrics(String processName, String sessionId, int[] intervals) {
+ JvmMetrics jvmMetrics = null;
+
+ NameNodeMetrics(String processName, String sessionId, int[] intervals,
+ final JvmMetrics jvmMetrics) {
+ this.jvmMetrics = jvmMetrics;
registry.tag(ProcessName, processName).tag(SessionId, sessionId);
final int len = intervals.length;
@@ -124,14 +128,19 @@
String sessionId = conf.get(DFSConfigKeys.DFS_METRICS_SESSION_ID_KEY);
String processName = r.toString();
MetricsSystem ms = DefaultMetricsSystem.instance();
- JvmMetrics.create(processName, sessionId, ms);
+ JvmMetrics jm = JvmMetrics.create(processName, sessionId, ms);
// Percentile measurement is off by default, by watching no intervals
int[] intervals =
conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
- return ms.register(new NameNodeMetrics(processName, sessionId, intervals));
+ return ms.register(new NameNodeMetrics(processName, sessionId,
+ intervals, jm));
}
+ public JvmMetrics getJvmMetrics() {
+ return jvmMetrics;
+ }
+
public void shutdown() {
DefaultMetricsSystem.shutdown();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
index e32f78a..52adfc6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
@@ -159,7 +159,7 @@
// resize the array.
final BlockInfo[] newBlocks;
if (n == 0) {
- newBlocks = null;
+ newBlocks = BlockInfo.EMPTY_ARRAY;
} else {
newBlocks = new BlockInfo[n];
System.arraycopy(oldBlocks, 0, newBlocks, 0, n);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index a3238df..f8dcc39 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -344,8 +344,6 @@
*/
private synchronized void resetStateToFailOver() {
currentNNAddrIndex = (currentNNAddrIndex + 1) % nnAddrs.length;
- delegationToken = null;
- tokenAspect.reset();
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto
index 9e70a37..cb86ff2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/xattr.proto
@@ -35,8 +35,8 @@
}
message XAttrEditLogProto {
- required string src = 1;
- optional XAttrProto xAttr = 2;
+ optional string src = 1;
+ repeated XAttrProto xAttrs = 2;
}
enum XAttrSetFlagProto {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm
index 45d43f1..3d27b8e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/Federation.apt.vm
@@ -287,13 +287,14 @@
Policy could be:
- * <<<node>>> - this is the <default> policy. This balances the storage at
+ * <<<datanode>>> - this is the <default> policy. This balances the storage at
the datanode level. This is similar to balancing policy from prior releases.
* <<<blockpool>>> - this balances the storage at the block pool level.
Balancing at block pool level balances storage at the datanode level also.
- Note that Balander only balances the data and does not balance the namespace.
+ Note that Balancer only balances the data and does not balance the namespace.
+ For the complete command usage, see {{{../hadoop-common/CommandsManual.html#balancer}balancer}}.
** Decommissioning
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm
index e3d6b8c..54544cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsNfsGateway.apt.vm
@@ -88,6 +88,25 @@
</property>
----
+ The AIX NFS client has a {{{https://issues.apache.org/jira/browse/HDFS-6549}few known issues}}
+ that prevent it from working correctly by default with the HDFS NFS
+ Gateway. If you want to be able to access the HDFS NFS Gateway from AIX, you
+ should set the following configuration setting to enable work-arounds for these
+ issues:
+
+----
+<property>
+ <name>nfs.aix.compatibility.mode.enabled</name>
+ <value>true</value>
+</property>
+----
+
+ Note that regular, non-AIX clients should NOT enable AIX compatibility mode.
+ The work-arounds implemented by AIX compatibility mode effectively disable
+ safeguards to ensure that listing of directory contents via NFS returns
+ consistent results, and that all data sent to the NFS server can be assured to
+ have been committed.
+
It's strongly recommended for the users to update a few configuration properties based on their use
cases. All the related configuration properties can be added or updated in hdfs-site.xml.
@@ -322,6 +341,22 @@
Then the users can access HDFS as part of the local file system except that,
hard link and random write are not supported yet.
+* {Allow mounts from unprivileged clients}
+
+ In environments where root access on client machines is not generally
+ available, some measure of security can be obtained by ensuring that only NFS
+ clients originating from privileged ports can connect to the NFS server. This
+ feature is referred to as "port monitoring." This feature is not enabled by default
+ in the HDFS NFS Gateway, but can be optionally enabled by setting the
+ following config in hdfs-site.xml on the NFS Gateway machine:
+
+-------------------------------------------------------------------
+<property>
+ <name>nfs.port.monitoring.disabled</name>
+ <value>false</value>
+</property>
+-------------------------------------------------------------------
+
* {User authentication and mapping}
NFS gateway in this release uses AUTH_UNIX style authentication. When the user on NFS client
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm
index 9d6aeb9..443ace6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsUserGuide.apt.vm
@@ -77,7 +77,7 @@
* <<<fetchdt>>>: a utility to fetch DelegationToken and store it in a
file on the local system.
- * Rebalancer: tool to balance the cluster when the data is
+ * Balancer: tool to balance the cluster when the data is
unevenly distributed among DataNodes.
* Upgrade and rollback: after a software upgrade, it is possible
@@ -316,7 +316,7 @@
For command usage, see {{{../hadoop-common/CommandsManual.html#namenode}namenode}}.
-* Rebalancer
+* Balancer
HDFS data might not always be be placed uniformly across the DataNode.
One common reason is addition of new DataNodes to an existing cluster.
@@ -338,7 +338,7 @@
Due to multiple competing considerations, data might not be uniformly
placed across the DataNodes. HDFS provides a tool for administrators
that analyzes block placement and rebalanaces data across the DataNode.
- A brief administrator's guide for rebalancer as a PDF is attached to
+ A brief administrator's guide for balancer is available at
{{{https://issues.apache.org/jira/browse/HADOOP-1652}HADOOP-1652}}.
For command usage, see {{{../hadoop-common/CommandsManual.html#balancer}balancer}}.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml
index e5612d8..c369f3b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml
@@ -217,7 +217,7 @@
<subsection name="DFSAdmin Commands" id="dfsadminCommands">
<h4><code>dfsadmin -rollingUpgrade</code></h4>
- <source>hdfs dfsadmin -rollingUpgrade <query|start|finalize></source>
+ <source>hdfs dfsadmin -rollingUpgrade <query|prepare|finalize></source>
<p>
Execute a rolling upgrade action.
<ul><li>Options:<table>
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
index 04a0d22..6d981fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecovery2.java
@@ -153,6 +153,15 @@
verifyFile(dfs, filepath1, actual, size);
}
+ @Test
+ public void testLeaseRecoverByAnotherUser() throws Exception {
+ byte [] actual = new byte[FILE_SIZE];
+ cluster.setLeasePeriod(SHORT_LEASE_PERIOD, LONG_LEASE_PERIOD);
+ Path filepath = createFile("/immediateRecoverLease-x", 0, true);
+ recoverLeaseUsingCreate2(filepath);
+ verifyFile(dfs, filepath, actual, 0);
+ }
+
private Path createFile(final String filestr, final int size,
final boolean triggerLeaseRenewerInterrupt)
throws IOException, InterruptedException {
@@ -196,7 +205,7 @@
}
private void recoverLeaseUsingCreate(Path filepath)
- throws IOException, InterruptedException {
+ throws IOException, InterruptedException {
FileSystem dfs2 = getFSAsAnotherUser(conf);
for(int i = 0; i < 10; i++) {
AppendTestUtil.LOG.info("i=" + i);
@@ -216,6 +225,20 @@
fail("recoverLeaseUsingCreate failed");
}
+ private void recoverLeaseUsingCreate2(Path filepath)
+ throws Exception {
+ FileSystem dfs2 = getFSAsAnotherUser(conf);
+ int size = AppendTestUtil.nextInt(FILE_SIZE);
+ DistributedFileSystem dfsx = (DistributedFileSystem) dfs2;
+ //create file using dfsx
+ Path filepath2 = new Path("/immediateRecoverLease-x2");
+ FSDataOutputStream stm = dfsx.create(filepath2, true, BUF_SIZE,
+ REPLICATION_NUM, BLOCK_SIZE);
+ assertTrue(dfsx.dfs.exists("/immediateRecoverLease-x2"));
+ try {Thread.sleep(10000);} catch (InterruptedException e) {}
+ dfsx.append(filepath);
+ }
+
private void verifyFile(FileSystem dfs, Path filepath, byte[] actual,
int size) throws IOException {
AppendTestUtil.LOG.info("Lease for file " + filepath + " is recovered. "
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
index 78a77c4..7cfe423 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockInfo.java
@@ -29,6 +29,8 @@
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.junit.Assert;
import org.junit.Test;
/**
@@ -42,6 +44,34 @@
private static final Log LOG = LogFactory
.getLog("org.apache.hadoop.hdfs.TestBlockInfo");
+
+ @Test
+ public void testAddStorage() throws Exception {
+ BlockInfo blockInfo = new BlockInfo(3);
+
+ final DatanodeStorageInfo storage = DFSTestUtil.createDatanodeStorageInfo("storageID", "127.0.0.1");
+
+ boolean added = blockInfo.addStorage(storage);
+
+ Assert.assertTrue(added);
+ Assert.assertEquals(storage, blockInfo.getStorageInfo(0));
+ }
+
+
+ @Test
+ public void testReplaceStorageIfDifferetnOneAlreadyExistedFromSameDataNode() throws Exception {
+ BlockInfo blockInfo = new BlockInfo(3);
+
+ final DatanodeStorageInfo storage1 = DFSTestUtil.createDatanodeStorageInfo("storageID1", "127.0.0.1");
+ final DatanodeStorageInfo storage2 = new DatanodeStorageInfo(storage1.getDatanodeDescriptor(), new DatanodeStorage("storageID2"));
+
+ blockInfo.addStorage(storage1);
+ boolean added = blockInfo.addStorage(storage2);
+
+ Assert.assertFalse(added);
+ Assert.assertEquals(storage2, blockInfo.getStorageInfo(0));
+ }
+
@Test
public void testBlockListMoveToHead() throws Exception {
LOG.info("BlockInfo moveToHead tests...");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
index dac8c0f..45be905 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestCommitBlockSynchronization.java
@@ -32,7 +32,6 @@
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Mockito.*;
/**
@@ -50,6 +49,7 @@
final DatanodeStorageInfo[] targets = {};
FSNamesystem namesystem = new FSNamesystem(conf, image);
+ namesystem.setImageLoaded(true);
FSNamesystem namesystemSpy = spy(namesystem);
BlockInfoUnderConstruction blockInfo = new BlockInfoUnderConstruction(
block, 1, HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION, targets);
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
index 6e31f2c..011901d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirectory.java
@@ -24,7 +24,9 @@
import java.io.StringReader;
import java.util.EnumSet;
import java.util.List;
+import java.util.Random;
+import com.google.common.collect.ImmutableList;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -36,7 +38,6 @@
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.test.GenericTestUtils;
import org.junit.After;
import org.junit.Assert;
@@ -45,6 +46,11 @@
import com.google.common.collect.Lists;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
/**
* Test {@link FSDirectory}, the in-memory namespace tree.
*/
@@ -74,6 +80,10 @@
private DistributedFileSystem hdfs;
+ private static final int numGeneratedXAttrs = 256;
+ private static final ImmutableList<XAttr> generatedXAttrs =
+ ImmutableList.copyOf(generateXAttrs(numGeneratedXAttrs));
+
@Before
public void setUp() throws Exception {
conf = new Configuration();
@@ -119,25 +129,16 @@
for(; (line = in.readLine()) != null; ) {
line = line.trim();
if (!line.isEmpty() && !line.contains("snapshot")) {
- Assert.assertTrue("line=" + line,
+ assertTrue("line=" + line,
line.startsWith(INodeDirectory.DUMPTREE_LAST_ITEM)
- || line.startsWith(INodeDirectory.DUMPTREE_EXCEPT_LAST_ITEM));
+ || line.startsWith(INodeDirectory.DUMPTREE_EXCEPT_LAST_ITEM)
+ );
checkClassName(line);
}
}
}
@Test
- public void testReset() throws Exception {
- fsdir.reset();
- Assert.assertFalse(fsdir.isReady());
- final INodeDirectory root = (INodeDirectory) fsdir.getINode("/");
- Assert.assertTrue(root.getChildrenList(Snapshot.CURRENT_STATE_ID).isEmpty());
- fsdir.imageLoadComplete();
- Assert.assertTrue(fsdir.isReady());
- }
-
- @Test
public void testSkipQuotaCheck() throws Exception {
try {
// set quota. nsQuota of 1 means no files can be created
@@ -176,7 +177,7 @@
int i = line.lastIndexOf('(');
int j = line.lastIndexOf('@');
final String classname = line.substring(i+1, j);
- Assert.assertTrue(classname.startsWith(INodeFile.class.getSimpleName())
+ assertTrue(classname.startsWith(INodeFile.class.getSimpleName())
|| classname.startsWith(INodeDirectory.class.getSimpleName()));
}
@@ -193,22 +194,185 @@
// Adding a system namespace xAttr, isn't affected by inode xAttrs limit.
XAttr newXAttr = (new XAttr.Builder()).setNameSpace(XAttr.NameSpace.SYSTEM).
setName("a3").setValue(new byte[]{0x33, 0x33, 0x33}).build();
- List<XAttr> xAttrs = fsdir.setINodeXAttr(existingXAttrs, newXAttr,
+ List<XAttr> newXAttrs = Lists.newArrayListWithCapacity(1);
+ newXAttrs.add(newXAttr);
+ List<XAttr> xAttrs = fsdir.setINodeXAttrs(existingXAttrs, newXAttrs,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
- Assert.assertEquals(xAttrs.size(), 3);
+ assertEquals(xAttrs.size(), 3);
// Adding a trusted namespace xAttr, is affected by inode xAttrs limit.
XAttr newXAttr1 = (new XAttr.Builder()).setNameSpace(
XAttr.NameSpace.TRUSTED).setName("a4").
setValue(new byte[]{0x34, 0x34, 0x34}).build();
+ newXAttrs.set(0, newXAttr1);
try {
- fsdir.setINodeXAttr(existingXAttrs, newXAttr1,
+ fsdir.setINodeXAttrs(existingXAttrs, newXAttrs,
EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
- Assert.fail("Setting user visable xattr on inode should fail if " +
+ fail("Setting user visible xattr on inode should fail if " +
"reaching limit.");
} catch (IOException e) {
GenericTestUtils.assertExceptionContains("Cannot add additional XAttr " +
"to inode, would exceed limit", e);
}
}
+
+ /**
+ * Verify that the first <i>num</i> generatedXAttrs are present in
+ * newXAttrs.
+ */
+ private static void verifyXAttrsPresent(List<XAttr> newXAttrs,
+ final int num) {
+ assertEquals("Unexpected number of XAttrs after multiset", num,
+ newXAttrs.size());
+ for (int i=0; i<num; i++) {
+ XAttr search = generatedXAttrs.get(i);
+ assertTrue("Did not find set XAttr " + search + " + after multiset",
+ newXAttrs.contains(search));
+ }
+ }
+
+ private static List<XAttr> generateXAttrs(final int numXAttrs) {
+ List<XAttr> generatedXAttrs = Lists.newArrayListWithCapacity(numXAttrs);
+ for (int i=0; i<numXAttrs; i++) {
+ XAttr xAttr = (new XAttr.Builder())
+ .setNameSpace(XAttr.NameSpace.SYSTEM)
+ .setName("a" + i)
+ .setValue(new byte[] { (byte) i, (byte) (i + 1), (byte) (i + 2) })
+ .build();
+ generatedXAttrs.add(xAttr);
+ }
+ return generatedXAttrs;
+ }
+
+ /**
+ * Test setting and removing multiple xattrs via single operations
+ */
+ @Test(timeout=300000)
+ public void testXAttrMultiSetRemove() throws Exception {
+ List<XAttr> existingXAttrs = Lists.newArrayListWithCapacity(0);
+
+ // Keep adding a random number of xattrs and verifying until exhausted
+ final Random rand = new Random(0xFEEDA);
+ int numExpectedXAttrs = 0;
+ while (numExpectedXAttrs < numGeneratedXAttrs) {
+ LOG.info("Currently have " + numExpectedXAttrs + " xattrs");
+ final int numToAdd = rand.nextInt(5)+1;
+
+ List<XAttr> toAdd = Lists.newArrayListWithCapacity(numToAdd);
+ for (int i = 0; i < numToAdd; i++) {
+ if (numExpectedXAttrs >= numGeneratedXAttrs) {
+ break;
+ }
+ toAdd.add(generatedXAttrs.get(numExpectedXAttrs));
+ numExpectedXAttrs++;
+ }
+ LOG.info("Attempting to add " + toAdd.size() + " XAttrs");
+ for (int i = 0; i < toAdd.size(); i++) {
+ LOG.info("Will add XAttr " + toAdd.get(i));
+ }
+ List<XAttr> newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
+ EnumSet.of(XAttrSetFlag.CREATE));
+ verifyXAttrsPresent(newXAttrs, numExpectedXAttrs);
+ existingXAttrs = newXAttrs;
+ }
+
+ // Keep removing a random number of xattrs and verifying until all gone
+ while (numExpectedXAttrs > 0) {
+ LOG.info("Currently have " + numExpectedXAttrs + " xattrs");
+ final int numToRemove = rand.nextInt(5)+1;
+ List<XAttr> toRemove = Lists.newArrayListWithCapacity(numToRemove);
+ for (int i = 0; i < numToRemove; i++) {
+ if (numExpectedXAttrs == 0) {
+ break;
+ }
+ toRemove.add(generatedXAttrs.get(numExpectedXAttrs-1));
+ numExpectedXAttrs--;
+ }
+ final int expectedNumToRemove = toRemove.size();
+ LOG.info("Attempting to remove " + expectedNumToRemove + " XAttrs");
+ List<XAttr> removedXAttrs = Lists.newArrayList();
+ List<XAttr> newXAttrs = fsdir.filterINodeXAttrs(existingXAttrs,
+ toRemove, removedXAttrs);
+ assertEquals("Unexpected number of removed XAttrs",
+ expectedNumToRemove, removedXAttrs.size());
+ verifyXAttrsPresent(newXAttrs, numExpectedXAttrs);
+ existingXAttrs = newXAttrs;
+ }
+ }
+
+ @Test(timeout=300000)
+ public void testXAttrMultiAddRemoveErrors() throws Exception {
+
+ // Test that the same XAttr can not be multiset twice
+ List<XAttr> existingXAttrs = Lists.newArrayList();
+ List<XAttr> toAdd = Lists.newArrayList();
+ toAdd.add(generatedXAttrs.get(0));
+ toAdd.add(generatedXAttrs.get(1));
+ toAdd.add(generatedXAttrs.get(2));
+ toAdd.add(generatedXAttrs.get(0));
+ try {
+ fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
+ .CREATE));
+ fail("Specified the same xattr to be set twice");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("Cannot specify the same " +
+ "XAttr to be set", e);
+ }
+
+ // Test that CREATE and REPLACE flags are obeyed
+ toAdd.remove(generatedXAttrs.get(0));
+ existingXAttrs.add(generatedXAttrs.get(0));
+ try {
+ fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
+ .CREATE));
+ fail("Set XAttr that is already set without REPLACE flag");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("already exists", e);
+ }
+ try {
+ fsdir.setINodeXAttrs(existingXAttrs, toAdd, EnumSet.of(XAttrSetFlag
+ .REPLACE));
+ fail("Set XAttr that does not exist without the CREATE flag");
+ } catch (IOException e) {
+ GenericTestUtils.assertExceptionContains("does not exist", e);
+ }
+
+ // Sanity test for CREATE
+ toAdd.remove(generatedXAttrs.get(0));
+ List<XAttr> newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
+ EnumSet.of(XAttrSetFlag.CREATE));
+ assertEquals("Unexpected toAdd size", 2, toAdd.size());
+ for (XAttr x : toAdd) {
+ assertTrue("Did not find added XAttr " + x, newXAttrs.contains(x));
+ }
+ existingXAttrs = newXAttrs;
+
+ // Sanity test for REPLACE
+ toAdd = Lists.newArrayList();
+ for (int i=0; i<3; i++) {
+ XAttr xAttr = (new XAttr.Builder())
+ .setNameSpace(XAttr.NameSpace.SYSTEM)
+ .setName("a" + i)
+ .setValue(new byte[] { (byte) (i*2) })
+ .build();
+ toAdd.add(xAttr);
+ }
+ newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
+ EnumSet.of(XAttrSetFlag.REPLACE));
+ assertEquals("Unexpected number of new XAttrs", 3, newXAttrs.size());
+ for (int i=0; i<3; i++) {
+ assertArrayEquals("Unexpected XAttr value",
+ new byte[] {(byte)(i*2)}, newXAttrs.get(i).getValue());
+ }
+ existingXAttrs = newXAttrs;
+
+ // Sanity test for CREATE+REPLACE
+ toAdd = Lists.newArrayList();
+ for (int i=0; i<4; i++) {
+ toAdd.add(generatedXAttrs.get(i));
+ }
+ newXAttrs = fsdir.setINodeXAttrs(existingXAttrs, toAdd,
+ EnumSet.of(XAttrSetFlag.CREATE, XAttrSetFlag.REPLACE));
+ verifyXAttrsPresent(newXAttrs, 4);
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
index 3af20a7..3d0259e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
@@ -35,6 +35,7 @@
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
import org.apache.hadoop.hdfs.server.namenode.ha.HAState;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.junit.After;
import org.junit.Test;
import org.mockito.Mockito;
@@ -194,4 +195,22 @@
assertFalse(rwLock.isWriteLockedByCurrentThread());
assertEquals(0, rwLock.getWriteHoldCount());
}
+
+ @Test
+ public void testReset() throws Exception {
+ Configuration conf = new Configuration();
+ FSEditLog fsEditLog = Mockito.mock(FSEditLog.class);
+ FSImage fsImage = Mockito.mock(FSImage.class);
+ Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog);
+ FSNamesystem fsn = new FSNamesystem(conf, fsImage);
+ fsn.imageLoadComplete();
+ assertTrue(fsn.isImageLoaded());
+ fsn.clear();
+ assertFalse(fsn.isImageLoaded());
+ final INodeDirectory root = (INodeDirectory) fsn.getFSDirectory()
+ .getINode("/");
+ assertTrue(root.getChildrenList(Snapshot.CURRENT_STATE_ID).isEmpty());
+ fsn.imageLoadComplete();
+ assertTrue(fsn.isImageLoaded());
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
index 0f4a2b8..577d505 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsLimits.java
@@ -19,12 +19,9 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.server.common.Util.fileAsURI;
-import static org.apache.hadoop.util.Time.now;
import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.anyObject;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
@@ -57,7 +54,7 @@
FSEditLog editLog = mock(FSEditLog.class);
doReturn(editLog).when(fsImage).getEditLog();
FSNamesystem fsn = new FSNamesystem(conf, fsImage);
- fsn.getFSDirectory().setReady(fsIsReady);
+ fsn.setImageLoaded(fsIsReady);
return fsn;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
index baa6edb..c88aaf2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
@@ -171,8 +171,6 @@
private static boolean existsInDiffReport(List<DiffReportEntry> entries,
DiffType type, String relativePath) {
for (DiffReportEntry entry : entries) {
- System.out.println("DiffEntry is:" + entry.getType() + "\""
- + new String(entry.getRelativePath()) + "\"");
if ((entry.getType() == type)
&& ((new String(entry.getRelativePath())).compareTo(relativePath) == 0)) {
return true;
@@ -2374,4 +2372,46 @@
// save namespace and restart
restartClusterAndCheckImage(true);
}
+
+ @Test
+ public void testRenameWithOverWrite() throws Exception {
+ final Path root = new Path("/");
+ final Path foo = new Path(root, "foo");
+ final Path file1InFoo = new Path(foo, "file1");
+ final Path file2InFoo = new Path(foo, "file2");
+ final Path file3InFoo = new Path(foo, "file3");
+ DFSTestUtil.createFile(hdfs, file1InFoo, 1L, REPL, SEED);
+ DFSTestUtil.createFile(hdfs, file2InFoo, 1L, REPL, SEED);
+ DFSTestUtil.createFile(hdfs, file3InFoo, 1L, REPL, SEED);
+ final Path bar = new Path(root, "bar");
+ hdfs.mkdirs(bar);
+
+ SnapshotTestHelper.createSnapshot(hdfs, root, "s0");
+ // move file1 from foo to bar
+ final Path fileInBar = new Path(bar, "file1");
+ hdfs.rename(file1InFoo, fileInBar);
+ // rename bar to newDir
+ final Path newDir = new Path(root, "newDir");
+ hdfs.rename(bar, newDir);
+ // move file2 from foo to newDir
+ final Path file2InNewDir = new Path(newDir, "file2");
+ hdfs.rename(file2InFoo, file2InNewDir);
+ // move file3 from foo to newDir and rename it to file1, this will overwrite
+ // the original file1
+ final Path file1InNewDir = new Path(newDir, "file1");
+ hdfs.rename(file3InFoo, file1InNewDir, Rename.OVERWRITE);
+ SnapshotTestHelper.createSnapshot(hdfs, root, "s1");
+
+ SnapshotDiffReport report = hdfs.getSnapshotDiffReport(root, "s0", "s1");
+ LOG.info("DiffList is \n\"" + report.toString() + "\"");
+ List<DiffReportEntry> entries = report.getDiffList();
+ assertEquals(7, entries.size());
+ assertTrue(existsInDiffReport(entries, DiffType.MODIFY, ""));
+ assertTrue(existsInDiffReport(entries, DiffType.MODIFY, foo.getName()));
+ assertTrue(existsInDiffReport(entries, DiffType.DELETE, bar.getName()));
+ assertTrue(existsInDiffReport(entries, DiffType.CREATE, newDir.getName()));
+ assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file1"));
+ assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file2"));
+ assertTrue(existsInDiffReport(entries, DiffType.DELETE, "foo/file3"));
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
index fba48fc..c7b6b7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotBlocksMap.java
@@ -28,12 +28,14 @@
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
@@ -396,4 +398,39 @@
assertEquals(1, blks.length);
assertEquals(BLOCKSIZE, blks[0].getNumBytes());
}
+
+ /**
+ * Make sure that a delete of a non-zero-length file which results in a
+ * zero-length file in a snapshot works.
+ */
+ @Test
+ public void testDeletionOfLaterBlocksWithZeroSizeFirstBlock() throws Exception {
+ final Path foo = new Path("/foo");
+ final Path bar = new Path(foo, "bar");
+ final byte[] testData = "foo bar baz".getBytes();
+
+ // Create a zero-length file.
+ DFSTestUtil.createFile(hdfs, bar, 0, REPLICATION, 0L);
+ assertEquals(0, fsdir.getINode4Write(bar.toString()).asFile().getBlocks().length);
+
+ // Create a snapshot that includes that file.
+ SnapshotTestHelper.createSnapshot(hdfs, foo, "s0");
+
+ // Extend that file.
+ FSDataOutputStream out = hdfs.append(bar);
+ out.write(testData);
+ out.close();
+ INodeFile barNode = fsdir.getINode4Write(bar.toString()).asFile();
+ BlockInfo[] blks = barNode.getBlocks();
+ assertEquals(1, blks.length);
+ assertEquals(testData.length, blks[0].getNumBytes());
+
+ // Delete the file.
+ hdfs.delete(bar, true);
+
+ // Now make sure that the NN can still save an fsimage successfully.
+ cluster.getNameNode().getRpcServer().setSafeMode(
+ SafeModeAction.SAFEMODE_ENTER, false);
+ cluster.getNameNode().getRpcServer().saveNamespace();
+ }
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java
index 87b856e..85c2fda 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestXAttrWithSnapshot.java
@@ -249,10 +249,10 @@
private static void doSnapshotRootRemovalAssertions(Path path,
Path snapshotPath) throws Exception {
Map<String, byte[]> xattrs = hdfs.getXAttrs(path);
- Assert.assertEquals(xattrs.size(), 0);
+ Assert.assertEquals(0, xattrs.size());
xattrs = hdfs.getXAttrs(snapshotPath);
- Assert.assertEquals(xattrs.size(), 2);
+ Assert.assertEquals(2, xattrs.size());
Assert.assertArrayEquals(value1, xattrs.get(name1));
Assert.assertArrayEquals(value2, xattrs.get(name2));
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
index c41b65b..a3561cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
Binary files differ
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
index 54f9876..7e9b8cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
@@ -13,8 +13,8 @@
<TXID>2</TXID>
<DELEGATION_KEY>
<KEY_ID>1</KEY_ID>
- <EXPIRY_DATE>1394849922137</EXPIRY_DATE>
- <KEY>37e1a64049bbef35</KEY>
+ <EXPIRY_DATE>1403590428625</EXPIRY_DATE>
+ <KEY>16f34bfba67b2552</KEY>
</DELEGATION_KEY>
</DATA>
</RECORD>
@@ -24,8 +24,8 @@
<TXID>3</TXID>
<DELEGATION_KEY>
<KEY_ID>2</KEY_ID>
- <EXPIRY_DATE>1394849922140</EXPIRY_DATE>
- <KEY>7c0bf5039242fc54</KEY>
+ <EXPIRY_DATE>1403590428631</EXPIRY_DATE>
+ <KEY>dbe6282854469833</KEY>
</DELEGATION_KEY>
</DATA>
</RECORD>
@@ -37,18 +37,18 @@
<INODEID>16386</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1394158722811</MTIME>
- <ATIME>1394158722811</ATIME>
+ <MTIME>1402899229669</MTIME>
+ <ATIME>1402899229669</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_221786725_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>6</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>8</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -59,13 +59,13 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1394158722832</MTIME>
- <ATIME>1394158722811</ATIME>
+ <MTIME>1402899229711</MTIME>
+ <ATIME>1402899229669</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -78,9 +78,9 @@
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
- <TIMESTAMP>1394158722836</TIMESTAMP>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>8</RPC_CALLID>
+ <TIMESTAMP>1402899229718</TIMESTAMP>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>10</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -89,9 +89,9 @@
<TXID>7</TXID>
<LENGTH>0</LENGTH>
<PATH>/file_moved</PATH>
- <TIMESTAMP>1394158722842</TIMESTAMP>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>9</RPC_CALLID>
+ <TIMESTAMP>1402899229730</TIMESTAMP>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>11</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -101,9 +101,9 @@
<LENGTH>0</LENGTH>
<INODEID>16387</INODEID>
<PATH>/directory_mkdir</PATH>
- <TIMESTAMP>1394158722848</TIMESTAMP>
+ <TIMESTAMP>1402899229748</TIMESTAMP>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>493</MODE>
</PERMISSION_STATUS>
@@ -136,8 +136,8 @@
<TXID>12</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTNAME>snapshot1</SNAPSHOTNAME>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>14</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>16</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -147,8 +147,8 @@
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTOLDNAME>snapshot1</SNAPSHOTOLDNAME>
<SNAPSHOTNEWNAME>snapshot2</SNAPSHOTNEWNAME>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>15</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>17</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -157,8 +157,8 @@
<TXID>14</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTNAME>snapshot2</SNAPSHOTNAME>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>16</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>18</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -169,18 +169,18 @@
<INODEID>16388</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1394158722872</MTIME>
- <ATIME>1394158722872</ATIME>
+ <MTIME>1402899229871</MTIME>
+ <ATIME>1402899229871</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_221786725_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>17</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>19</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -191,13 +191,13 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1394158722874</MTIME>
- <ATIME>1394158722872</ATIME>
+ <MTIME>1402899229881</MTIME>
+ <ATIME>1402899229871</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -253,10 +253,10 @@
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
- <TIMESTAMP>1394158722890</TIMESTAMP>
+ <TIMESTAMP>1402899229963</TIMESTAMP>
<OPTIONS>NONE</OPTIONS>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>24</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>26</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -267,18 +267,18 @@
<INODEID>16389</INODEID>
<PATH>/file_concat_target</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1394158722895</MTIME>
- <ATIME>1394158722895</ATIME>
+ <MTIME>1402899229981</MTIME>
+ <ATIME>1402899229981</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_221786725_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>26</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>28</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -383,8 +383,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_target</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1394158722986</MTIME>
- <ATIME>1394158722895</ATIME>
+ <MTIME>1402899230219</MTIME>
+ <ATIME>1402899229981</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -404,7 +404,7 @@
<GENSTAMP>1003</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -418,18 +418,18 @@
<INODEID>16390</INODEID>
<PATH>/file_concat_0</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1394158722989</MTIME>
- <ATIME>1394158722989</ATIME>
+ <MTIME>1402899230235</MTIME>
+ <ATIME>1402899230235</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_221786725_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>39</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>41</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -534,8 +534,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_0</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1394158723010</MTIME>
- <ATIME>1394158722989</ATIME>
+ <MTIME>1402899230307</MTIME>
+ <ATIME>1402899230235</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -555,7 +555,7 @@
<GENSTAMP>1006</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -569,18 +569,18 @@
<INODEID>16391</INODEID>
<PATH>/file_concat_1</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1394158723012</MTIME>
- <ATIME>1394158723012</ATIME>
+ <MTIME>1402899230320</MTIME>
+ <ATIME>1402899230320</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_221786725_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>51</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>53</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -685,8 +685,8 @@
<INODEID>0</INODEID>
<PATH>/file_concat_1</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1394158723035</MTIME>
- <ATIME>1394158723012</ATIME>
+ <MTIME>1402899230383</MTIME>
+ <ATIME>1402899230320</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -706,7 +706,7 @@
<GENSTAMP>1009</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -718,13 +718,13 @@
<TXID>56</TXID>
<LENGTH>0</LENGTH>
<TRG>/file_concat_target</TRG>
- <TIMESTAMP>1394158723039</TIMESTAMP>
+ <TIMESTAMP>1402899230394</TIMESTAMP>
<SOURCES>
<SOURCE1>/file_concat_0</SOURCE1>
<SOURCE2>/file_concat_1</SOURCE2>
</SOURCES>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>62</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>64</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -735,15 +735,15 @@
<INODEID>16392</INODEID>
<PATH>/file_symlink</PATH>
<VALUE>/file_concat_target</VALUE>
- <MTIME>1394158723044</MTIME>
- <ATIME>1394158723044</ATIME>
+ <MTIME>1402899230406</MTIME>
+ <ATIME>1402899230406</ATIME>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>511</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>63</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>65</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -754,18 +754,18 @@
<INODEID>16393</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1394158723047</MTIME>
- <ATIME>1394158723047</ATIME>
+ <MTIME>1402899230413</MTIME>
+ <ATIME>1402899230413</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_221786725_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_1233039831_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>64</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>66</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -821,7 +821,7 @@
<OPCODE>OP_REASSIGN_LEASE</OPCODE>
<DATA>
<TXID>64</TXID>
- <LEASEHOLDER>DFSClient_NONMAPREDUCE_221786725_1</LEASEHOLDER>
+ <LEASEHOLDER>DFSClient_NONMAPREDUCE_1233039831_1</LEASEHOLDER>
<PATH>/hard-lease-recovery-test</PATH>
<NEWHOLDER>HDFS_NameNode</NEWHOLDER>
</DATA>
@@ -834,8 +834,8 @@
<INODEID>0</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1394158725708</MTIME>
- <ATIME>1394158723047</ATIME>
+ <MTIME>1402899232526</MTIME>
+ <ATIME>1402899230413</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -845,7 +845,7 @@
<GENSTAMP>1011</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>andrew</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -856,13 +856,13 @@
<DATA>
<TXID>66</TXID>
<POOLNAME>pool1</POOLNAME>
- <OWNERNAME>jing</OWNERNAME>
- <GROUPNAME>staff</GROUPNAME>
+ <OWNERNAME>andrew</OWNERNAME>
+ <GROUPNAME>andrew</GROUPNAME>
<MODE>493</MODE>
<LIMIT>9223372036854775807</LIMIT>
<MAXRELATIVEEXPIRY>2305843009213693951</MAXRELATIVEEXPIRY>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>71</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>73</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -871,8 +871,8 @@
<TXID>67</TXID>
<POOLNAME>pool1</POOLNAME>
<LIMIT>99</LIMIT>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>72</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>74</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -883,9 +883,9 @@
<PATH>/path</PATH>
<REPLICATION>1</REPLICATION>
<POOL>pool1</POOL>
- <EXPIRATION>2305844403372420029</EXPIRATION>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>73</RPC_CALLID>
+ <EXPIRATION>2305844412112927450</EXPIRATION>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>75</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -894,8 +894,8 @@
<TXID>69</TXID>
<ID>1</ID>
<REPLICATION>2</REPLICATION>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>74</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>76</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -903,8 +903,8 @@
<DATA>
<TXID>70</TXID>
<ID>1</ID>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>75</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>77</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -912,8 +912,8 @@
<DATA>
<TXID>71</TXID>
<POOLNAME>pool1</POOLNAME>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
- <RPC_CALLID>76</RPC_CALLID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>78</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -921,51 +921,91 @@
<DATA>
<TXID>72</TXID>
<SRC>/file_concat_target</SRC>
- </DATA>
- </RECORD>
- <RECORD>
- <OPCODE>OP_ROLLING_UPGRADE_START</OPCODE>
- <DATA>
- <TXID>73</TXID>
- <STARTTIME>1394158726098</STARTTIME>
- </DATA>
- </RECORD>
- <RECORD>
- <OPCODE>OP_ROLLING_UPGRADE_FINALIZE</OPCODE>
- <DATA>
- <TXID>74</TXID>
- <FINALIZETIME>1394158726098</FINALIZETIME>
+ <ENTRY>
+ <SCOPE>ACCESS</SCOPE>
+ <TYPE>USER</TYPE>
+ <PERM>rw-</PERM>
+ </ENTRY>
+ <ENTRY>
+ <SCOPE>ACCESS</SCOPE>
+ <TYPE>USER</TYPE>
+ <NAME>user</NAME>
+ <PERM>rw-</PERM>
+ </ENTRY>
+ <ENTRY>
+ <SCOPE>ACCESS</SCOPE>
+ <TYPE>GROUP</TYPE>
+ <PERM>-w-</PERM>
+ </ENTRY>
+ <ENTRY>
+ <SCOPE>ACCESS</SCOPE>
+ <TYPE>MASK</TYPE>
+ <PERM>rw-</PERM>
+ </ENTRY>
+ <ENTRY>
+ <SCOPE>ACCESS</SCOPE>
+ <TYPE>OTHER</TYPE>
+ <PERM>---</PERM>
+ </ENTRY>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_XATTR</OPCODE>
<DATA>
- <TXID>75</TXID>
+ <TXID>73</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
<NAME>a1</NAME>
<VALUE>0x313233</VALUE>
</XATTR>
- <RPC_CLIENTID>9b85a845-bbfa-42f6-8a16-c433614b8eb9</RPC_CLIENTID>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
<RPC_CALLID>80</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
+ <OPCODE>OP_SET_XATTR</OPCODE>
+ <DATA>
+ <TXID>74</TXID>
+ <SRC>/file_concat_target</SRC>
+ <XATTR>
+ <NAMESPACE>USER</NAMESPACE>
+ <NAME>a2</NAME>
+ <VALUE>0x373839</VALUE>
+ </XATTR>
+ <RPC_CLIENTID>e03f4a52-3d85-4e05-8942-286185e639bd</RPC_CLIENTID>
+ <RPC_CALLID>81</RPC_CALLID>
+ </DATA>
+ </RECORD>
+ <RECORD>
<OPCODE>OP_REMOVE_XATTR</OPCODE>
<DATA>
- <TXID>76</TXID>
+ <TXID>75</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
- <NAME>a1</NAME>
+ <NAME>a2</NAME>
</XATTR>
</DATA>
</RECORD>
<RECORD>
+ <OPCODE>OP_ROLLING_UPGRADE_START</OPCODE>
+ <DATA>
+ <TXID>76</TXID>
+ <STARTTIME>1402899233646</STARTTIME>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_ROLLING_UPGRADE_FINALIZE</OPCODE>
+ <DATA>
+ <TXID>77</TXID>
+ <FINALIZETIME>1402899233647</FINALIZETIME>
+ </DATA>
+ </RECORD>
+ <RECORD>
<OPCODE>OP_END_LOG_SEGMENT</OPCODE>
<DATA>
- <TXID>77</TXID>
+ <TXID>78</TXID>
</DATA>
</RECORD>
</EDITS>
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index f3c4d0a..8feab76 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -213,6 +213,12 @@
MAPREDUCE-5834. Increased test-timeouts in TestGridMixClasses to avoid
occassional failures. (Mit Desai via vinodkv)
+ MAPREDUCE-5896. InputSplits should indicate which locations have the block
+ cached in memory. (Sandy Ryza via kasha)
+
+ MAPREDUCE-5844. Add a configurable delay to reducer-preemption.
+ (Maysam Yabandeh via kasha)
+
OPTIMIZATIONS
BUG FIXES
diff --git a/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml b/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml
index 5bdd618..dd4892b 100644
--- a/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml
+++ b/hadoop-mapreduce-project/dev-support/findbugs-exclude.xml
@@ -475,8 +475,8 @@
<Match>
<Class name="org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator" />
<Or>
- <Field name="mapResourceReqt" />
- <Field name="reduceResourceReqt" />
+ <Field name="mapResourceRequest" />
+ <Field name="reduceResourceRequest" />
<Field name="maxReduceRampupLimit" />
<Field name="reduceSlowStart" />
</Or>
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index b9d283f..11bc406 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -73,6 +73,7 @@
import org.apache.hadoop.yarn.client.api.NMTokenCache;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.RackResolver;
import com.google.common.annotations.VisibleForTesting;
@@ -143,15 +144,21 @@
private int lastCompletedTasks = 0;
private boolean recalculateReduceSchedule = false;
- private int mapResourceReqt;//memory
- private int reduceResourceReqt;//memory
+ private int mapResourceRequest;//memory
+ private int reduceResourceRequest;//memory
private boolean reduceStarted = false;
private float maxReduceRampupLimit = 0;
private float maxReducePreemptionLimit = 0;
+ /**
+ * after this threshold, if the container request is not allocated, it is
+ * considered delayed.
+ */
+ private long allocationDelayThresholdMs = 0;
private float reduceSlowStart = 0;
private long retryInterval;
private long retrystartTime;
+ private Clock clock;
private final AMPreemptionPolicy preemptionPolicy;
@@ -166,6 +173,7 @@
super(clientService, context);
this.preemptionPolicy = preemptionPolicy;
this.stopped = new AtomicBoolean(false);
+ this.clock = context.getClock();
}
@Override
@@ -180,6 +188,9 @@
maxReducePreemptionLimit = conf.getFloat(
MRJobConfig.MR_AM_JOB_REDUCE_PREEMPTION_LIMIT,
MRJobConfig.DEFAULT_MR_AM_JOB_REDUCE_PREEMPTION_LIMIT);
+ allocationDelayThresholdMs = conf.getInt(
+ MRJobConfig.MR_JOB_REDUCER_PREEMPT_DELAY_SEC,
+ MRJobConfig.DEFAULT_MR_JOB_REDUCER_PREEMPT_DELAY_SEC) * 1000;//sec -> ms
RackResolver.init(conf);
retryInterval = getConfig().getLong(MRJobConfig.MR_AM_TO_RM_WAIT_INTERVAL_MS,
MRJobConfig.DEFAULT_MR_AM_TO_RM_WAIT_INTERVAL_MS);
@@ -246,7 +257,7 @@
getJob().getTotalMaps(), completedMaps,
scheduledRequests.maps.size(), scheduledRequests.reduces.size(),
assignedRequests.maps.size(), assignedRequests.reduces.size(),
- mapResourceReqt, reduceResourceReqt,
+ mapResourceRequest, reduceResourceRequest,
pendingReduces.size(),
maxReduceRampupLimit, reduceSlowStart);
recalculateReduceSchedule = false;
@@ -268,6 +279,18 @@
scheduleStats.log("Final Stats: ");
}
+ @Private
+ @VisibleForTesting
+ AssignedRequests getAssignedRequests() {
+ return assignedRequests;
+ }
+
+ @Private
+ @VisibleForTesting
+ ScheduledRequests getScheduledRequests() {
+ return scheduledRequests;
+ }
+
public boolean getIsReduceStarted() {
return reduceStarted;
}
@@ -303,16 +326,16 @@
int supportedMaxContainerCapability =
getMaxContainerCapability().getMemory();
if (reqEvent.getAttemptID().getTaskId().getTaskType().equals(TaskType.MAP)) {
- if (mapResourceReqt == 0) {
- mapResourceReqt = reqEvent.getCapability().getMemory();
+ if (mapResourceRequest == 0) {
+ mapResourceRequest = reqEvent.getCapability().getMemory();
eventHandler.handle(new JobHistoryEvent(jobId,
new NormalizedResourceEvent(org.apache.hadoop.mapreduce.TaskType.MAP,
- mapResourceReqt)));
- LOG.info("mapResourceReqt:"+mapResourceReqt);
- if (mapResourceReqt > supportedMaxContainerCapability) {
+ mapResourceRequest)));
+ LOG.info("mapResourceRequest:"+ mapResourceRequest);
+ if (mapResourceRequest > supportedMaxContainerCapability) {
String diagMsg = "MAP capability required is more than the supported " +
- "max container capability in the cluster. Killing the Job. mapResourceReqt: " +
- mapResourceReqt + " maxContainerCapability:" + supportedMaxContainerCapability;
+ "max container capability in the cluster. Killing the Job. mapResourceRequest: " +
+ mapResourceRequest + " maxContainerCapability:" + supportedMaxContainerCapability;
LOG.info(diagMsg);
eventHandler.handle(new JobDiagnosticsUpdateEvent(
jobId, diagMsg));
@@ -320,20 +343,20 @@
}
}
//set the rounded off memory
- reqEvent.getCapability().setMemory(mapResourceReqt);
+ reqEvent.getCapability().setMemory(mapResourceRequest);
scheduledRequests.addMap(reqEvent);//maps are immediately scheduled
} else {
- if (reduceResourceReqt == 0) {
- reduceResourceReqt = reqEvent.getCapability().getMemory();
+ if (reduceResourceRequest == 0) {
+ reduceResourceRequest = reqEvent.getCapability().getMemory();
eventHandler.handle(new JobHistoryEvent(jobId,
new NormalizedResourceEvent(
org.apache.hadoop.mapreduce.TaskType.REDUCE,
- reduceResourceReqt)));
- LOG.info("reduceResourceReqt:"+reduceResourceReqt);
- if (reduceResourceReqt > supportedMaxContainerCapability) {
+ reduceResourceRequest)));
+ LOG.info("reduceResourceRequest:"+ reduceResourceRequest);
+ if (reduceResourceRequest > supportedMaxContainerCapability) {
String diagMsg = "REDUCE capability required is more than the " +
"supported max container capability in the cluster. Killing the " +
- "Job. reduceResourceReqt: " + reduceResourceReqt +
+ "Job. reduceResourceRequest: " + reduceResourceRequest +
" maxContainerCapability:" + supportedMaxContainerCapability;
LOG.info(diagMsg);
eventHandler.handle(new JobDiagnosticsUpdateEvent(
@@ -342,7 +365,7 @@
}
}
//set the rounded off memory
- reqEvent.getCapability().setMemory(reduceResourceReqt);
+ reqEvent.getCapability().setMemory(reduceResourceRequest);
if (reqEvent.getEarlierAttemptFailed()) {
//add to the front of queue for fail fast
pendingReduces.addFirst(new ContainerRequest(reqEvent, PRIORITY_REDUCE));
@@ -394,8 +417,22 @@
return host;
}
- private void preemptReducesIfNeeded() {
- if (reduceResourceReqt == 0) {
+ @Private
+ @VisibleForTesting
+ synchronized void setReduceResourceRequest(int mem) {
+ this.reduceResourceRequest = mem;
+ }
+
+ @Private
+ @VisibleForTesting
+ synchronized void setMapResourceRequest(int mem) {
+ this.mapResourceRequest = mem;
+ }
+
+ @Private
+ @VisibleForTesting
+ void preemptReducesIfNeeded() {
+ if (reduceResourceRequest == 0) {
return; //no reduces
}
//check if reduces have taken over the whole cluster and there are
@@ -403,9 +440,9 @@
if (scheduledRequests.maps.size() > 0) {
int memLimit = getMemLimit();
int availableMemForMap = memLimit - ((assignedRequests.reduces.size() -
- assignedRequests.preemptionWaitingReduces.size()) * reduceResourceReqt);
+ assignedRequests.preemptionWaitingReduces.size()) * reduceResourceRequest);
//availableMemForMap must be sufficient to run atleast 1 map
- if (availableMemForMap < mapResourceReqt) {
+ if (availableMemForMap < mapResourceRequest) {
//to make sure new containers are given to maps and not reduces
//ramp down all scheduled reduces if any
//(since reduces are scheduled at higher priority than maps)
@@ -414,22 +451,40 @@
pendingReduces.add(req);
}
scheduledRequests.reduces.clear();
-
- //preempt for making space for at least one map
- int premeptionLimit = Math.max(mapResourceReqt,
- (int) (maxReducePreemptionLimit * memLimit));
-
- int preemptMem = Math.min(scheduledRequests.maps.size() * mapResourceReqt,
- premeptionLimit);
-
- int toPreempt = (int) Math.ceil((float) preemptMem/reduceResourceReqt);
- toPreempt = Math.min(toPreempt, assignedRequests.reduces.size());
-
- LOG.info("Going to preempt " + toPreempt + " due to lack of space for maps");
- assignedRequests.preemptReduce(toPreempt);
+
+ //do further checking to find the number of map requests that were
+ //hanging around for a while
+ int hangingMapRequests = getNumOfHangingRequests(scheduledRequests.maps);
+ if (hangingMapRequests > 0) {
+ //preempt for making space for at least one map
+ int premeptionLimit = Math.max(mapResourceRequest,
+ (int) (maxReducePreemptionLimit * memLimit));
+
+ int preemptMem = Math.min(hangingMapRequests * mapResourceRequest,
+ premeptionLimit);
+
+ int toPreempt = (int) Math.ceil((float) preemptMem / reduceResourceRequest);
+ toPreempt = Math.min(toPreempt, assignedRequests.reduces.size());
+
+ LOG.info("Going to preempt " + toPreempt + " due to lack of space for maps");
+ assignedRequests.preemptReduce(toPreempt);
+ }
}
}
}
+
+ private int getNumOfHangingRequests(Map<TaskAttemptId, ContainerRequest> requestMap) {
+ if (allocationDelayThresholdMs <= 0)
+ return requestMap.size();
+ int hangingRequests = 0;
+ long currTime = clock.getTime();
+ for (ContainerRequest request: requestMap.values()) {
+ long delay = currTime - request.requestTimeMs;
+ if (delay > allocationDelayThresholdMs)
+ hangingRequests++;
+ }
+ return hangingRequests;
+ }
@Private
public void scheduleReduces(
@@ -715,11 +770,13 @@
@Private
public int getMemLimit() {
int headRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;
- return headRoom + assignedRequests.maps.size() * mapResourceReqt +
- assignedRequests.reduces.size() * reduceResourceReqt;
+ return headRoom + assignedRequests.maps.size() * mapResourceRequest +
+ assignedRequests.reduces.size() * reduceResourceRequest;
}
-
- private class ScheduledRequests {
+
+ @Private
+ @VisibleForTesting
+ class ScheduledRequests {
private final LinkedList<TaskAttemptId> earlierFailedMaps =
new LinkedList<TaskAttemptId>();
@@ -729,7 +786,8 @@
new HashMap<String, LinkedList<TaskAttemptId>>();
private final Map<String, LinkedList<TaskAttemptId>> mapsRackMapping =
new HashMap<String, LinkedList<TaskAttemptId>>();
- private final Map<TaskAttemptId, ContainerRequest> maps =
+ @VisibleForTesting
+ final Map<TaskAttemptId, ContainerRequest> maps =
new LinkedHashMap<TaskAttemptId, ContainerRequest>();
private final LinkedHashMap<TaskAttemptId, ContainerRequest> reduces =
@@ -825,22 +883,22 @@
int allocatedMemory = allocated.getResource().getMemory();
if (PRIORITY_FAST_FAIL_MAP.equals(priority)
|| PRIORITY_MAP.equals(priority)) {
- if (allocatedMemory < mapResourceReqt
+ if (allocatedMemory < mapResourceRequest
|| maps.isEmpty()) {
LOG.info("Cannot assign container " + allocated
+ " for a map as either "
- + " container memory less than required " + mapResourceReqt
+ + " container memory less than required " + mapResourceRequest
+ " or no pending map tasks - maps.isEmpty="
+ maps.isEmpty());
isAssignable = false;
}
}
else if (PRIORITY_REDUCE.equals(priority)) {
- if (allocatedMemory < reduceResourceReqt
+ if (allocatedMemory < reduceResourceRequest
|| reduces.isEmpty()) {
LOG.info("Cannot assign container " + allocated
+ " for a reduce as either "
- + " container memory less than required " + reduceResourceReqt
+ + " container memory less than required " + reduceResourceRequest
+ " or no pending reduce tasks - reduces.isEmpty="
+ reduces.isEmpty());
isAssignable = false;
@@ -1119,14 +1177,18 @@
}
}
- private class AssignedRequests {
+ @Private
+ @VisibleForTesting
+ class AssignedRequests {
private final Map<ContainerId, TaskAttemptId> containerToAttemptMap =
new HashMap<ContainerId, TaskAttemptId>();
private final LinkedHashMap<TaskAttemptId, Container> maps =
new LinkedHashMap<TaskAttemptId, Container>();
- private final LinkedHashMap<TaskAttemptId, Container> reduces =
+ @VisibleForTesting
+ final LinkedHashMap<TaskAttemptId, Container> reduces =
new LinkedHashMap<TaskAttemptId, Container>();
- private final Set<TaskAttemptId> preemptionWaitingReduces =
+ @VisibleForTesting
+ final Set<TaskAttemptId> preemptionWaitingReduces =
new HashSet<TaskAttemptId>();
void add(Container container, TaskAttemptId tId) {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
index a9b5ce5..1824211 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
@@ -29,8 +29,10 @@
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
@@ -96,6 +98,8 @@
super(clientService, context);
}
+ @Private
+ @VisibleForTesting
static class ContainerRequest {
final TaskAttemptId attemptID;
final Resource capability;
@@ -103,20 +107,39 @@
final String[] racks;
//final boolean earlierAttemptFailed;
final Priority priority;
-
+ /**
+ * the time when this request object was formed; can be used to avoid
+ * aggressive preemption for recently placed requests
+ */
+ final long requestTimeMs;
+
public ContainerRequest(ContainerRequestEvent event, Priority priority) {
this(event.getAttemptID(), event.getCapability(), event.getHosts(),
event.getRacks(), priority);
}
-
+
+ public ContainerRequest(ContainerRequestEvent event, Priority priority,
+ long requestTimeMs) {
+ this(event.getAttemptID(), event.getCapability(), event.getHosts(),
+ event.getRacks(), priority, requestTimeMs);
+ }
+
public ContainerRequest(TaskAttemptId attemptID,
- Resource capability, String[] hosts, String[] racks,
- Priority priority) {
+ Resource capability, String[] hosts, String[] racks,
+ Priority priority) {
+ this(attemptID, capability, hosts, racks, priority,
+ System.currentTimeMillis());
+ }
+
+ public ContainerRequest(TaskAttemptId attemptID,
+ Resource capability, String[] hosts, String[] racks,
+ Priority priority, long requestTimeMs) {
this.attemptID = attemptID;
this.capability = capability;
this.hosts = hosts;
this.racks = racks;
this.priority = priority;
+ this.requestTimeMs = requestTimeMs;
}
public String toString() {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
similarity index 93%
rename from hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
rename to hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index 9c04187..4c74d7b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hadoop.mapreduce.v2.app;
+package org.apache.hadoop.mapreduce.v2.app.rm;
import static org.mockito.Matchers.anyFloat;
import static org.mockito.Matchers.anyInt;
@@ -40,6 +40,10 @@
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.ClusterInfo;
+import org.apache.hadoop.mapreduce.v2.app.ControlledClock;
+import org.apache.hadoop.mapreduce.v2.app.MRApp;
import org.junit.Assert;
import org.apache.commons.logging.Log;
@@ -65,10 +69,6 @@
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent;
import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl;
import org.apache.hadoop.mapreduce.v2.app.job.impl.TaskAttemptImpl;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerFailedEvent;
-import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent;
-import org.apache.hadoop.mapreduce.v2.app.rm.RMContainerAllocator;
import org.apache.hadoop.mapreduce.v2.app.rm.preemption.NoopAMPreemptionPolicy;
import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -80,6 +80,7 @@
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -422,6 +423,115 @@
killEventMessage.contains(RMContainerAllocator.RAMPDOWN_DIAGNOSTIC));
}
+ @Test(timeout = 30000)
+ public void testPreemptReducers() throws Exception {
+ LOG.info("Running testPreemptReducers");
+
+ Configuration conf = new Configuration();
+ MyResourceManager rm = new MyResourceManager(conf);
+ rm.start();
+ DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
+ .getDispatcher();
+
+ // Submit the application
+ RMApp app = rm.submitApp(1024);
+ dispatcher.await();
+
+ MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
+ amNodeManager.nodeHeartbeat(true);
+ dispatcher.await();
+
+ ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
+ .getAppAttemptId();
+ rm.sendAMLaunched(appAttemptId);
+ dispatcher.await();
+
+ JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
+ Job mockJob = mock(Job.class);
+ when(mockJob.getReport()).thenReturn(
+ MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
+ 0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));
+ MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
+ appAttemptId, mockJob, new SystemClock());
+ allocator.setMapResourceRequest(1024);
+ allocator.setReduceResourceRequest(1024);
+ RMContainerAllocator.AssignedRequests assignedRequests =
+ allocator.getAssignedRequests();
+ RMContainerAllocator.ScheduledRequests scheduledRequests =
+ allocator.getScheduledRequests();
+ ContainerRequestEvent event1 =
+ createReq(jobId, 1, 2048, new String[] { "h1" }, false, false);
+ scheduledRequests.maps.put(mock(TaskAttemptId.class),
+ new RMContainerRequestor.ContainerRequest(event1, null));
+ assignedRequests.reduces.put(mock(TaskAttemptId.class),
+ mock(Container.class));
+
+ allocator.preemptReducesIfNeeded();
+ Assert.assertEquals("The reducer is not preempted",
+ 1, assignedRequests.preemptionWaitingReduces.size());
+ }
+
+ @Test(timeout = 30000)
+ public void testNonAggressivelyPreemptReducers() throws Exception {
+ LOG.info("Running testPreemptReducers");
+
+ final int preemptThreshold = 2; //sec
+ Configuration conf = new Configuration();
+ conf.setInt(
+ MRJobConfig.MR_JOB_REDUCER_PREEMPT_DELAY_SEC,
+ preemptThreshold);
+
+ MyResourceManager rm = new MyResourceManager(conf);
+ rm.start();
+ DrainDispatcher dispatcher = (DrainDispatcher) rm.getRMContext()
+ .getDispatcher();
+
+ // Submit the application
+ RMApp app = rm.submitApp(1024);
+ dispatcher.await();
+
+ MockNM amNodeManager = rm.registerNode("amNM:1234", 2048);
+ amNodeManager.nodeHeartbeat(true);
+ dispatcher.await();
+
+ ApplicationAttemptId appAttemptId = app.getCurrentAppAttempt()
+ .getAppAttemptId();
+ rm.sendAMLaunched(appAttemptId);
+ dispatcher.await();
+
+ JobId jobId = MRBuilderUtils.newJobId(appAttemptId.getApplicationId(), 0);
+ Job mockJob = mock(Job.class);
+ when(mockJob.getReport()).thenReturn(
+ MRBuilderUtils.newJobReport(jobId, "job", "user", JobState.RUNNING, 0,
+ 0, 0, 0, 0, 0, 0, "jobfile", null, false, ""));
+ ControlledClock clock = new ControlledClock(null);
+ clock.setTime(1);
+ MyContainerAllocator allocator = new MyContainerAllocator(rm, conf,
+ appAttemptId, mockJob, clock);
+ allocator.setMapResourceRequest(1024);
+ allocator.setReduceResourceRequest(1024);
+ RMContainerAllocator.AssignedRequests assignedRequests =
+ allocator.getAssignedRequests();
+ RMContainerAllocator.ScheduledRequests scheduledRequests =
+ allocator.getScheduledRequests();
+ ContainerRequestEvent event1 =
+ createReq(jobId, 1, 2048, new String[] { "h1" }, false, false);
+ scheduledRequests.maps.put(mock(TaskAttemptId.class),
+ new RMContainerRequestor.ContainerRequest(event1, null, clock.getTime()));
+ assignedRequests.reduces.put(mock(TaskAttemptId.class),
+ mock(Container.class));
+
+ clock.setTime(clock.getTime() + 1);
+ allocator.preemptReducesIfNeeded();
+ Assert.assertEquals("The reducer is aggressively preeempted", 0,
+ assignedRequests.preemptionWaitingReduces.size());
+
+ clock.setTime(clock.getTime() + (preemptThreshold) * 1000);
+ allocator.preemptReducesIfNeeded();
+ Assert.assertEquals("The reducer is not preeempted", 1,
+ assignedRequests.preemptionWaitingReduces.size());
+ }
+
@Test
public void testMapReduceScheduling() throws Exception {
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
index 9863427..0ae5671 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
@@ -295,6 +295,15 @@
String[] hosts) {
return new FileSplit(file, start, length, hosts);
}
+
+ /**
+ * A factory that makes the split for this class. It can be overridden
+ * by sub-classes to make sub-types
+ */
+ protected FileSplit makeSplit(Path file, long start, long length,
+ String[] hosts, String[] inMemoryHosts) {
+ return new FileSplit(file, start, length, hosts, inMemoryHosts);
+ }
/** Splits files returned by {@link #listStatus(JobConf)} when
* they're too big.*/
@@ -337,22 +346,22 @@
long bytesRemaining = length;
while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {
- String[] splitHosts = getSplitHosts(blkLocations,
+ String[][] splitHosts = getSplitHostsAndCachedHosts(blkLocations,
length-bytesRemaining, splitSize, clusterMap);
splits.add(makeSplit(path, length-bytesRemaining, splitSize,
- splitHosts));
+ splitHosts[0], splitHosts[1]));
bytesRemaining -= splitSize;
}
if (bytesRemaining != 0) {
- String[] splitHosts = getSplitHosts(blkLocations, length
+ String[][] splitHosts = getSplitHostsAndCachedHosts(blkLocations, length
- bytesRemaining, bytesRemaining, clusterMap);
splits.add(makeSplit(path, length - bytesRemaining, bytesRemaining,
- splitHosts));
+ splitHosts[0], splitHosts[1]));
}
} else {
- String[] splitHosts = getSplitHosts(blkLocations,0,length,clusterMap);
- splits.add(makeSplit(path, 0, length, splitHosts));
+ String[][] splitHosts = getSplitHostsAndCachedHosts(blkLocations,0,length,clusterMap);
+ splits.add(makeSplit(path, 0, length, splitHosts[0], splitHosts[1]));
}
} else {
//Create empty hosts array for zero length files
@@ -538,10 +547,30 @@
* @param blkLocations The list of block locations
* @param offset
* @param splitSize
- * @return array of hosts that contribute most to this split
+ * @return an array of hosts that contribute most to this split
* @throws IOException
*/
protected String[] getSplitHosts(BlockLocation[] blkLocations,
+ long offset, long splitSize, NetworkTopology clusterMap) throws IOException {
+ return getSplitHostsAndCachedHosts(blkLocations, offset, splitSize,
+ clusterMap)[0];
+ }
+
+ /**
+ * This function identifies and returns the hosts that contribute
+ * most for a given split. For calculating the contribution, rack
+ * locality is treated on par with host locality, so hosts from racks
+ * that contribute the most are preferred over hosts on racks that
+ * contribute less
+ * @param blkLocations The list of block locations
+ * @param offset
+ * @param splitSize
+ * @return two arrays - one of hosts that contribute most to this split, and
+ * one of hosts that contribute most to this split that have the data
+ * cached on them
+ * @throws IOException
+ */
+ private String[][] getSplitHostsAndCachedHosts(BlockLocation[] blkLocations,
long offset, long splitSize, NetworkTopology clusterMap)
throws IOException {
@@ -552,7 +581,8 @@
//If this is the only block, just return
if (bytesInThisBlock >= splitSize) {
- return blkLocations[startIndex].getHosts();
+ return new String[][] { blkLocations[startIndex].getHosts(),
+ blkLocations[startIndex].getCachedHosts() };
}
long bytesInFirstBlock = bytesInThisBlock;
@@ -639,7 +669,9 @@
} // for all indices
- return identifyHosts(allTopos.length, racksMap);
+ // We don't yet support cached hosts when bytesInThisBlock > splitSize
+ return new String[][] { identifyHosts(allTopos.length, racksMap),
+ new String[0]};
}
private String[] identifyHosts(int replicationFactor,
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileSplit.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileSplit.java
index fb1c651..c38f2f7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileSplit.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileSplit.java
@@ -24,6 +24,7 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.fs.Path;
/** A section of an input file. Returned by {@link
@@ -33,7 +34,7 @@
@InterfaceAudience.Public
@InterfaceStability.Stable
public class FileSplit extends org.apache.hadoop.mapreduce.InputSplit
- implements InputSplit {
+ implements InputSplitWithLocationInfo {
org.apache.hadoop.mapreduce.lib.input.FileSplit fs;
protected FileSplit() {
fs = new org.apache.hadoop.mapreduce.lib.input.FileSplit();
@@ -62,6 +63,20 @@
length, hosts);
}
+ /** Constructs a split with host information
+ *
+ * @param file the file name
+ * @param start the position of the first byte in the file to process
+ * @param length the number of bytes in the file to process
+ * @param hosts the list of hosts containing the block, possibly null
+ * @param inMemoryHosts the list of hosts containing the block in memory
+ */
+ public FileSplit(Path file, long start, long length, String[] hosts,
+ String[] inMemoryHosts) {
+ fs = new org.apache.hadoop.mapreduce.lib.input.FileSplit(file, start,
+ length, hosts, inMemoryHosts);
+ }
+
public FileSplit(org.apache.hadoop.mapreduce.lib.input.FileSplit fs) {
this.fs = fs;
}
@@ -92,4 +107,9 @@
return fs.getLocations();
}
+ @Override
+ @Evolving
+ public SplitLocationInfo[] getLocationInfo() throws IOException {
+ return fs.getLocationInfo();
+ }
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/InputSplitWithLocationInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/InputSplitWithLocationInfo.java
new file mode 100644
index 0000000..bb95882
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/InputSplitWithLocationInfo.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
+@Public
+@Evolving
+public interface InputSplitWithLocationInfo extends InputSplit {
+ /**
+ * Gets info about which nodes the input split is stored on and how it is
+ * stored at each location.
+ *
+ * @return list of <code>SplitLocationInfo</code>s describing how the split
+ * data is stored at each location. A null value indicates that all the
+ * locations have the data stored on disk.
+ * @throws IOException
+ */
+ SplitLocationInfo[] getLocationInfo() throws IOException;
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SplitLocationInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SplitLocationInfo.java
new file mode 100644
index 0000000..a8e69fb
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/SplitLocationInfo.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+
+@Public
+@Evolving
+public class SplitLocationInfo {
+ private boolean inMemory;
+ private String location;
+
+ public SplitLocationInfo(String location, boolean inMemory) {
+ this.location = location;
+ this.inMemory = inMemory;
+ }
+
+ public boolean isOnDisk() {
+ return true;
+ }
+
+ public boolean isInMemory() {
+ return inMemory;
+ }
+
+ public String getLocation() {
+ return location;
+ }
+}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/InputSplit.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/InputSplit.java
index 95d4a8c..515b423 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/InputSplit.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/InputSplit.java
@@ -22,6 +22,8 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.mapred.SplitLocationInfo;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.RecordReader;
@@ -51,10 +53,25 @@
/**
* Get the list of nodes by name where the data for the split would be local.
* The locations do not need to be serialized.
+ *
* @return a new array of the node nodes.
* @throws IOException
* @throws InterruptedException
*/
public abstract
String[] getLocations() throws IOException, InterruptedException;
+
+ /**
+ * Gets info about which nodes the input split is stored on and how it is
+ * stored at each location.
+ *
+ * @return list of <code>SplitLocationInfo</code>s describing how the split
+ * data is stored at each location. A null value indicates that all the
+ * locations have the data stored on disk.
+ * @throws IOException
+ */
+ @Evolving
+ public SplitLocationInfo[] getLocationInfo() throws IOException {
+ return null;
+ }
}
\ No newline at end of file
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 33d79ee..4795af7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -579,7 +579,17 @@
MR_AM_PREFIX + "history.use-batched-flush.queue-size.threshold";
public static final int DEFAULT_MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD =
50;
-
+
+ /**
+ * The threshold in terms of seconds after which an unsatisfied mapper request
+ * triggers reducer preemption to free space. Default 0 implies that the reduces
+ * should be preempted immediately after allocation if there is currently no
+ * room for newly allocated mappers.
+ */
+ public static final String MR_JOB_REDUCER_PREEMPT_DELAY_SEC =
+ "mapreduce.job.reducer.preempt.delay.sec";
+ public static final int DEFAULT_MR_JOB_REDUCER_PREEMPT_DELAY_SEC = 0;
+
public static final String MR_AM_ENV =
MR_AM_PREFIX + "env";
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
index 5f32f11..56fb9fcd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
@@ -35,6 +35,7 @@
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.mapred.LocatedFileStatusFetcher;
+import org.apache.hadoop.mapred.SplitLocationInfo;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
@@ -359,6 +360,15 @@
String[] hosts) {
return new FileSplit(file, start, length, hosts);
}
+
+ /**
+ * A factory that makes the split for this class. It can be overridden
+ * by sub-classes to make sub-types
+ */
+ protected FileSplit makeSplit(Path file, long start, long length,
+ String[] hosts, String[] inMemoryHosts) {
+ return new FileSplit(file, start, length, hosts, inMemoryHosts);
+ }
/**
* Generate the list of files and make them into FileSplits.
@@ -392,17 +402,20 @@
while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {
int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
splits.add(makeSplit(path, length-bytesRemaining, splitSize,
- blkLocations[blkIndex].getHosts()));
+ blkLocations[blkIndex].getHosts(),
+ blkLocations[blkIndex].getCachedHosts()));
bytesRemaining -= splitSize;
}
if (bytesRemaining != 0) {
int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
splits.add(makeSplit(path, length-bytesRemaining, bytesRemaining,
- blkLocations[blkIndex].getHosts()));
+ blkLocations[blkIndex].getHosts(),
+ blkLocations[blkIndex].getCachedHosts()));
}
} else { // not splitable
- splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts()));
+ splits.add(makeSplit(path, 0, length, blkLocations[0].getHosts(),
+ blkLocations[0].getCachedHosts()));
}
} else {
//Create empty hosts array for zero length files
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileSplit.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileSplit.java
index 72c8450..9fba79c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileSplit.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileSplit.java
@@ -22,11 +22,13 @@
import java.io.DataInput;
import java.io.DataOutput;
+import org.apache.hadoop.mapred.SplitLocationInfo;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
@@ -41,6 +43,7 @@
private long start;
private long length;
private String[] hosts;
+ private SplitLocationInfo[] hostInfos;
public FileSplit() {}
@@ -57,6 +60,31 @@
this.length = length;
this.hosts = hosts;
}
+
+ /** Constructs a split with host and cached-blocks information
+ *
+ * @param file the file name
+ * @param start the position of the first byte in the file to process
+ * @param length the number of bytes in the file to process
+ * @param hosts the list of hosts containing the block
+ * @param inMemoryHosts the list of hosts containing the block in memory
+ */
+ public FileSplit(Path file, long start, long length, String[] hosts,
+ String[] inMemoryHosts) {
+ this(file, start, length, hosts);
+ hostInfos = new SplitLocationInfo[hosts.length];
+ for (int i = 0; i < hosts.length; i++) {
+ // because N will be tiny, scanning is probably faster than a HashSet
+ boolean inMemory = false;
+ for (String inMemoryHost : inMemoryHosts) {
+ if (inMemoryHost.equals(hosts[i])) {
+ inMemory = true;
+ break;
+ }
+ }
+ hostInfos[i] = new SplitLocationInfo(hosts[i], inMemory);
+ }
+ }
/** The file containing this split's data. */
public Path getPath() { return file; }
@@ -98,4 +126,10 @@
return this.hosts;
}
}
+
+ @Override
+ @Evolving
+ public SplitLocationInfo[] getLocationInfo() throws IOException {
+ return hostInfos;
+ }
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 9034064..508b033 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -83,6 +83,16 @@
</property>
<property>
+ <name>mapreduce.job.reducer.preempt.delay.sec</name>
+ <value>0</value>
+ <description>The threshold in terms of seconds after which an unsatisfied mapper
+ request triggers reducer preemption to free space. Default 0 implies that the
+ reduces should be preempted immediately after allocation if there is currently no
+ room for newly allocated mappers.
+ </description>
+</property>
+
+<property>
<name>mapreduce.job.max.split.locations</name>
<value>10</value>
<description>The max number of block locations to store for each split for
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java
index 0bb4e96..ba636b6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestFileInputFormat.java
@@ -103,6 +103,29 @@
}
@Test
+ public void testSplitLocationInfo() throws Exception {
+ Configuration conf = getConfiguration();
+ conf.set(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR,
+ "test:///a1/a2");
+ JobConf job = new JobConf(conf);
+ TextInputFormat fileInputFormat = new TextInputFormat();
+ fileInputFormat.configure(job);
+ FileSplit[] splits = (FileSplit[]) fileInputFormat.getSplits(job, 1);
+ String[] locations = splits[0].getLocations();
+ Assert.assertEquals(2, locations.length);
+ SplitLocationInfo[] locationInfo = splits[0].getLocationInfo();
+ Assert.assertEquals(2, locationInfo.length);
+ SplitLocationInfo localhostInfo = locations[0].equals("localhost") ?
+ locationInfo[0] : locationInfo[1];
+ SplitLocationInfo otherhostInfo = locations[0].equals("otherhost") ?
+ locationInfo[0] : locationInfo[1];
+ Assert.assertTrue(localhostInfo.isOnDisk());
+ Assert.assertTrue(localhostInfo.isInMemory());
+ Assert.assertTrue(otherhostInfo.isOnDisk());
+ Assert.assertFalse(otherhostInfo.isInMemory());
+ }
+
+ @Test
public void testListStatusSimple() throws IOException {
Configuration conf = new Configuration();
conf.setInt(FileInputFormat.LIST_STATUS_NUM_THREADS, numThreads);
@@ -223,8 +246,9 @@
public BlockLocation[] getFileBlockLocations(Path p, long start, long len)
throws IOException {
return new BlockLocation[] {
- new BlockLocation(new String[] { "localhost:50010" },
- new String[] { "localhost" }, 0, len) };
+ new BlockLocation(new String[] { "localhost:50010", "otherhost:50010" },
+ new String[] { "localhost", "otherhost" }, new String[] { "localhost" },
+ new String[0], 0, len, false) };
}
@Override
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java
index 246c158..3f877f1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/input/TestFileInputFormat.java
@@ -39,6 +39,7 @@
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.RawLocalFileSystem;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.mapred.SplitLocationInfo;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
import org.junit.After;
@@ -139,6 +140,28 @@
1, mockFs.numListLocatedStatusCalls);
FileSystem.closeAll();
}
+
+ @Test
+ public void testSplitLocationInfo() throws Exception {
+ Configuration conf = getConfiguration();
+ conf.set(org.apache.hadoop.mapreduce.lib.input.FileInputFormat.INPUT_DIR,
+ "test:///a1/a2");
+ Job job = Job.getInstance(conf);
+ TextInputFormat fileInputFormat = new TextInputFormat();
+ List<InputSplit> splits = fileInputFormat.getSplits(job);
+ String[] locations = splits.get(0).getLocations();
+ Assert.assertEquals(2, locations.length);
+ SplitLocationInfo[] locationInfo = splits.get(0).getLocationInfo();
+ Assert.assertEquals(2, locationInfo.length);
+ SplitLocationInfo localhostInfo = locations[0].equals("localhost") ?
+ locationInfo[0] : locationInfo[1];
+ SplitLocationInfo otherhostInfo = locations[0].equals("otherhost") ?
+ locationInfo[0] : locationInfo[1];
+ Assert.assertTrue(localhostInfo.isOnDisk());
+ Assert.assertTrue(localhostInfo.isInMemory());
+ Assert.assertTrue(otherhostInfo.isOnDisk());
+ Assert.assertFalse(otherhostInfo.isInMemory());
+ }
@Test
public void testListStatusSimple() throws IOException {
@@ -402,9 +425,9 @@
public BlockLocation[] getFileBlockLocations(Path p, long start, long len)
throws IOException {
return new BlockLocation[] {
- new BlockLocation(new String[] { "localhost:50010" },
- new String[] { "localhost" }, 0, len) };
- }
+ new BlockLocation(new String[] { "localhost:50010", "otherhost:50010" },
+ new String[] { "localhost", "otherhost" }, new String[] { "localhost" },
+ new String[0], 0, len, false) }; }
@Override
protected RemoteIterator<LocatedFileStatus> listLocatedStatus(Path f,
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index f359a4d..472febf 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -775,6 +775,12 @@
</dependency>
<dependency>
+ <groupId>org.apache.directory.server</groupId>
+ <artifactId>apacheds-kerberos-codec</artifactId>
+ <version>2.0.0-M15</version>
+ </dependency>
+
+ <dependency>
<groupId>com.microsoft.windowsazure.storage</groupId>
<artifactId>microsoft-windowsazure-storage-sdk</artifactId>
<version>0.6.0</version>
