SENTRY-2329: Integrate sentry with Hadoop 3.1.1 (kalyan kumar kalvagadda, reviewed by Sergio Pena)
This integration is compatible with old Hadoop 2.x versions. You can compile with Hadoop 2.x by
changing the pom.xml or you can add the Sentry/HDFS binding jars built with Hadoop3 in the
Hadoop 2 classpath. Both ways are verified and working.
diff --git a/pom.xml b/pom.xml
index acbdcc2..46ca38e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -70,7 +70,6 @@
<easymock.version>3.0</easymock.version>
<fest.reflect.version>1.4.1</fest.reflect.version>
<guava.version>14.0.1</guava.version>
- <hadoop.version>2.7.5</hadoop.version>
<hamcrest.version>1.3</hamcrest.version>
<hive.version>2.3.3</hive.version>
<jackson.version>1.9.13</jackson.version>
@@ -101,6 +100,10 @@
<test.sentry.hadoop.classpath>${maven.test.classpath}</test.sentry.hadoop.classpath>
<zookeeper.version>3.4.5</zookeeper.version>
<maven.jar.plugin.version>3.0.2</maven.jar.plugin.version>
+ <httpcomponents.version>4.5.3</httpcomponents.version>
+
+ <!-- Package versions for Sentry binding components -->
+ <hadoop.version>3.1.1</hadoop.version>
<!-- Datanucleus package versions -->
<datanucleus.maven.plugin.version>4.0.5</datanucleus.maven.plugin.version>
@@ -197,6 +200,7 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
<version>${hadoop.version}</version>
+ <scope>test</scope>
<exclusions>
<exclusion>
<artifactId>curator-client</artifactId>
@@ -410,6 +414,10 @@
<artifactId>jackson-mapper-asl</artifactId>
<groupId>org.codehaus.jackson</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-archives</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -521,6 +529,10 @@
<artifactId>jackson-xc</artifactId>
<groupId>org.codehaus.jackson</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-resourcemanager</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -608,6 +620,10 @@
<artifactId>jackson-xc</artifactId>
<groupId>org.codehaus.jackson</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -945,6 +961,10 @@
<artifactId>jackson-xc</artifactId>
<groupId>org.codehaus.jackson</groupId>
</exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-archives</artifactId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
diff --git a/sentry-binding/sentry-binding-solr/pom.xml b/sentry-binding/sentry-binding-solr/pom.xml
index f2a5fca..97b1879 100644
--- a/sentry-binding/sentry-binding-solr/pom.xml
+++ b/sentry-binding/sentry-binding-solr/pom.xml
@@ -29,6 +29,22 @@
<name>Sentry Binding for Solr</name>
<dependencies>
+ <!-- This jetty dependency must be added before the solr-core dependency to avoid conflicting
+ other jetty dependencies -->
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- This jetty dependency must be added before the solr-core dependency to avoid conflicting
+ other jetty dependencies -->
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-webapp</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-core</artifactId>
diff --git a/sentry-hdfs/sentry-hdfs-common/pom.xml b/sentry-hdfs/sentry-hdfs-common/pom.xml
index df6f04c..4d70d13 100644
--- a/sentry-hdfs/sentry-hdfs-common/pom.xml
+++ b/sentry-hdfs/sentry-hdfs-common/pom.xml
@@ -80,6 +80,12 @@
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${httpcomponents.version}</version>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
diff --git a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
index c9ecc40..edc3afc 100644
--- a/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
+++ b/sentry-hdfs/sentry-hdfs-common/src/main/java/org/apache/sentry/hdfs/PathsUpdate.java
@@ -26,11 +26,10 @@
import com.google.common.annotations.VisibleForTesting;
import org.apache.sentry.hdfs.service.thrift.TPathChanges;
import org.apache.sentry.hdfs.service.thrift.TPathsUpdate;
-import org.apache.commons.httpclient.util.URIUtil;
-import org.apache.commons.httpclient.URIException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.conf.Configuration;
+import org.apache.http.client.utils.URIBuilder;
import org.apache.thrift.TException;
@@ -137,11 +136,10 @@
URI uri;
try {
- uri = new URI(URIUtil.encodePath(path));
+ // Below converts unescaped path to escaped string and the constructs a URI from it.
+ uri = new URI(StringUtils.stripStart(new URIBuilder().setPath(path).toString(), "/"));
} catch (URISyntaxException e) {
throw new SentryMalformedPathException("Incomprehensible path [" + path + "]", e);
- } catch (URIException e) {
- throw new SentryMalformedPathException("Unable to create URI from path[" + path + "]", e);
}
String scheme = uri.getScheme();
diff --git a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java
index 18b6265..4d2ef00 100644
--- a/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java
+++ b/sentry-hdfs/sentry-hdfs-namenode-plugin/src/main/java/org/apache/sentry/hdfs/SentryINodeAttributesProvider.java
@@ -331,6 +331,11 @@
public INodeAttributes getAttributes(String[] pathElements,
INodeAttributes inode) {
Preconditions.checkNotNull(pathElements);
+
+ if (pathElements.length == 0) {
+ return inode;
+ }
+
pathElements = "".equals(pathElements[0]) && pathElements.length > 1 ?
Arrays.copyOfRange(pathElements, 1, pathElements.length) :
pathElements;
diff --git a/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/api/service/thrift/TestSentryWebServerWithKerberos.java b/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/api/service/thrift/TestSentryWebServerWithKerberos.java
index 5d94d4b..add2c28 100644
--- a/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/api/service/thrift/TestSentryWebServerWithKerberos.java
+++ b/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/api/service/thrift/TestSentryWebServerWithKerberos.java
@@ -88,7 +88,7 @@
new AuthenticatedURL(new KerberosAuthenticator()).openConnection(url, new AuthenticatedURL.Token());
fail("Here should fail.");
} catch (Exception e) {
- boolean isExpectError = e.getMessage().contains("No valid credentials provided");
+ boolean isExpectError = exceptionContainsMessage(e,"No valid credentials provided");
Assert.assertTrue("Here should fail by 'No valid credentials provided'," +
" but the exception is:" + e, isExpectError);
}
@@ -124,7 +124,7 @@
fail("Here should fail.");
} catch (AuthenticationException e) {
String expectedError = "status code: 403";
- if (!e.getMessage().contains(expectedError)) {
+ if (!exceptionContainsMessage(e, expectedError)) {
LOG.error("UnexpectedError: " + e.getMessage(), e);
fail("UnexpectedError: " + e.getMessage());
}
@@ -155,7 +155,7 @@
fail("Login with user1 should fail");
} catch (AuthenticationException e) {
String expectedError = "status code: 403";
- if (!e.getMessage().contains(expectedError)) {
+ if (!exceptionContainsMessage(e, expectedError)) {
LOG.error("UnexpectedError: " + e.getMessage(), e);
fail("UnexpectedError: " + e.getMessage());
}
@@ -172,4 +172,19 @@
conn.setRequestMethod("TRACE");
Assert.assertEquals(HttpURLConnection.HTTP_FORBIDDEN, conn.getResponseCode());
}
+
+ /*
+ * Check if the exception contains the specified message in any of the exception message
+ * or cause message.
+ *
+ * <p/>i.e. Hadoop 2.x has a 'no valid privileges' message in the e.getMessage() whereas
+ * Hadoop 3.x has the 'no valid privileges' message in the e.getCause().getMessage().
+ */
+ private boolean exceptionContainsMessage(Exception e, String message) {
+ if (e.getMessage().contains(message)) {
+ return true;
+ }
+
+ return e.getCause().getMessage().contains(message);
+ }
}
