HBASE-22393 HBOSS: Shaded external dependencies to avoid conflicts with Hadoop and HBase.
Closes #2
Signed-off-by: Josh Elser <elserj@apache.org>
Co-authored-by: Wellington Chevreuil <wellington.chevreuil@gmail.com>
Co-authored-by: Sean Busbey <busbey@apache.org>
diff --git a/hbase-oss/pom.xml b/hbase-oss/pom.xml
index aa4c475..74cbc0a 100644
--- a/hbase-oss/pom.xml
+++ b/hbase-oss/pom.xml
@@ -35,6 +35,7 @@
<properties>
<fs.hboss.sync.impl>org.apache.hadoop.hbase.oss.sync.ZKTreeLockManager</fs.hboss.sync.impl>
<aws-java-sdk.version>1.11.525</aws-java-sdk.version>
+ <shading.prefix>org.apache.hadoop.hbase.oss.thirdparty</shading.prefix>
</properties>
<profiles>
@@ -70,21 +71,200 @@
</systemProperties>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>aggregate-into-a-jar-with-relocated-third-parties</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createDependencyReducedPom>true</createDependencyReducedPom>
+ <!-- XXX If you change the set of dependencies, you should uncomment this so that
+ you can ensure we don't gain additional transitive dependencies that need to
+ be relocated. It can't be true when we build or the shade plugin will drop
+ all of our direct dependencies at provided and test scope. :/
+ -->
+ <!--
+ <promoteTransitiveDependencies>true</promoteTransitiveDependencies>
+ -->
+ <!-- XXX If you change what gets included, please review the generated
+ LICENSE/NOTICE files to make sure they're accurate
+ -->
+ <artifactSet>
+ <includes>
+ <include>com.google.*:*:jar:*</include>
+ <include>org.apache.commons:commons-lang3:jar:*</include>
+ <include>org.apache.curator:curator-*:jar:*</include>
+ <include>org.apache.yetus:audience-annotations:jar:*</include>
+ <include>org.apache.zookeeper:zookeeper:jar:*</include>
+ <!-- transitives -->
+ <include>io.netty:netty</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.google</pattern>
+ <shadedPattern>${shading.prefix}.com.google</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.commons</pattern>
+ <shadedPattern>${shading.prefix}.org.apache.commons</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.curator</pattern>
+ <shadedPattern>${shading.prefix}.org.apache.curator</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.jute</pattern>
+ <shadedPattern>${shading.prefix}.org.apache.jute</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.yetus</pattern>
+ <shadedPattern>${shading.prefix}.org.apache.yetus</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.apache.zookeeper</pattern>
+ <shadedPattern>${shading.prefix}.org.apache.zookeeper</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>org.codehaus</pattern>
+ <shadedPattern>${shading.prefix}.org.codehaus</shadedPattern>
+ </relocation>
+ <!-- netty family -->
+ <relocation>
+ <pattern>org.jboss.netty</pattern>
+ <shadedPattern>${shading.prefix}.org.jboss.netty</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>io.netty</pattern>
+ <shadedPattern>${shading.prefix}.io.netty</shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <!-- Where notices exist, just concat them -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer">
+ <addHeader>false</addHeader>
+ <projectName>${project.name}</projectName>
+ </transformer>
+ </transformers>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <!-- We track needed LICENSE details manually -->
+ <exclude>LICENSE.txt</exclude>
+ <exclude>META-INF/LICENSE.txt</exclude>
+ <exclude>META-INF/license/LICENSE*</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <!-- we manually contruct the needed NOTICE details for netty -->
+ <artifact>io.netty:netty</artifact>
+ <excludes>
+ <exclude>META-INF/NOTICE.txt</exclude>
+ </excludes>
+ </filter>
+ </filters>
+
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
<dependencies>
+ <!-- the dependency plugin's dependency:tree output defaults to only showing transitive
+ dependencies coming off of the first direct dependency to need it, even when the
+ scope changes because another direct depenendcy has a wider scope. to make the output
+ easier to read, list provided and test dependencies after compile scope dependencies.
+
+ for example, we need netty at compile scope due to our zookeeper dependency. it's also
+ a transitive dependency of our hadoop dependency that's listed as provided.
+
+ (the verbose mode of dependency tree will show transitive dependencies multiple times
+ when needed by multiple direct dependencies, but it uses a different code path than
+ normal resolution)
+ -->
<dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-annotations</artifactId>
- <version>${hadoop.version}</version>
- <scope>provided</scope>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons-lang3.version}</version>
</dependency>
<dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ <exclusions>
+ <exclusion>
+ <!-- Banned import in HBase -->
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ <exclusion>
+ <!-- this should have been marked optional by guava because
+ the annotation are retention CLASS -->
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>animal-sniffer-annotations</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.yetus</groupId>
+ <artifactId>audience-annotations</artifactId>
+ <version>${audience-annotations.version}</version>
+ </dependency>
+ <!-- For ZooKeeper implementation -->
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>${zookeeper.version}</version>
+ <exclusions>
+ <exclusion>
+ <!-- jline is optional and we're not providing a cli -->
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-client</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-framework</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-recipes</artifactId>
+ <version>${curator.version}</version>
+ </dependency>
+
+ <!-- Provided dependencies we expect to be around at runtime. i.e. Hadoop and HBase -->
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
<scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <!-- Banned import in HBase -->
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
@@ -92,35 +272,6 @@
<version>${hbase.version}</version>
<scope>provided</scope>
</dependency>
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- <version>${commons-lang3.version}</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase.thirdparty</groupId>
- <artifactId>hbase-shaded-miscellaneous</artifactId>
- <version>${hbase-thirdparty.version}</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>${slf4j.version}</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
- <scope>compile</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.yetus</groupId>
- <artifactId>audience-annotations</artifactId>
- <version>${audience-annotations.version}</version>
- <scope>compile</scope>
- </dependency>
<!-- Test dependencies -->
<dependency>
@@ -175,6 +326,18 @@
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <version>${log4j.version}</version>
+ <scope>test</scope>
+ </dependency>
<!-- For testing against S3 -->
<dependency>
@@ -196,12 +359,6 @@
<scope>test</scope>
</dependency>
- <!-- For ZooKeeper implementation -->
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- <version>${zookeeper.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-zookeeper</artifactId>
@@ -215,21 +372,6 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-client</artifactId>
- <version>${curator.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <version>${curator.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- <version>${curator.version}</version>
- </dependency>
</dependencies>
diff --git a/hbase-oss/src/main/appended-resources/META-INF/LICENSE b/hbase-oss/src/main/appended-resources/META-INF/LICENSE
new file mode 100644
index 0000000..80cbf2c
--- /dev/null
+++ b/hbase-oss/src/main/appended-resources/META-INF/LICENSE
@@ -0,0 +1,72 @@
+----
+This work bundles additional works available under a different license.
+
+Parts of the Netty Project are licensed under:
+
+This product contains the following works licensed under the following Public Domain dedication:
+
+* extensions to Java Collections Framework which has been derived from the works by JSR-166 EG,
+ Doug Lea, and Jason T. Greene
+* modified version of Robert Harder's Public Domain Base64 Encoder and Decoder.
+
+The person or persons who have associated work with this document (the
+"Dedicator" or "Certifier") hereby either (a) certifies that, to the best of
+his knowledge, the work of authorship identified is in the public domain of
+the country from which the work is published, or (b) hereby dedicates whatever
+copyright the dedicators holds in the work of authorship identified below (the
+"Work") to the public domain. A certifier, moreover, dedicates any copyright
+interest he may have in the associated work, and for these purposes, is
+described as a "dedicator" below.
+
+A certifier has taken reasonable steps to verify the copyright status of this
+work. Certifier recognizes that his good faith efforts may not shield him from
+liability if in fact the work certified is not in the public domain.
+
+Dedicator makes this dedication for the benefit of the public at large and to
+the detriment of the Dedicator's heirs and successors. Dedicator intends this
+dedication to be an overt act of relinquishment in perpetuity of all present
+and future rights under copyright law, whether vested or contingent, in the
+Work. Dedicator understands that such relinquishment of all rights includes
+the relinquishment of all rights to enforce (by lawsuit or otherwise) those
+copyrights in the Work.
+
+Dedicator recognizes that, once placed in the public domain, the Work may be
+freely reproduced, distributed, transmitted, used, modified, built upon, or
+otherwise exploited by anyone for any purpose, commercial or non-commercial,
+and in any way, including by methods that have not yet been invented or
+conceived.
+
+This product contains the following works licensed under the BSD 3-Clause license:
+
+* modified version of 'JZlib' Copyright (c) 2000,2001,2002,2003,2004 ymnk, JCraft,Inc. All rights
+ reserved.
+* modified version of 'Webbit' Copyright (c) 2011, Joe Walnes, Aslak Hellesøy and contributors
+ All rights reserved.
+
+Copyright (c) <YEAR>, <OWNER>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+ may be used to endorse or promote products derived from this software without
+ specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/hbase-oss/src/main/appended-resources/META-INF/NOTICE b/hbase-oss/src/main/appended-resources/META-INF/NOTICE
new file mode 100644
index 0000000..f3d4c1d
--- /dev/null
+++ b/hbase-oss/src/main/appended-resources/META-INF/NOTICE
@@ -0,0 +1,45 @@
+----
+This work bundles parts of the Netty Project licensed under the Apache License v2.0.
+
+Relevant copy of details from its NOTICE file, including works bundled as a result of including it.
+Please see LICENSE for details of anything licensed under something other than Apache License v2.0.
+
+Please visit the Netty web site for more information:
+
+ * http://netty.io/
+
+Copyright 2011 The Netty Project
+
+This product contains the extensions to Java Collections Framework which has
+been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene available
+under Public Domain.
+
+ * HOMEPAGE:
+ * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/
+ * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/
+
+This product contains a modified version of Robert Harder's Public Domain
+Base64 Encoder and Decoder, which can be obtained at:
+
+ * HOMEPAGE:
+ * http://iharder.sourceforge.net/current/java/base64/
+
+This product contains a modified version of 'JZlib', a re-implementation of
+zlib in pure Java under a BSD style license, which can be obtained at:
+
+ * HOMEPAGE:
+ * http://www.jcraft.com/jzlib/
+
+This product contains a modified version of 'Webbit', a Java event based
+WebSocket and HTTP server, under a BSD license.
+
+ * HOMEPAGE:
+ * https://github.com/joewalnes/webbit
+
+----
+This work bundles additional works licensed under the Apache License v2.0.
+
+Copies of relevant NOTICE files follow:
+
+Apache ZooKeeper
+Copyright 2009-2017 The Apache Software Foundation
diff --git a/hbase-oss/src/main/java/org/apache/hadoop/hbase/oss/HBaseObjectStoreSemantics.java b/hbase-oss/src/main/java/org/apache/hadoop/hbase/oss/HBaseObjectStoreSemantics.java
index bebc37c..11bf9b8 100644
--- a/hbase-oss/src/main/java/org/apache/hadoop/hbase/oss/HBaseObjectStoreSemantics.java
+++ b/hbase-oss/src/main/java/org/apache/hadoop/hbase/oss/HBaseObjectStoreSemantics.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.oss;
+import com.google.common.annotations.VisibleForTesting;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
@@ -56,7 +57,6 @@
import org.apache.hadoop.hbase.oss.sync.TreeLockManager.Depth;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.util.Progressable;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
diff --git a/hbase-oss/src/main/java/org/apache/hadoop/hbase/oss/sync/TreeLockManager.java b/hbase-oss/src/main/java/org/apache/hadoop/hbase/oss/sync/TreeLockManager.java
index 73be990..8ea06af 100644
--- a/hbase-oss/src/main/java/org/apache/hadoop/hbase/oss/sync/TreeLockManager.java
+++ b/hbase-oss/src/main/java/org/apache/hadoop/hbase/oss/sync/TreeLockManager.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.oss.sync;
+import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.net.URI;
import java.util.Arrays;
@@ -30,7 +31,6 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.oss.Constants;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/sync/LocalTreeLockManager.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/sync/LocalTreeLockManager.java
index 372fa2e..97cba26 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/sync/LocalTreeLockManager.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/sync/LocalTreeLockManager.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.oss.sync;
+import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
@@ -28,7 +29,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/sync/NullTreeLockManager.java b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/sync/NullTreeLockManager.java
index c9f3fbb..b135de7 100644
--- a/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/sync/NullTreeLockManager.java
+++ b/hbase-oss/src/test/java/org/apache/hadoop/hbase/oss/sync/NullTreeLockManager.java
@@ -18,10 +18,10 @@
package org.apache.hadoop.hbase.oss.sync;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hbase.thirdparty.com.google.common.annotations.VisibleForTesting;
/**
* Bypasses all synchronization to effectively make HBOSS operations no-ops.
diff --git a/pom.xml b/pom.xml
index c14a6b5..319c990 100644
--- a/pom.xml
+++ b/pom.xml
@@ -15,6 +15,13 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache</groupId>
+ <artifactId>apache</artifactId>
+ <version>21</version>
+ <relativePath/>
+ <!-- no parent resolution -->
+ </parent>
<groupId>org.apache.hbase.filesystem</groupId>
<artifactId>hbase-filesystem</artifactId>
@@ -34,16 +41,20 @@
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
+ <!-- XXX If you change these versions please ensure the bundled LICENSE/NOTICE in
+ hbase-oss is correct
+ -->
<audience-annotations.version>0.5.0</audience-annotations.version>
<aws-java-sdk.version>1.11.525</aws-java-sdk.version>
<commons-io.version>2.5</commons-io.version>
<commons-lang3.version>3.6</commons-lang3.version>
<curator.version>4.0.0</curator.version>
+ <guava.version>23.0</guava.version>
<hadoop2.version>2.9.2</hadoop2.version>
<hadoop3.version>3.2.0</hadoop3.version>
<hbase.version>2.1.4</hbase.version>
- <hbase-thirdparty.version>2.2.0</hbase-thirdparty.version>
<junit.version>4.12</junit.version>
+ <log4j.version>1.2.17</log4j.version>
<slf4j.version>1.7.25</slf4j.version>
<zookeeper.version>3.4.10</zookeeper.version>
</properties>