PHOENIX-6249 Create shaded phoenix-spark connector jar
diff --git a/phoenix-spark-base/phoenix4-spark/pom.xml b/phoenix-spark-base/phoenix4-spark/pom.xml
index 53bbfde..903c75f 100644
--- a/phoenix-spark-base/phoenix4-spark/pom.xml
+++ b/phoenix-spark-base/phoenix4-spark/pom.xml
@@ -61,6 +61,10 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ </plugin>
</plugins>
</build>
diff --git a/phoenix-spark-base/phoenix5-spark/pom.xml b/phoenix-spark-base/phoenix5-spark/pom.xml
index 3fd1a9e..3a9105f 100644
--- a/phoenix-spark-base/phoenix5-spark/pom.xml
+++ b/phoenix-spark-base/phoenix5-spark/pom.xml
@@ -39,7 +39,6 @@
<hadoop.version>${hadoop-three.version}</hadoop.version>
<jdk.version>1.8</jdk.version>
<fasterxml.jackson.version>2.10.0</fasterxml.jackson.version>
- <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
<phoenix.main.version>5</phoenix.main.version>
</properties>
@@ -48,6 +47,21 @@
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-connectors-phoenix5-compat</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs-client</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ </exclusions>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<dependencyManagement>
@@ -90,6 +104,10 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-failsafe-plugin</artifactId>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ </plugin>
</plugins>
</build>
diff --git a/phoenix-spark-base/pom.xml b/phoenix-spark-base/pom.xml
index fe01511..00fe45c 100644
--- a/phoenix-spark-base/pom.xml
+++ b/phoenix-spark-base/pom.xml
@@ -144,6 +144,7 @@
<artifactId>netty</artifactId>
</exclusion>
</exclusions>
+ <scope>provided</scope>
</dependency>
<dependency>
@@ -175,6 +176,7 @@
<artifactId>netty</artifactId>
</exclusion>
</exclusions>
+ <scope>provided</scope>
</dependency>
@@ -379,6 +381,53 @@
<scope>test</scope>
</dependency>
+ <!-- Mark every Hadoop jar as provided -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-annotations</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-auth</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <!-- We want to take the implementation from Spark -->
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>provided</scope>
+ </dependency>
</dependencies>
<build>
@@ -461,6 +510,7 @@
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<configuration>
+ <ignoreNonCompile>true</ignoreNonCompile>
<ignoredUnusedDeclaredDependencies>
<!-- These are all used -->
<ignoredUnusedDeclaredDependency>
@@ -546,7 +596,6 @@
<target>1.8</target>
</configuration>
</plugin>
-
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
@@ -564,6 +613,219 @@
</execution>
</executions>
</plugin>
+ <!-- Taken from phoenix-client-parent
+ this should be kept in sync with Phoenix as much as possible -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <configuration>
+ <finalName>phoenix${phoenix.main.version}-${project.version}-spark</finalName>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ <exclude>META-INF/license/*</exclude>
+ <exclude>META-INF/NOTICE</exclude>
+ <exclude>LICENSE.*</exclude>
+ <exclude>NOTICE.*</exclude>
+ <exclude>NOTICE</exclude>
+ <exclude>README*</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>org.apache.hadoop:hadoop-yarn-common</artifact>
+ <excludes>
+ <exclude>org/apache/hadoop/yarn/factories/package-info.class</exclude>
+ <exclude>org/apache/hadoop/yarn/util/package-info.class</exclude>
+ <exclude>org/apache/hadoop/yarn/factory/providers/package-info.class</exclude>
+ <exclude>org/apache/hadoop/yarn/client/api/impl/package-info.class</exclude>
+ <exclude>org/apache/hadoop/yarn/client/api/package-info.class</exclude>
+ <exclude>webapps/**</exclude>
+ </excludes>
+ </filter>
+ <!-- Phoenix specific -->
+ <filter>
+ <artifact>org.apache.commons:commons-math3</artifact>
+ <excludes>
+ <exclude>assets/**</exclude>
+ </excludes>
+ </filter>
+ <filter>
+ <artifact>org.apache.hbase:hbase-server</artifact>
+ <excludes>
+ <exclude>hbase-webapps/**</exclude>
+ </excludes>
+ </filter>
+ <!-- Phoenix specific -->
+ </filters>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <resource>csv-bulk-load-config.properties</resource>
+ <file>
+ ${project.basedir}/../config/csv-bulk-load-config.properties
+ </file>
+ </transformer>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <resource>README.md</resource>
+ <file>${project.basedir}/../README.md</file>
+ </transformer>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <resource>LICENSE.txt</resource>
+ <file>${project.basedir}/../LICENSE</file>
+ </transformer>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.IncludeResourceTransformer">
+ <resource>NOTICE</resource>
+ <file>${project.basedir}/../NOTICE</file>
+ </transformer>
+ </transformers>
+ <relocations>
+ <relocation>
+ <pattern>org/</pattern>
+ <shadedPattern>${shaded.package}.org.</shadedPattern>
+ <excludes>
+ <exclude>org/apache/hadoop/**</exclude>
+ <!-- Our non-shaded htrace and logging libraries -->
+ <exclude>org/apache/htrace/**</exclude>
+ <exclude>org/slf4j/**</exclude>
+ <exclude>org/apache/commons/logging/**</exclude>
+ <exclude>org/apache/log4j/**</exclude>
+ <!-- Not the org/ packages that are a part of the jdk -->
+ <exclude>org/ietf/jgss/**</exclude>
+ <exclude>org/omg/**</exclude>
+ <exclude>org/w3c/dom/**</exclude>
+ <exclude>org/xml/sax/**</exclude>
+ <!-- Extras compared to Hadoop -->
+ <!-- Hbase classes - Maybe these could be shaded as well ? -->
+ <exclude>org/apache/hbase/**</exclude>
+ <!-- We use the spark classpath directly -->
+ <exclude>org/apache/spark/**</exclude>
+ <!-- Phoenix classes -->
+ <exclude>org/apache/phoenix/**</exclude>
+ <exclude>org/apache/omid/**</exclude>
+ <!-- Do want/need to expose Tephra as well ? -->
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>com/</pattern>
+ <shadedPattern>${shaded.package}.com.</shadedPattern>
+ <excludes>
+ <!-- Not the com/ packages that are a part of particular jdk implementations -->
+ <exclude>com/sun/tools/**</exclude>
+ <exclude>com/sun/javadoc/**</exclude>
+ <exclude>com/sun/security/**</exclude>
+ <exclude>com/sun/jndi/**</exclude>
+ <exclude>com/sun/management/**</exclude>
+ </excludes>
+ </relocation>
+ <relocation>
+ <pattern>io/</pattern>
+ <shadedPattern>${shaded.package}.io.</shadedPattern>
+ <excludes>
+ <!-- Exclude config keys for Hadoop that look like package names -->
+ <exclude>io/compression/**</exclude>
+ <exclude>io/mapfile/**</exclude>
+ <exclude>io/map/index/*</exclude>
+ <exclude>io/seqfile/**</exclude>
+ <exclude>io/file/buffer/size</exclude>
+ <exclude>io/skip/checksum/errors</exclude>
+ <exclude>io/sort/*</exclude>
+ <exclude>io/serializations</exclude>
+ </excludes>
+ </relocation>
+ <!-- JSRs that haven't made it to inclusion in J2SE -->
+ <relocation>
+ <pattern>javax/el/</pattern>
+ <shadedPattern>${shaded.package}.javax.el.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>javax/cache/</pattern>
+ <shadedPattern>${shaded.package}.javax.cache.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>javax/servlet/</pattern>
+ <shadedPattern>${shaded.package}.javax.servlet.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>javax/ws/</pattern>
+ <shadedPattern>${shaded.package}.javax.ws.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>javax/annotation/</pattern>
+ <shadedPattern>${shaded.package}.javax.annotation.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>javax/inject/</pattern>
+ <shadedPattern>${shaded.package}.javax.inject.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>javax/validation/</pattern>
+ <shadedPattern>${shaded.package}.javax.validation.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>net/</pattern>
+ <shadedPattern>${shaded.package}.net.</shadedPattern>
+ <excludes>
+ <!-- Exclude config keys for Hadoop that look like package names -->
+ <exclude>net/topology/**</exclude>
+ </excludes>
+ </relocation>
+ <!-- okio declares a top level package instead of nested -->
+ <relocation>
+ <pattern>okio/</pattern>
+ <shadedPattern>${shaded.package}.okio.</shadedPattern>
+ </relocation>
+ <!-- Phoenix specific relocations -->
+ <relocation>
+ <!-- Tephra -->
+ <pattern>co/</pattern>
+ <shadedPattern>${shaded.package}.co.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>it/</pattern>
+ <shadedPattern>${shaded.package}.it.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>javassist/</pattern>
+ <shadedPattern>${shaded.package}.javassist.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>jersey/</pattern>
+ <shadedPattern>${shaded.package}.jersey.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>jline/</pattern>
+ <shadedPattern>${shaded.package}.jline.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>mozilla/</pattern>
+ <shadedPattern>${shaded.package}.mozilla.</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>tables/</pattern>
+ <shadedPattern>${shaded.package}.tables.</shadedPattern>
+ </relocation>
+ <!-- Phoenix specific relocations end -->
+ </relocations>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</pluginManagement>
</build>
diff --git a/phoenix4-connectors-assembly/pom.xml b/phoenix4-connectors-assembly/pom.xml
index 4da2d8b..7f8bd88 100644
--- a/phoenix4-connectors-assembly/pom.xml
+++ b/phoenix4-connectors-assembly/pom.xml
@@ -133,7 +133,7 @@
<arguments>
<argument>-fnsv</argument>
<argument>
- phoenix4-spark-${project.version}.jar
+ phoenix4-${project.version}-spark.jar
</argument>
<argument>
phoenix4-spark.jar
diff --git a/phoenix4-connectors-assembly/src/build/components/phoenix4-jars.xml b/phoenix4-connectors-assembly/src/build/components/phoenix4-jars.xml
index 2fe4e2f..5c509d5 100644
--- a/phoenix4-connectors-assembly/src/build/components/phoenix4-jars.xml
+++ b/phoenix4-connectors-assembly/src/build/components/phoenix4-jars.xml
@@ -51,7 +51,7 @@
<directory>${project.basedir}/../phoenix-spark-base/phoenix4-spark/target</directory>
<outputDirectory>/</outputDirectory>
<includes>
- <include>phoenix4-spark-${project.version}.jar</include>
+ <include>phoenix4-${project.version}-spark.jar</include>
<include>phoenix4-spark.jar</include>
</includes>
</fileSet>
diff --git a/phoenix5-connectors-assembly/pom.xml b/phoenix5-connectors-assembly/pom.xml
index a48238e..ec2bdb5 100644
--- a/phoenix5-connectors-assembly/pom.xml
+++ b/phoenix5-connectors-assembly/pom.xml
@@ -140,7 +140,7 @@
<arguments>
<argument>-fnsv</argument>
<argument>
- phoenix5-spark-${project.version}.jar
+ phoenix5-${project.version}-spark.jar
</argument>
<argument>
phoenix5-spark.jar
diff --git a/phoenix5-connectors-assembly/src/build/components/phoenix5-jars.xml b/phoenix5-connectors-assembly/src/build/components/phoenix5-jars.xml
index 660974c..ac1630e 100644
--- a/phoenix5-connectors-assembly/src/build/components/phoenix5-jars.xml
+++ b/phoenix5-connectors-assembly/src/build/components/phoenix5-jars.xml
@@ -51,7 +51,7 @@
<directory>${project.basedir}/../phoenix-spark-base/phoenix5-spark/target</directory>
<outputDirectory>/</outputDirectory>
<includes>
- <include>phoenix5-spark-${project.version}.jar</include>
+ <include>phoenix5-${project.version}-spark.jar</include>
<include>phoenix5-spark.jar</include>
</includes>
</fileSet>
diff --git a/pom.xml b/pom.xml
index bcaf6fc..2b538d4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -62,7 +62,7 @@
<!-- Hadoop Versions -->
<hbase-one.version>1.4.0</hbase-one.version>
<hbase-two.version>2.1.10</hbase-two.version>
- <hbase-thirdparty-version>2.2.1</hbase-thirdparty-version>
+ <hbase-thirdparty-version>2.1.0</hbase-thirdparty-version>
<hadoop-two.version>2.7.5</hadoop-two.version>
<hadoop-three.version>3.0.3</hadoop-three.version>
@@ -102,8 +102,6 @@
<jodatime.version>2.10.5</jodatime.version>
<commons-cli.version>1.4</commons-cli.version>
<commons-compress.version>1.9</commons-compress.version>
- <!-- For pig -->
- <commons-logging.version>1.2</commons-logging.version>
<!-- For Kafka -->
<com-101tek-zkclient.version>0.7</com-101tek-zkclient.version>
<!-- For hive -->
@@ -117,6 +115,7 @@
<maven-project-info-reports-plugin.version>3.1.1</maven-project-info-reports-plugin.version>
<maven-checkstyle-plugin.version>3.1.0</maven-checkstyle-plugin.version>
<maven-build-helper-plugin.version>3.2.0</maven-build-helper-plugin.version>
+
<exec-maven-plugin.version>1.6.0</exec-maven-plugin.version>
<spotbugs-maven-plugin.version>4.1.3</spotbugs-maven-plugin.version>
<spotbugs.version>4.1.3</spotbugs.version>
@@ -574,7 +573,6 @@
<groupId>org.apache.hbase.thirdparty</groupId>
<artifactId>hbase-shaded-miscellaneous</artifactId>
<version>${hbase-thirdparty-version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hbase.thirdparty</groupId>
@@ -697,6 +695,11 @@
<artifactId>hadoop-hdfs-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <version>${hadoop.version}</version>
+ </dependency>
<!-- End for exclusion only -->
<dependency>
@@ -788,11 +791,6 @@
<artifactId>protobuf-java</artifactId>
<version>2.5.0</version>
</dependency>
- <dependency>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- <version>${commons-logging.version}</version>
- </dependency>
<dependency>
<groupId>org.slf4j</groupId>
