PHOENIX-7842 Reduce the shading contribution to build time (#2460)
diff --git a/BUILDING.md b/BUILDING.md
index 1ba2838..a09d988 100644
--- a/BUILDING.md
+++ b/BUILDING.md
@@ -69,6 +69,39 @@
Note: javadocs are generated in target/apidocs
+Note: by default `mvn package` does not build `phoenix-mapreduce-byo-shaded-hbase`,
+and the resulting binary tarball does not contain its uberjar. RC / publish builds
+should use `-Prelease`, which adds that module back to the reactor and bundles its
+uberjar into the tarball. See "Faster builds" below.
+
+Faster builds
+-------------
+
+The following options can dramatically reduce wall-clock time:
+
+* `phoenix-mapreduce-byo-shaded-hbase` is the most expensive shade in the project
+ and is only needed for releases, so it is built only under `-Prelease`. A normal
+ `mvn package` skips it, and the binary tarball does not contain its uberjar.
+ Pass `-Prelease` to restore the previous behavior (this is what the ASF release
+ build does).
+
+* The shade modules are reactor-siblings and the shade plugin is thread-safe, so
+ they build concurrently with parallel reactor builds:
+
+ `$ mvn -T 1C clean package -DskipTests`
+
+ (`-T 1C` uses one build thread per CPU core. `-T 4` is also fine.)
+
+* Skip the shaded artifacts and assembly entirely when you are iterating on
+ `phoenix-core*` and do not need the shaded uberjars or the binary tarball.
+ Pass `-DPhoenixPatchProcess` to deactivate the `shade-and-assembly` profile
+ (this is what the ASF Yetus precommit and the Jenkinsfile use):
+
+ `$ mvn package -DskipTests -DPhoenixPatchProcess`
+
+* Shaded sources jars are off by default. `-Prelease` (or `-DshadeSources=true`)
+ re-enables them; RC / publish builds get them automatically.
+
HBase version compatibility
---------------------------
diff --git a/phoenix-assembly/pom.xml b/phoenix-assembly/pom.xml
index ae52be7..d9b9718 100644
--- a/phoenix-assembly/pom.xml
+++ b/phoenix-assembly/pom.xml
@@ -32,6 +32,11 @@
<source.skip>true</source.skip>
<jacocoArgLine/>
<main.basedir>${project.parent.basedir}</main.basedir>
+ <!-- Selects the (otherwise unused) component descriptor referenced by
+ src/build/package-to-tar-all.xml. Default = no-op. The release
+ profile overrides this to add the byo-shaded-hbase mapreduce uberjar
+ to the binary tarball. -->
+ <assembly.mapreduce.componentDescriptor>src/build/components/assembly-noop.xml</assembly.mapreduce.componentDescriptor>
</properties>
<dependencies>
@@ -40,10 +45,9 @@
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-server-${hbase.suffix}</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.phoenix</groupId>
- <artifactId>phoenix-mapreduce-byo-shaded-hbase-${hbase.suffix}</artifactId>
- </dependency>
+ <!-- See the `release` profile below for
+ phoenix-mapreduce-byo-shaded-hbase-${hbase.suffix}. It is built only for
+ releases. -->
<dependency>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-client-embedded-${hbase.suffix}</artifactId>
@@ -139,25 +143,8 @@
</arguments>
</configuration>
</execution>
- <execution>
- <id>mapreduce without version</id>
- <goals>
- <goal>exec</goal>
- </goals>
- <phase>compile</phase>
- <configuration>
- <executable>ln</executable>
- <workingDirectory>${project.basedir}/../phoenix-mapreduce-byo-shaded-hbase/target</workingDirectory>
- <arguments>
- <argument>-fnsv</argument>
- <argument>phoenix-mapreduce-byo-shaded-hbase-${hbase.suffix}-${project.version}.jar</argument>
- <argument>
- <!-- We are overwriting the unshaded JAR, but we don't care -->
- phoenix-mapreduce-byo-shaded-hbase-${hbase.suffix}.jar
- </argument>
- </arguments>
- </configuration>
- </execution>
+ <!-- See the `release` profile below for the mapreduce-byo-shaded-hbase
+ symlink exec. That uberjar is only built for releases. -->
<execution>
<id>pherf without version</id>
<goals>
@@ -215,6 +202,51 @@
<profiles>
<profile>
+ <!-- Release-only artifacts. The byo-shaded-hbase mapreduce uberjar is
+ expensive to shade and is not needed for normal dev builds, so it
+ is built only when -Prelease is active (see root pom.xml). This
+ profile wires the resulting jar into the binary tarball. -->
+ <id>release</id>
+ <properties>
+ <assembly.mapreduce.componentDescriptor>src/build/components/release-only-jars.xml</assembly.mapreduce.componentDescriptor>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-mapreduce-byo-shaded-hbase-${hbase.suffix}</artifactId>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>mapreduce-byo-shaded-hbase</id>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <phase>compile</phase>
+ <configuration>
+ <executable>ln</executable>
+ <workingDirectory>${project.basedir}/../phoenix-mapreduce-byo-shaded-hbase/target</workingDirectory>
+ <arguments>
+ <argument>-fnsv</argument>
+ <argument>phoenix-mapreduce-byo-shaded-hbase-${hbase.suffix}-${project.version}.jar</argument>
+ <argument>
+ <!-- We are overwriting the unshaded JAR, but we don't care -->
+ phoenix-mapreduce-byo-shaded-hbase-${hbase.suffix}.jar
+ </argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
<id>coverage</id>
<dependencies>
<dependency>
diff --git a/phoenix-assembly/src/build/components/all-common-jars.xml b/phoenix-assembly/src/build/components/all-common-jars.xml
index c2f943f..3b9dd48 100644
--- a/phoenix-assembly/src/build/components/all-common-jars.xml
+++ b/phoenix-assembly/src/build/components/all-common-jars.xml
@@ -47,14 +47,8 @@
<include>phoenix-server-${hbase.suffix}.jar</include>
</includes>
</fileSet>
- <fileSet>
- <directory>${project.basedir}/../phoenix-mapreduce-byo-shaded-hbase/target</directory>
- <outputDirectory>/</outputDirectory>
- <includes>
- <include>phoenix-mapreduce-byo-shaded-hbase-${hbase.suffix}-${project.version}.jar</include>
- <include>phoenix-mapreduce-byo-shaded-hbase-${hbase.suffix}.jar</include>
- </includes>
- </fileSet>
+ <!-- The byo-shaded-hbase mapreduce uberjar is added only under -Prelease via
+ components/release-only-jars.xml (see phoenix-assembly/pom.xml). -->
<fileSet>
<directory>${project.basedir}/../phoenix-pherf/target</directory>
<outputDirectory>/</outputDirectory>
diff --git a/phoenix-assembly/src/build/components/assembly-noop.xml b/phoenix-assembly/src/build/components/assembly-noop.xml
new file mode 100644
index 0000000..2c5f05f
--- /dev/null
+++ b/phoenix-assembly/src/build/components/assembly-noop.xml
@@ -0,0 +1,31 @@
+<?xml version='1.0'?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<!--
+ Empty placeholder component descriptor used in default builds where the
+ release-only jars (mapreduce byo-shaded-hbase) are not produced.
+ Selected by the `assembly.mapreduce.componentDescriptor` property
+ defined in phoenix-assembly/pom.xml.
+
+ An empty `<component/>` is a valid component descriptor per the assembly
+ plugin schema. Every child of <component> is optional.
+-->
+<component/>
diff --git a/phoenix-assembly/src/build/components/release-only-jars.xml b/phoenix-assembly/src/build/components/release-only-jars.xml
new file mode 100644
index 0000000..0b5917b
--- /dev/null
+++ b/phoenix-assembly/src/build/components/release-only-jars.xml
@@ -0,0 +1,40 @@
+<?xml version='1.0'?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<!--
+ Component descriptor wired in only under the release profile.
+ Adds the byo-shaded-hbase mapreduce uberjar to the binary tarball.
+ In default (non-release) builds this descriptor is replaced with the
+ no-op `assembly-noop.xml` via the `assembly.mapreduce.componentDescriptor`
+ property defined in phoenix-assembly/pom.xml.
+-->
+<component>
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}/../phoenix-mapreduce-byo-shaded-hbase/target</directory>
+ <outputDirectory>/</outputDirectory>
+ <includes>
+ <include>phoenix-mapreduce-byo-shaded-hbase-${hbase.suffix}-${project.version}.jar</include>
+ <include>phoenix-mapreduce-byo-shaded-hbase-${hbase.suffix}.jar</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+</component>
diff --git a/phoenix-assembly/src/build/package-to-tar-all.xml b/phoenix-assembly/src/build/package-to-tar-all.xml
index 9dd2928..93c9191 100644
--- a/phoenix-assembly/src/build/package-to-tar-all.xml
+++ b/phoenix-assembly/src/build/package-to-tar-all.xml
@@ -35,5 +35,8 @@
<componentDescriptor>src/build/components/all-common-jars.xml</componentDescriptor>
<componentDescriptor>src/build/components/all-common-files.xml</componentDescriptor>
<componentDescriptor>src/build/components/all-common-dependencies.xml</componentDescriptor>
+ <!-- Resolves to assembly-noop.xml in default builds and
+ release-only-jars.xml under -Prelease. See phoenix-assembly/pom.xml. -->
+ <componentDescriptor>${assembly.mapreduce.componentDescriptor}</componentDescriptor>
</componentDescriptors>
</assembly>
diff --git a/pom.xml b/pom.xml
index 97d497c..c90d831 100644
--- a/pom.xml
+++ b/pom.xml
@@ -92,7 +92,10 @@
<test.output.tofile>true</test.output.tofile>
<top.dir>${project.basedir}/..</top.dir>
<test.tmp.dir>${project.build.directory}</test.tmp.dir>
- <shadeSources>true</shadeSources>
+ <!-- Sources-jar shading roughly doubles the time of every shade execution.
+ Default it OFF here; the `release` profile re-enables it so RC / publish
+ builds still produce shaded sources jars. -->
+ <shadeSources>false</shadeSources>
<!-- Dependency versions -->
<jackson-bom.version>2.18.4.1</jackson-bom.version>
@@ -161,7 +164,7 @@
<lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
<maven-bundle-plugin.version>5.1.9</maven-bundle-plugin.version>
<exec-maven-plugin.version>3.1.1</exec-maven-plugin.version>
- <maven-shade-plugin.version>3.6.0</maven-shade-plugin.version>
+ <maven-shade-plugin.version>3.6.2</maven-shade-plugin.version>
<!-- This overrides the property in the ASF parent project-->
<version.maven-site-plugin>3.21.0</version.maven-site-plugin>
<spotless.version>2.30.0</spotless.version>
@@ -2094,13 +2097,22 @@
<module>phoenix-client-parent/phoenix-client-embedded</module>
<module>phoenix-client-parent/phoenix-client-lite</module>
<module>phoenix-server</module>
- <module>phoenix-mapreduce-byo-shaded-hbase</module>
<module>phoenix-assembly</module>
</modules>
</profile>
<!-- this profile should be activated for release builds -->
<profile>
<id>release</id>
+ <!-- The byo-shaded-hbase mapreduce uberjar is a release-only artifact.
+ It costs several minutes to shade and is not needed for normal
+ dev builds, so we only add it to the reactor under -Prelease. -->
+ <modules>
+ <module>phoenix-mapreduce-byo-shaded-hbase</module>
+ </modules>
+ <properties>
+ <!-- Re-enable shaded sources jars for RC / publish builds. -->
+ <shadeSources>true</shadeSources>
+ </properties>
<build>
<plugins>
<plugin>
