[LIVY-593] Add flags to support hadoop, spark and scala version specific builds (#405)
Enable flags to build livy against specific hadoop, spark and scala versions. Currently supports:
1. scala 2.11 and scala 2.12 with Spark 2 and Hadoop 2
2. scala 2.12 with Spark 3 and hadoop 2 (Spark 3 doesn't come with scala 2.11 builds)
Co-authored-by: Sumit Kumar <ksumit@apache.org>
diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml
index d74fa8e..e52df4d 100644
--- a/.github/workflows/integration-tests.yaml
+++ b/.github/workflows/integration-tests.yaml
@@ -27,7 +27,10 @@
container: ghcr.io/${{ github.repository_owner }}/livy-ci:latest
strategy:
matrix:
- spark_version: ["2.4", "3.0"]
+ maven_profile:
+ - "-Pscala-2.11 -Pspark2"
+ - "-Pscala-2.12 -Pspark2"
+ - "-Pscala-2.12 -Pspark3"
steps:
-
name: Checkout
@@ -42,13 +45,9 @@
key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml', '*/pom.xml', 'thriftserver/*/pom.xml', 'core/*/pom.xml', 'repl/*/pom.xml', 'scala-api/*/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- -
- name: Set profile based on matrix
- run: |
- if [ "${{ matrix.spark_version }}" = "3.0" ]; then echo "mvn_profile=-Pspark-3.0" >> $GITHUB_ENV; fi
-
name: Build with Maven
- run: mvn -Pthriftserver ${{ env.mvn_profile }} -DskipTests -Dmaven.javadoc.skip=true -B -V -e verify
+ run: mvn -Pthriftserver ${{ matrix.mvn_profile }} -DskipTests -Dmaven.javadoc.skip=true -B -V -e verify
-
name: Upload coverage to codecov
uses: codecov/codecov-action@v3
diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml
index 7d8c42b..552642f 100644
--- a/.github/workflows/unit-tests.yaml
+++ b/.github/workflows/unit-tests.yaml
@@ -25,7 +25,10 @@
container: ghcr.io/${{ github.repository_owner }}/livy-ci:latest
strategy:
matrix:
- spark_version: ["2.4", "3.0"]
+ maven_profile:
+ - "-Pscala-2.11 -Pspark2"
+ - "-Pscala-2.12 -Pspark2"
+ - "-Pscala-2.12 -Pspark3"
steps:
-
name: Checkout
@@ -40,13 +43,9 @@
key: ${{ runner.os }}-maven-${{ hashFiles('pom.xml', '*/pom.xml', 'thriftserver/*/pom.xml', 'core/*/pom.xml', 'repl/*/pom.xml', 'scala-api/*/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- -
- name: Set profile based on matrix
- run: |
- if [ "${{ matrix.spark_version }}" = "3.0" ]; then echo "mvn_profile=-Pspark-3.0" >> $GITHUB_ENV; fi
-
name: Build with Maven
- run: mvn -Pthriftserver ${{ env.mvn_profile }} -DskipITs -Dmaven.javadoc.skip=true -B -V -e verify
+ run: mvn -Pthriftserver ${{ matrix.mvn_profile }} -DskipITs -Dmaven.javadoc.skip=true -B -V -e verify
-
name: Upload coverage to codecov
uses: codecov/codecov-action@v3
diff --git a/.rat-excludes b/.rat-excludes
index cf49653..b22edb2 100644
--- a/.rat-excludes
+++ b/.rat-excludes
@@ -1,6 +1,7 @@
.rat-excludes
.github/*
logs/*
+**/logs/**
**/*.conf
**/*.json
**/*.md
@@ -28,4 +29,4 @@
**/jquery-2.1.1.min.js
docs/**/*.html
docs/**/JB/**
-venv/*
\ No newline at end of file
+venv/*
diff --git a/.travis.yml b/.travis.yml
index 7a12a11..d661e30 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,18 +18,28 @@
sudo: required
dist: xenial
language: scala
- - 2.11.12
matrix:
include:
- - name: "Spark 2.4 Unit Tests"
- env: MVN_FLAG='-Pthriftserver -DskipITs'
- - name: "Spark 2.4 ITs"
- env: MVN_FLAG='-Pthriftserver -DskipTests'
- - name: "Spark 3.0 Unit Tests"
- env: MVN_FLAG='-Pthriftserver -Pspark-3.0 -DskipITs'
- - name: "Spark 3.0 ITs"
- env: MVN_FLAG='-Pthriftserver -Pspark-3.0 -DskipTests'
+ - name: "Spark 2.4 Unit Tests (Scala 2.11)"
+ scala: 2.11.12
+ env: MVN_FLAG='-Pscala-2.11 -Pspark2 -Pthriftserver -DskipITs'
+ - name: "Spark 2.4 ITs (Scala 2.11)"
+ scala: 2.11.12
+ env: MVN_FLAG='-Pscala-2.11 -Pspark2 -Pthriftserver -DskipTests'
+ - name: "Spark 2.4 Unit Tests (Scala 2.12)"
+ scala: 2.12.10
+ env: MVN_FLAG='-Pscala-2.12 -Pspark2 -Pthriftserver -DskipITs'
+ - name: "Spark 2.4 ITs (Scala 2.12)"
+ scala: 2.12.10
+ env: MVN_FLAG='-Pscala-2.12 -Pspark2 -Pthriftserver -DskipTests'
+# No scala 2.11.x build for spark3
+ - name: "Spark 3.0 Unit Tests (Scala 2.12)"
+ scala: 2.12.10
+ env: MVN_FLAG='-Pscala-2.12 -Pspark3 -Pthriftserver -DskipITs'
+ - name: "Spark 3.0 ITs (Scala 2.12)"
+ scala: 2.12.10
+ env: MVN_FLAG='-Pscala-2.12 -Pspark3 -Pthriftserver -DskipITs'
jdk:
- openjdk8
diff --git a/README.md b/README.md
index 9dfec12..f3ddbad 100644
--- a/README.md
+++ b/README.md
@@ -88,3 +88,13 @@
The Livy package itself does not contain a Spark distribution. It will work with any supported
version of Spark without needing to rebuild.
+
+### Build Profiles
+
+| Flag | Purpose |
+|--------------|--------------------------------------------------------------------|
+| -Phadoop2 | Choose Hadoop2 based build dependencies (default configuration) |
+| -Pspark2 | Choose Spark 2.x based build dependencies (default configuration) |
+| -Pspark3 | Choose Spark 3.x based build dependencies |
+| -Pscala-2.11 | Choose Scala 2.11 based build dependencies (default configuration) |
+| -Pscala-2.12 | Choose scala 2.12 based build dependencies |
diff --git a/assembly/assembly.xml b/assembly/assembly.xml
index eaefbb5..d75371c 100644
--- a/assembly/assembly.xml
+++ b/assembly/assembly.xml
@@ -56,15 +56,8 @@
</includes>
</fileSet>
<fileSet>
- <directory>${project.parent.basedir}/repl/scala-2.11/target/jars</directory>
- <outputDirectory>${assembly.name}/repl_2.11-jars</outputDirectory>
- <includes>
- <include>*</include>
- </includes>
- </fileSet>
- <fileSet>
- <directory>${project.parent.basedir}/repl/scala-2.12/target/jars</directory>
- <outputDirectory>${assembly.name}/repl_2.12-jars</outputDirectory>
+ <directory>${project.parent.basedir}/repl/scala-${scala.binary.version}/target/jars</directory>
+ <outputDirectory>${assembly.name}/repl_${scala.binary.version}-jars</outputDirectory>
<includes>
<include>*</include>
</includes>
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 380a6cf..005cead 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -29,7 +29,7 @@
<packaging>pom</packaging>
<properties>
- <assembly.name>apache-livy-${project.version}-bin</assembly.name>
+ <assembly.name>apache-livy-${project.version}_${scala.binary.version}-bin</assembly.name>
<assembly.format>zip</assembly.format>
<skipDeploy>true</skipDeploy>
</properties>
@@ -43,13 +43,7 @@
<dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>livy-repl_2.11</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>livy-repl_2.12</artifactId>
+ <artifactId>livy-repl_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/core/scala-2.11/pom.xml b/core/scala-2.11/pom.xml
index 247f1d1..d9b8354 100644
--- a/core/scala-2.11/pom.xml
+++ b/core/scala-2.11/pom.xml
@@ -29,11 +29,6 @@
<relativePath>../pom.xml</relativePath>
</parent>
- <properties>
- <scala.version>${scala-2.11.version}</scala.version>
- <scala.binary.version>2.11</scala.binary.version>
- </properties>
-
<build>
<plugins>
<plugin>
diff --git a/core/scala-2.12/pom.xml b/core/scala-2.12/pom.xml
index f6e16bb..1df6475 100644
--- a/core/scala-2.12/pom.xml
+++ b/core/scala-2.12/pom.xml
@@ -29,11 +29,6 @@
<relativePath>../pom.xml</relativePath>
</parent>
- <properties>
- <scala.version>${scala-2.12.version}</scala.version>
- <scala.binary.version>2.12</scala.binary.version>
- </properties>
-
<build>
<plugins>
<plugin>
diff --git a/coverage/pom.xml b/coverage/pom.xml
index db59cc2..f7e6a9e 100644
--- a/coverage/pom.xml
+++ b/coverage/pom.xml
@@ -52,25 +52,13 @@
<dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>livy-core_2.11</artifactId>
+ <artifactId>livy-core_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>livy-core_2.12</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>livy-repl_2.11</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>livy-repl_2.12</artifactId>
+ <artifactId>livy-repl_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
@@ -88,13 +76,7 @@
<dependency>
<groupId>${project.groupId}</groupId>
- <artifactId>livy-scala-api_2.11</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>${project.groupId}</groupId>
- <artifactId>livy-scala-api_2.12</artifactId>
+ <artifactId>livy-scala-api_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/pom.xml b/pom.xml
index 46fcd51..52c64c4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -79,7 +79,6 @@
<properties>
<asynchttpclient.version>2.10.1</asynchttpclient.version>
- <hadoop.version>2.7.3</hadoop.version>
<hadoop.scope>compile</hadoop.scope>
<slf4j.version>1.7.36</slf4j.version>
<reload4j.version>1.2.25</reload4j.version>
@@ -94,33 +93,37 @@
<jackson-databind.version>2.12.7.1</jackson-databind.version>
<javax.servlet-api.version>3.1.0</javax.servlet-api.version>
<jetty.version>9.4.50.v20221201</jetty.version>
- <json4s.spark-2.11.version>3.5.3</json4s.spark-2.11.version>
- <json4s.spark-2.12.version>3.5.3</json4s.spark-2.12.version>
- <json4s.version>${json4s.spark-2.11.version}</json4s.version>
<junit.version>4.13.1</junit.version>
<libthrift.version>0.9.3</libthrift.version>
<kryo.version>4.0.2</kryo.version>
<metrics.version>3.1.0</metrics.version>
<mockito.version>1.10.19</mockito.version>
- <netty.spark-2.11.version>4.1.86.Final</netty.spark-2.11.version>
- <netty.spark-2.12.version>4.1.86.Final</netty.spark-2.12.version>
- <netty.version>${netty.spark-2.11.version}</netty.version>
+ <netty.version>4.1.86.Final</netty.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<py4j.version>0.10.7</py4j.version>
- <scala-2.11.version>2.11.12</scala-2.11.version>
- <scala-2.12.version>2.12.10</scala-2.12.version>
- <scala.binary.version>2.11</scala.binary.version>
- <scala.version>${scala-2.11.version}</scala.version>
<scalatest.version>3.0.8</scalatest.version>
<scalatra.version>2.6.5</scalatra.version>
<java.version>1.8</java.version>
<test.redirectToFile>true</test.redirectToFile>
<execution.root>${user.dir}</execution.root>
<spark.home>${execution.root}/dev/spark</spark.home>
+ <!-- profile controlled parameters, we are setting the defaults here so we can run "mvn package" with default profile values -->
+ <!-- hadoop2 -->
+ <hadoop.major-minor.version>2.7</hadoop.major-minor.version>
+ <hadoop.version>2.7.3</hadoop.version>
+ <!-- scala2.11 -->
+ <scala.binary.version>2.11</scala.binary.version>
+ <scala.version>2.11.12</scala.version>
+ <!-- spark2 -->
+ <spark.version>2.4.5</spark.version>
+ <java.version>1.8</java.version>
+ <py4j.version>0.10.9</py4j.version>
+ <json4s.version>3.5.3</json4s.version>
+ <spark.bin.name>spark-${spark.version}-bin-hadoop${hadoop.major-minor.version}</spark.bin.name>
<spark.bin.download.url>
- https://archive.apache.org/dist/spark/spark-2.4.5/spark-2.4.5-bin-hadoop2.7.tgz
+ https://archive.apache.org/dist/spark/spark-${spark.version}/${spark.bin.name}.tgz
</spark.bin.download.url>
- <spark.bin.name>spark-2.4.5-bin-hadoop2.7</spark.bin.name>
+
<!-- used for testing, NCSARequestLog use it for access log -->
<livy.log.dir>${basedir}/target</livy.log.dir>
@@ -207,19 +210,16 @@
<module>client-common</module>
<module>client-http</module>
<module>core</module>
- <module>core/scala-2.11</module>
- <module>core/scala-2.12</module>
+ <module>core/scala-${scala.binary.version}</module>
<module>coverage</module>
<module>examples</module>
<module>python-api</module>
<module>repl</module>
- <module>repl/scala-2.11</module>
- <module>repl/scala-2.12</module>
+ <module>repl/scala-${scala.binary.version}</module>
<module>rsc</module>
<module>scala</module>
<module>scala-api</module>
- <module>scala-api/scala-2.11</module>
- <module>scala-api/scala-2.12</module>
+ <module>scala-api/scala-${scala.binary.version}</module>
<module>server</module>
<module>test-lib</module>
<module>integration-test</module>
@@ -971,6 +971,7 @@
</goals>
<configuration>
<target>
+ <echo message="execRoot: ${execution.root}, buildDir: ${project.build.directory}"/>
<copy file="${execution.root}/scalastyle.xml"
tofile="${project.build.directory}/scalastyle.xml" />
<replace file="${project.build.directory}/scalastyle.xml"
@@ -1019,13 +1020,12 @@
<artifactId>maven-checkstyle-plugin</artifactId>
<version>2.17</version>
<configuration>
- <verbose>false</verbose>
<failOnViolation>true</failOnViolation>
<includeTestSourceDirectory>true</includeTestSourceDirectory>
- <failOnWarning>false</failOnWarning>
<sourceDirectory>${basedir}/src/main/java</sourceDirectory>
<testSourceDirectory>${basedir}/src/test/java</testSourceDirectory>
<configLocation>${project.build.directory}/checkstyle.xml</configLocation>
+ <headerLocation>${project.build.directory}/checkstyle.header</headerLocation>
<outputFile>${basedir}/target/checkstyle-output.xml</outputFile>
<inputEncoding>${project.build.sourceEncoding}</inputEncoding>
<outputEncoding>${project.reporting.outputEncoding}</outputEncoding>
@@ -1134,6 +1134,28 @@
<profiles>
<profile>
+ <id>hadoop2</id>
+ <properties>
+ <hadoop.major-minor.version>2.7</hadoop.major-minor.version>
+ <hadoop.version>2.7.3</hadoop.version>
+ </properties>
+ </profile>
+ <profile>
+ <id>scala-2.11</id>
+ <properties>
+ <scala.binary.version>2.11</scala.binary.version>
+ <scala.version>2.11.12</scala.version>
+ </properties>
+ </profile>
+ <profile>
+ <id>scala-2.12</id>
+ <properties>
+ <scala.binary.version>2.12</scala.binary.version>
+ <scala.version>2.12.10</scala.version>
+ </properties>
+ </profile>
+
+ <profile>
<id>thriftserver</id>
<modules>
<module>thriftserver/server</module>
@@ -1141,30 +1163,31 @@
<module>thriftserver/client</module>
</modules>
</profile>
-
<profile>
- <id>spark-3.0</id>
- <activation>
- <property>
- <name>spark-3.0</name>
- </property>
- </activation>
+ <id>spark2</id>
<properties>
- <spark.scala-2.12.version>3.0.0</spark.scala-2.12.version>
- <spark.scala-2.11.version>2.4.5</spark.scala-2.11.version>
- <spark.version>${spark.scala-2.11.version}</spark.version>
- <netty.spark-2.12.version>4.1.86.Final</netty.spark-2.12.version>
- <netty.spark-2.11.version>4.1.86.Final</netty.spark-2.11.version>
- <netty.version>${netty.spark-2.11.version}</netty.version>
+ <spark.version>2.4.5</spark.version>
<java.version>1.8</java.version>
<py4j.version>0.10.9</py4j.version>
- <json4s.spark-2.11.version>3.5.3</json4s.spark-2.11.version>
- <json4s.spark-2.12.version>3.6.6</json4s.spark-2.12.version>
- <json4s.version>${json4s.spark-2.11.version}</json4s.version>
+ <json4s.version>3.5.3</json4s.version>
+ <spark.bin.name>spark-${spark.version}-bin-hadoop${hadoop.major-minor.version}</spark.bin.name>
<spark.bin.download.url>
- https://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz
+ https://archive.apache.org/dist/spark/spark-${spark.version}/${spark.bin.name}.tgz
</spark.bin.download.url>
- <spark.bin.name>spark-3.0.0-bin-hadoop2.7</spark.bin.name>
+ </properties>
+ </profile>
+
+ <profile>
+ <id>spark3</id>
+ <properties>
+ <spark.version>3.0.0</spark.version>
+ <java.version>1.8</java.version>
+ <py4j.version>0.10.9</py4j.version>
+ <json4s.version>3.6.6</json4s.version>
+ <spark.bin.name>spark-${spark.version}-bin-hadoop${hadoop.major-minor.version}</spark.bin.name>
+ <spark.bin.download.url>
+ https://archive.apache.org/dist/spark/spark-${spark.version}/${spark.bin.name}.tgz
+ </spark.bin.download.url>
</properties>
</profile>
diff --git a/repl/scala-2.11/pom.xml b/repl/scala-2.11/pom.xml
index fb0d15f..1ab76bd 100644
--- a/repl/scala-2.11/pom.xml
+++ b/repl/scala-2.11/pom.xml
@@ -31,12 +31,4 @@
<relativePath>../pom.xml</relativePath>
</parent>
- <properties>
- <scala.version>${scala-2.11.version}</scala.version>
- <scala.binary.version>2.11</scala.binary.version>
- <spark.version>${spark.scala-2.11.version}</spark.version>
- <netty.version>${netty.spark-2.11.version}</netty.version>
- <json4s.version>${json4s.spark-2.11.version}</json4s.version>
- </properties>
-
</project>
diff --git a/repl/scala-2.12/pom.xml b/repl/scala-2.12/pom.xml
index b1904e8..cee00fb 100644
--- a/repl/scala-2.12/pom.xml
+++ b/repl/scala-2.12/pom.xml
@@ -31,12 +31,4 @@
<relativePath>../pom.xml</relativePath>
</parent>
- <properties>
- <scala.version>${scala-2.12.version}</scala.version>
- <scala.binary.version>2.12</scala.binary.version>
- <spark.version>${spark.scala-2.12.version}</spark.version>
- <netty.version>${netty.spark-2.12.version}</netty.version>
- <json4s.version>${json4s.spark-2.12.version}</json4s.version>
- </properties>
-
</project>
diff --git a/scala-api/scala-2.11/pom.xml b/scala-api/scala-2.11/pom.xml
index e06b7f2..a2ed04f 100644
--- a/scala-api/scala-2.11/pom.xml
+++ b/scala-api/scala-2.11/pom.xml
@@ -29,10 +29,4 @@
<relativePath>../pom.xml</relativePath>
</parent>
- <properties>
- <scala.version>${scala-2.11.version}</scala.version>
- <scala.binary.version>2.11</scala.binary.version>
- <spark.version>${spark.scala-2.11.version}</spark.version>
- <netty.version>${netty.spark-2.11.version}</netty.version>
- </properties>
</project>
diff --git a/scala-api/scala-2.12/pom.xml b/scala-api/scala-2.12/pom.xml
index a4f362d..4040188 100644
--- a/scala-api/scala-2.12/pom.xml
+++ b/scala-api/scala-2.12/pom.xml
@@ -28,11 +28,4 @@
<version>0.8.0-incubating-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
-
- <properties>
- <scala.version>${scala-2.12.version}</scala.version>
- <scala.binary.version>2.12</scala.binary.version>
- <spark.version>${spark.scala-2.12.version}</spark.version>
- <netty.version>${netty.spark-2.12.version}</netty.version>
- </properties>
</project>
