| <?xml version="1.0" encoding="UTF-8"?> |
| <!-- |
| ~ Licensed to the Apache Software Foundation (ASF) under one or more |
| ~ contributor license agreements. See the NOTICE file distributed with |
| ~ this work for additional information regarding copyright ownership. |
| ~ The ASF licenses this file to You under the Apache License, Version 2.0 |
| ~ (the "License"); you may not use this file except in compliance with |
| ~ the License. You may obtain a copy of the License at |
| ~ |
| ~ http://www.apache.org/licenses/LICENSE-2.0 |
| ~ |
| ~ Unless required by applicable law or agreed to in writing, software |
| ~ distributed under the License is distributed on an "AS IS" BASIS, |
| ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| ~ See the License for the specific language governing permissions and |
| ~ limitations under the License. |
| --> |
| |
| <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
| xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> |
| <modelVersion>4.0.0</modelVersion> |
| |
| <parent> |
| <artifactId>spark-parent</artifactId> |
| <groupId>org.apache.zeppelin</groupId> |
| <version>0.11.0-SNAPSHOT</version> |
| <relativePath>../pom.xml</relativePath> |
| </parent> |
| |
| <artifactId>spark-interpreter</artifactId> |
| <packaging>jar</packaging> |
| <name>Zeppelin: Spark Interpreter</name> |
| <description>Zeppelin spark support</description> |
| |
| <properties> |
| <interpreter.name>spark</interpreter.name> |
| <!--library versions--> |
| <maven.plugin.api.version>3.0</maven.plugin.api.version> |
| <aether.version>1.12</aether.version> |
| <maven.aeither.provider.version>3.0.3</maven.aeither.provider.version> |
| <wagon.version>2.7</wagon.version> |
| |
| <datanucleus.rdbms.version>3.2.9</datanucleus.rdbms.version> |
| <datanucleus.apijdo.version>3.2.6</datanucleus.apijdo.version> |
| <datanucleus.core.version>3.2.10</datanucleus.core.version> |
| |
| <!-- spark versions --> |
| <spark.version>3.1.2</spark.version> |
| <protobuf.version>2.5.0</protobuf.version> |
| <py4j.version>0.10.9</py4j.version> |
| <spark.scala.version>2.12.7</spark.scala.version> |
| <spark.scala.binary.version>2.12</spark.scala.binary.version> |
| |
| <spark.archive>spark-${spark.version}</spark.archive> |
| <spark.src.download.url> |
| https://archive.apache.org/dist/spark/${spark.archive}/${spark.archive}.tgz |
| </spark.src.download.url> |
| <spark.bin.download.url> |
| https://archive.apache.org/dist/spark/${spark.archive}/${spark.archive}-bin-without-hadoop.tgz |
| </spark.bin.download.url> |
| |
| <scala.compile.version>${spark.scala.version}</scala.compile.version> |
| <!-- settings --> |
| <pyspark.test.exclude>**/PySparkInterpreterMatplotlibTest.java</pyspark.test.exclude> |
| <pyspark.test.include>**/*Test.*</pyspark.test.include> |
| </properties> |
| |
| <dependencies> |
| <dependency> |
| <groupId>org.apache.zeppelin</groupId> |
| <artifactId>zeppelin-display</artifactId> |
| <version>${project.version}</version> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.zeppelin</groupId> |
| <artifactId>spark2-shims</artifactId> |
| <version>${project.version}</version> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.zeppelin</groupId> |
| <artifactId>spark3-shims</artifactId> |
| <version>${project.version}</version> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.zeppelin</groupId> |
| <artifactId>zeppelin-kotlin</artifactId> |
| <version>${project.version}</version> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.zeppelin</groupId> |
| <artifactId>zeppelin-python</artifactId> |
| <version>${project.version}</version> |
| <exclusions> |
| <exclusion> |
| <groupId>net.sf.py4j</groupId> |
| <artifactId>py4j</artifactId> |
| </exclusion> |
| </exclusions> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.zeppelin</groupId> |
| <artifactId>zeppelin-jupyter-interpreter</artifactId> |
| <version>${project.version}</version> |
| <exclusions> |
| <exclusion> |
| <groupId>net.sf.py4j</groupId> |
| <artifactId>py4j</artifactId> |
| </exclusion> |
| <exclusion> |
| <groupId>io.netty</groupId> |
| <artifactId>*</artifactId> |
| </exclusion> |
| </exclusions> |
| </dependency> |
| |
| <dependency> |
| <groupId>${project.groupId}</groupId> |
| <artifactId>zeppelin-python</artifactId> |
| <version>${project.version}</version> |
| <classifier>tests</classifier> |
| <scope>test</scope> |
| <exclusions> |
| <exclusion> |
| <groupId>net.sf.py4j</groupId> |
| <artifactId>py4j</artifactId> |
| </exclusion> |
| <exclusion> |
| <groupId>io.netty</groupId> |
| <artifactId>*</artifactId> |
| </exclusion> |
| </exclusions> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.zeppelin</groupId> |
| <artifactId>r</artifactId> |
| <version>${project.version}</version> |
| <classifier>tests</classifier> |
| <scope>test</scope> |
| <exclusions> |
| <exclusion> |
| <groupId>org.apache.spark</groupId> |
| <artifactId>spark-core_2.11</artifactId> |
| </exclusion> |
| </exclusions> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.zeppelin</groupId> |
| <artifactId>zeppelin-jupyter-interpreter</artifactId> |
| <version>${project.version}</version> |
| <classifier>tests</classifier> |
| <scope>test</scope> |
| <exclusions> |
| <exclusion> |
| <groupId>net.sf.py4j</groupId> |
| <artifactId>py4j</artifactId> |
| </exclusion> |
| <exclusion> |
| <groupId>io.netty</groupId> |
| <artifactId>*</artifactId> |
| </exclusion> |
| </exclusions> |
| </dependency> |
| |
| <dependency> |
| <groupId>${project.groupId}</groupId> |
| <artifactId>r</artifactId> |
| <version>${project.version}</version> |
| <exclusions> |
| <exclusion> |
| <groupId>*</groupId> |
| <artifactId>*</artifactId> |
| </exclusion> |
| </exclusions> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.spark</groupId> |
| <artifactId>spark-repl_${spark.scala.binary.version}</artifactId> |
| <version>${spark.version}</version> |
| <scope>provided</scope> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.spark</groupId> |
| <artifactId>spark-core_${spark.scala.binary.version}</artifactId> |
| <version>${spark.version}</version> |
| <scope>provided</scope> |
| <exclusions> |
| <exclusion> |
| <groupId>org.apache.hadoop</groupId> |
| <artifactId>hadoop-client</artifactId> |
| </exclusion> |
| </exclusions> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.hadoop</groupId> |
| <artifactId>hadoop-client</artifactId> |
| <version>${hadoop.version}</version> |
| <scope>provided</scope> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.hadoop</groupId> |
| <artifactId>hadoop-common</artifactId> |
| <version>${hadoop.version}</version> |
| <scope>provided</scope> |
| <exclusions> |
| <exclusion> |
| <groupId>com.google.protobuf</groupId> |
| <artifactId>protobuf-java</artifactId> |
| </exclusion> |
| <exclusion> |
| <groupId>commons-lang</groupId> |
| <artifactId>commons-lang</artifactId> |
| </exclusion> |
| </exclusions> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.spark</groupId> |
| <artifactId>spark-hive_${spark.scala.binary.version}</artifactId> |
| <version>${spark.version}</version> |
| <scope>provided</scope> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.commons</groupId> |
| <artifactId>commons-exec</artifactId> |
| <version>${commons.exec.version}</version> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.scala-lang</groupId> |
| <artifactId>scala-library</artifactId> |
| <version>${spark.scala.version}</version> |
| <scope>provided</scope> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.scala-lang</groupId> |
| <artifactId>scala-compiler</artifactId> |
| <version>${spark.scala.version}</version> |
| <scope>provided</scope> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.scala-lang</groupId> |
| <artifactId>scala-reflect</artifactId> |
| <version>${spark.scala.version}</version> |
| <scope>provided</scope> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.commons</groupId> |
| <artifactId>commons-lang3</artifactId> |
| <scope>provided</scope> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.apache.commons</groupId> |
| <artifactId>commons-compress</artifactId> |
| <version>${commons.compress.version}</version> |
| <scope>provided</scope> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.jsoup</groupId> |
| <artifactId>jsoup</artifactId> |
| <version>${jsoup.version}</version> |
| </dependency> |
| |
| <!--test libraries--> |
| <dependency> |
| <groupId>org.datanucleus</groupId> |
| <artifactId>datanucleus-core</artifactId> |
| <version>${datanucleus.core.version}</version> |
| <scope>test</scope> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.datanucleus</groupId> |
| <artifactId>datanucleus-api-jdo</artifactId> |
| <version>${datanucleus.apijdo.version}</version> |
| <scope>test</scope> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.datanucleus</groupId> |
| <artifactId>datanucleus-rdbms</artifactId> |
| <version>${datanucleus.rdbms.version}</version> |
| <scope>test</scope> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.mockito</groupId> |
| <artifactId>mockito-core</artifactId> |
| <scope>test</scope> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.powermock</groupId> |
| <artifactId>powermock-api-mockito2</artifactId> |
| <scope>test</scope> |
| </dependency> |
| |
| <dependency> |
| <groupId>org.powermock</groupId> |
| <artifactId>powermock-module-junit4</artifactId> |
| <scope>test</scope> |
| <exclusions> |
| <exclusion> |
| <groupId>org.hamcrest</groupId> |
| <artifactId>hamcrest-core</artifactId> |
| </exclusion> |
| </exclusions> |
| </dependency> |
| |
| <dependency> |
| <groupId>net.jodah</groupId> |
| <artifactId>concurrentunit</artifactId> |
| <version>0.4.4</version> |
| <scope>test</scope> |
| </dependency> |
| |
| <dependency> |
| <groupId>com.mashape.unirest</groupId> |
| <artifactId>unirest-java</artifactId> |
| <version>1.4.9</version> |
| <scope>test</scope> |
| </dependency> |
| |
| </dependencies> |
| |
| <build> |
| <plugins> |
| |
| <plugin> |
| <groupId>com.googlecode.maven-download-plugin</groupId> |
| <artifactId>download-maven-plugin</artifactId> |
| <executions> |
| <!-- include pyspark by default --> |
| <execution> |
| <id>download-pyspark-files</id> |
| <phase>validate</phase> |
| <goals> |
| <goal>wget</goal> |
| </goals> |
| <configuration> |
| <readTimeOut>60000</readTimeOut> |
| <retries>5</retries> |
| <unpack>true</unpack> |
| <url>${spark.src.download.url}</url> |
| <outputDirectory>${project.build.directory}</outputDirectory> |
| </configuration> |
| </execution> |
| <!-- include sparkr by default --> |
| <execution> |
| <id>download-sparkr-files</id> |
| <phase>validate</phase> |
| <goals> |
| <goal>wget</goal> |
| </goals> |
| <configuration> |
| <readTimeOut>60000</readTimeOut> |
| <retries>5</retries> |
| <url>${spark.bin.download.url}</url> |
| <unpack>true</unpack> |
| <outputDirectory>${project.build.directory}</outputDirectory> |
| </configuration> |
| </execution> |
| </executions> |
| </plugin> |
| |
| <plugin> |
| <groupId>org.apache.maven.plugins</groupId> |
| <artifactId>maven-antrun-plugin</artifactId> |
| <executions> |
| <execution> |
| <id>zip-pyspark-files</id> |
| <phase>generate-resources</phase> |
| <goals> |
| <goal>run</goal> |
| </goals> |
| <configuration> |
| <target> |
| <delete dir="../../interpreter/spark/pyspark" /> |
| <copy file="${project.build.directory}/${spark.archive}/python/lib/py4j-${py4j.version}-src.zip" todir="${project.build.directory}/../../../interpreter/spark/pyspark" /> |
| <zip basedir="${project.build.directory}/${spark.archive}/python" destfile="${project.build.directory}/../../../interpreter/spark/pyspark/pyspark.zip" includes="pyspark/*.py,pyspark/**/*.py" /> |
| </target> |
| </configuration> |
| </execution> |
| </executions> |
| </plugin> |
| |
| <plugin> |
| <artifactId>maven-resources-plugin</artifactId> |
| <executions> |
| <execution> |
| <id>copy-sparkr-files</id> |
| <phase>generate-resources</phase> |
| <goals> |
| <goal>copy-resources</goal> |
| </goals> |
| <configuration> |
| <outputDirectory>${project.build.directory}/../../../interpreter/spark/R/lib</outputDirectory> |
| <resources> |
| <resource> |
| <directory> |
| ${project.build.directory}/spark-${spark.version}-bin-without-hadoop/R/lib |
| </directory> |
| </resource> |
| </resources> |
| </configuration> |
| </execution> |
| <execution> |
| <id>copy-interpreter-setting</id> |
| <phase>package</phase> |
| <goals> |
| <goal>resources</goal> |
| </goals> |
| <configuration> |
| <outputDirectory>${project.build.directory}/../../../interpreter/${interpreter.name}</outputDirectory> |
| </configuration> |
| </execution> |
| </executions> |
| </plugin> |
| |
| <plugin> |
| <groupId>org.scalatest</groupId> |
| <artifactId>scalatest-maven-plugin</artifactId> |
| </plugin> |
| |
| <plugin> |
| <groupId>org.apache.maven.plugins</groupId> |
| <artifactId>maven-surefire-plugin</artifactId> |
| <configuration> |
| <forkCount>1</forkCount> |
| <reuseForks>false</reuseForks> |
| <argLine>-Xmx3072m -XX:MaxMetaspaceSize=256m</argLine> |
| <excludes> |
| <exclude>${pyspark.test.exclude}</exclude> |
| <exclude>${tests.to.exclude}</exclude> |
| </excludes> |
| <environmentVariables> |
| <PYTHONPATH>${project.build.directory}/../../../interpreter/spark/pyspark/pyspark.zip:${project.build.directory}/../../../interpreter/spark/pyspark/py4j-${py4j.version}-src.zip</PYTHONPATH> |
| <ZEPPELIN_HOME>${basedir}/../../</ZEPPELIN_HOME> |
| </environmentVariables> |
| </configuration> |
| </plugin> |
| |
| <plugin> |
| <groupId>org.apache.maven.plugins</groupId> |
| <artifactId>maven-shade-plugin</artifactId> |
| <configuration> |
| <filters> |
| <filter> |
| <artifact>*:*</artifact> |
| <excludes> |
| <exclude>org/datanucleus/**</exclude> |
| <exclude>META-INF/*.SF</exclude> |
| <exclude>META-INF/*.DSA</exclude> |
| <exclude>META-INF/*.RSA</exclude> |
| </excludes> |
| </filter> |
| </filters> |
| |
| <artifactSet> |
| <excludes> |
| <exclude>org.scala-lang:scala-library</exclude> |
| <exclude>org.scala-lang:scala-compiler</exclude> |
| <exclude>org.scala-lang:scala-reflect</exclude> |
| <exclude>commons-lang:commons-lang</exclude> |
| <exclude>org.apache.commons:commons-compress</exclude> |
| <exclude>org.apache.zeppelin:zeppelin-interpreter-shaded</exclude> |
| </excludes> |
| </artifactSet> |
| |
| <transformers> |
| <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" /> |
| <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> |
| <resource>reference.conf</resource> |
| </transformer> |
| </transformers> |
| <relocations> |
| <relocation> |
| <pattern>io.netty</pattern> |
| <shadedPattern>org.apache.zeppelin.io.netty</shadedPattern> |
| </relocation> |
| <relocation> |
| <pattern>com.google</pattern> |
| <shadedPattern>org.apache.zeppelin.com.google</shadedPattern> |
| </relocation> |
| <relocation> |
| <pattern>com.facebook.fb303</pattern> |
| <shadedPattern>org.apache.zeppelin.com.facebook.fb303</shadedPattern> |
| </relocation> |
| </relocations> |
| <outputFile>${project.basedir}/../../interpreter/${interpreter.name}/${project.artifactId}-${project.version}.jar</outputFile> |
| </configuration> |
| <executions> |
| <execution> |
| <phase>package</phase> |
| <goals> |
| <goal>shade</goal> |
| </goals> |
| </execution> |
| </executions> |
| </plugin> |
| </plugins> |
| </build> |
| |
| <profiles> |
| |
| <!-- profile spark-scala-x only affect the unit test in spark/interpreter module --> |
| |
| <profile> |
| <id>spark-scala-2.13</id> |
| <properties> |
| <spark.scala.version>2.13.4</spark.scala.version> |
| <spark.scala.binary.version>2.13</spark.scala.binary.version> |
| </properties> |
| </profile> |
| |
| <profile> |
| <id>spark-scala-2.12</id> |
| <activation> |
| <activeByDefault>true</activeByDefault> |
| </activation> |
| <properties> |
| <spark.scala.version>2.12.7</spark.scala.version> |
| <spark.scala.binary.version>2.12</spark.scala.binary.version> |
| </properties> |
| </profile> |
| |
| <profile> |
| <id>spark-scala-2.11</id> |
| <properties> |
| <spark.scala.version>2.11.12</spark.scala.version> |
| <spark.scala.binary.version>2.11</spark.scala.binary.version> |
| </properties> |
| </profile> |
| |
| <!-- profile spark-x only affect spark version used in test --> |
| |
| <profile> |
| <id>spark-3.3</id> |
| <activation> |
| <activeByDefault>true</activeByDefault> |
| </activation> |
| <properties> |
| <datanucleus.core.version>4.1.17</datanucleus.core.version> |
| <datanucleus.rdbms.version>4.1.19</datanucleus.rdbms.version> |
| <datanucleus.apijdo.version>4.2.4</datanucleus.apijdo.version> |
| <spark.version>3.3.0</spark.version> |
| <protobuf.version>2.5.0</protobuf.version> |
| <py4j.version>0.10.9.5</py4j.version> |
| </properties> |
| </profile> |
| |
| <profile> |
| <id>spark-3.2</id> |
| <properties> |
| <datanucleus.core.version>4.1.17</datanucleus.core.version> |
| <datanucleus.rdbms.version>4.1.19</datanucleus.rdbms.version> |
| <datanucleus.apijdo.version>4.2.4</datanucleus.apijdo.version> |
| <spark.version>3.2.0</spark.version> |
| <protobuf.version>2.5.0</protobuf.version> |
| <py4j.version>0.10.9.2</py4j.version> |
| </properties> |
| </profile> |
| |
| <profile> |
| <id>spark-3.1</id> |
| <properties> |
| <datanucleus.core.version>4.1.17</datanucleus.core.version> |
| <datanucleus.rdbms.version>4.1.19</datanucleus.rdbms.version> |
| <datanucleus.apijdo.version>4.2.4</datanucleus.apijdo.version> |
| <spark.version>3.1.2</spark.version> |
| <protobuf.version>2.5.0</protobuf.version> |
| <py4j.version>0.10.9</py4j.version> |
| </properties> |
| </profile> |
| |
| <profile> |
| <id>spark-3.0</id> |
| <properties> |
| <datanucleus.core.version>4.1.17</datanucleus.core.version> |
| <datanucleus.rdbms.version>4.1.19</datanucleus.rdbms.version> |
| <datanucleus.apijdo.version>4.2.4</datanucleus.apijdo.version> |
| <spark.version>3.0.3</spark.version> |
| <protobuf.version>2.5.0</protobuf.version> |
| <py4j.version>0.10.9</py4j.version> |
| </properties> |
| </profile> |
| |
| <profile> |
| <id>spark-2.4</id> |
| <properties> |
| <spark.version>2.4.5</spark.version> |
| <protobuf.version>2.5.0</protobuf.version> |
| <py4j.version>0.10.7</py4j.version> |
| </properties> |
| </profile> |
| |
| </profiles> |
| |
| </project> |