| <?xml version="1.0" encoding="UTF-8"?> |
| <!-- |
| ~ Licensed to the Apache Software Foundation (ASF) under one or more |
| ~ contributor license agreements. See the NOTICE file distributed with |
| ~ this work for additional information regarding copyright ownership. |
| ~ The ASF licenses this file to You under the Apache License, Version 2.0 |
| ~ (the "License"); you may not use this file except in compliance with |
| ~ the License. You may obtain a copy of the License at |
| ~ |
| ~ http://www.apache.org/licenses/LICENSE-2.0 |
| ~ |
| ~ Unless required by applicable law or agreed to in writing, software |
| ~ distributed under the License is distributed on an "AS IS" BASIS, |
| ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| ~ See the License for the specific language governing permissions and |
| ~ limitations under the License. |
| --> |
| |
| <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
| xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
| <modelVersion>4.0.0</modelVersion> |
| <parent> |
| <groupId>org.apache.spark</groupId> |
| <artifactId>spark-parent_2.12</artifactId> |
| <version>3.5.1</version> |
| <relativePath>../../pom.xml</relativePath> |
| </parent> |
| |
| <artifactId>spark-network-yarn_2.12</artifactId> |
| <packaging>jar</packaging> |
| <name>Spark Project YARN Shuffle Service</name> |
| <url>https://spark.apache.org/</url> |
| <properties> |
| <sbt.project.name>network-yarn</sbt.project.name> |
| <!-- Make sure all Hadoop dependencies are provided to avoid repackaging. --> |
| <hadoop.deps.scope>provided</hadoop.deps.scope> |
| <shuffle.jar>${project.build.directory}/scala-${scala.binary.version}/spark-${project.version}-yarn-shuffle.jar</shuffle.jar> |
| <shade>org/sparkproject/</shade> |
| </properties> |
| |
| <dependencies> |
| <!-- Core dependencies --> |
| <dependency> |
| <groupId>org.apache.spark</groupId> |
| <artifactId>spark-network-shuffle_${scala.binary.version}</artifactId> |
| <version>${project.version}</version> |
| </dependency> |
| <dependency> |
| <groupId>org.apache.spark</groupId> |
| <artifactId>spark-tags_${scala.binary.version}</artifactId> |
| <scope>test</scope> |
| </dependency> |
| |
| <!-- |
| This spark-tags test-dep is needed even though it isn't used in this module, otherwise testing-cmds that exclude |
| them will yield errors. |
| --> |
| <dependency> |
| <groupId>org.apache.spark</groupId> |
| <artifactId>spark-tags_${scala.binary.version}</artifactId> |
| <type>test-jar</type> |
| <scope>test</scope> |
| </dependency> |
| |
| <!-- Provided dependencies --> |
| <dependency> |
| <groupId>org.apache.hadoop</groupId> |
| <artifactId>hadoop-client-api</artifactId> |
| <version>${hadoop.version}</version> |
| </dependency> |
| <dependency> |
| <groupId>org.apache.hadoop</groupId> |
| <artifactId>hadoop-client-runtime</artifactId> |
| <version>${hadoop.version}</version> |
| </dependency> |
| <dependency> |
| <groupId>com.google.guava</groupId> |
| <artifactId>guava</artifactId> |
| <scope>provided</scope> |
| </dependency> |
| <dependency> |
| <groupId>org.slf4j</groupId> |
| <artifactId>slf4j-api</artifactId> |
| <scope>provided</scope> |
| </dependency> |
| </dependencies> |
| |
| <build> |
| <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory> |
| <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory> |
| <plugins> |
| <plugin> |
| <groupId>org.apache.maven.plugins</groupId> |
| <artifactId>maven-shade-plugin</artifactId> |
| <configuration> |
| <shadedArtifactAttached>false</shadedArtifactAttached> |
| <outputFile>${shuffle.jar}</outputFile> |
| <artifactSet> |
| <includes> |
| <include>*:*</include> |
| </includes> |
| <excludes> |
| <exclude>org.scala-lang:scala-library</exclude> |
| </excludes> |
| </artifactSet> |
| <filters> |
| <filter> |
| <artifact>*:*</artifact> |
| <excludes> |
| <exclude>META-INF/*.SF</exclude> |
| <exclude>META-INF/*.DSA</exclude> |
| <exclude>META-INF/*.RSA</exclude> |
| </excludes> |
| </filter> |
| </filters> |
| <relocations combine.children="append"> |
| <relocation> |
| <pattern>com.fasterxml.jackson</pattern> |
| <shadedPattern>${spark.shade.packageName}.com.fasterxml.jackson</shadedPattern> |
| <includes> |
| <include>com.fasterxml.jackson.**</include> |
| </includes> |
| </relocation> |
| <relocation> |
| <pattern>io.netty</pattern> |
| <shadedPattern>${spark.shade.packageName}.io.netty</shadedPattern> |
| <includes> |
| <include>io.netty.**</include> |
| </includes> |
| </relocation> |
| </relocations> |
| </configuration> |
| <executions> |
| <execution> |
| <phase>package</phase> |
| <goals> |
| <goal>shade</goal> |
| </goals> |
| <configuration> |
| <transformers> |
| <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> |
| </transformers> |
| </configuration> |
| </execution> |
| </executions> |
| </plugin> |
| <!-- shade the native netty libs as well --> |
| <plugin> |
| <groupId>org.codehaus.mojo</groupId> |
| <artifactId>build-helper-maven-plugin</artifactId> |
| <executions> |
| <execution> |
| <id>regex-property</id> |
| <goals> |
| <goal>regex-property</goal> |
| </goals> |
| <configuration> |
| <name>spark.shade.native.packageName</name> |
| <value>${spark.shade.packageName}</value> |
| <regex>\.</regex> |
| <replacement>_</replacement> |
| <failIfNoMatch>true</failIfNoMatch> |
| </configuration> |
| </execution> |
| </executions> |
| </plugin> |
| <plugin> |
| <groupId>org.apache.maven.plugins</groupId> |
| <artifactId>maven-antrun-plugin</artifactId> |
| <executions> |
| <execution> |
| <id>unpack</id> |
| <phase>package</phase> |
| <configuration> |
| <target> |
| <echo message="Shade netty native libraries to ${spark.shade.native.packageName}" /> |
| <unzip src="${shuffle.jar}" dest="${project.build.directory}/exploded/" /> |
| <move file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_epoll_x86_64.so" |
| tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_epoll_x86_64.so" /> |
| <move file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_kqueue_x86_64.jnilib" |
| tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_kqueue_x86_64.jnilib" /> |
| <move file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_epoll_aarch_64.so" |
| tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_epoll_aarch_64.so" /> |
| <move file="${project.build.directory}/exploded/META-INF/native/libnetty_transport_native_kqueue_aarch_64.jnilib" |
| tofile="${project.build.directory}/exploded/META-INF/native/lib${spark.shade.native.packageName}_netty_transport_native_kqueue_aarch_64.jnilib" /> |
| <jar destfile="${shuffle.jar}" basedir="${project.build.directory}/exploded" /> |
| </target> |
| </configuration> |
| <goals> |
| <goal>run</goal> |
| </goals> |
| </execution> |
| <!-- probes to validate that those dependencies which must be shaded are --> |
| <execution> |
| <phase>verify</phase> |
| <goals> |
| <goal>run</goal> |
| </goals> |
| <configuration> |
| <target> |
| <macrodef name="shaded"> |
| <attribute name="resource"/> |
| <sequential> |
| <fail message="Not found ${shade}@{resource}"> |
| <condition> |
| <not> |
| <resourceexists> |
| <zipentry zipfile="${shuffle.jar}" name="${shade}@{resource}"/> |
| </resourceexists> |
| </not> |
| </condition> |
| </fail> |
| </sequential> |
| </macrodef> |
| <echo>Verifying dependency shading</echo> |
| <shaded resource="com/fasterxml/jackson/core/JsonParser.class" /> |
| <shaded resource="com/fasterxml/jackson/annotation/JacksonAnnotation.class" /> |
| <shaded resource="com/fasterxml/jackson/databind/JsonSerializer.class" /> |
| </target> |
| </configuration> |
| </execution> |
| </executions> |
| </plugin> |
| </plugins> |
| </build> |
| </project> |