fe/be-java-extensions/preload-extensions/pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements.  See the NOTICE file
distributed with this work for additional information
regarding copyright ownership.  The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License.  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied.  See the License for the
specific language governing permissions and limitations
under the License.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <parent>
        <artifactId>be-java-extensions</artifactId>
        <groupId>org.apache.doris</groupId>
        <version>${revision}</version>
    </parent>
    <modelVersion>4.0.0</modelVersion>

    <artifactId>preload-extensions</artifactId>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <scala.binary.version>2.12</scala.binary.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.apache.parquet</groupId>
            <artifactId>parquet-format</artifactId>
            <version>2.12.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.parquet</groupId>
            <artifactId>parquet-hadoop-bundle</artifactId>
            <version>${parquet.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.doris</groupId>
            <artifactId>java-common</artifactId>
            <version>${project.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>org.apache.thrift</groupId>
                    <artifactId>libthrift</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <dependency>
            <groupId>commons-io</groupId>
            <artifactId>commons-io</artifactId>
            <version>${commons-io.version}</version>
        </dependency>
        <!-- Runtime-only: Hive-using JNI scanners (paimon-hive-connector / hudi / HiveConf)
             need org.apache.commons.lang.StringUtils (commons-lang 2.x) at runtime. It used to
             arrive transitively via fe-common's odps-sdk-core; after P4-T09 (a53f2b17b8d) made
             fe-common odps-free it was evicted from this shared preload classpath, breaking every
             scanner with NoClassDefFoundError. Restore it here (version managed by fe/pom.xml) so
             it lands in the preload-extensions runtime classpath, the shared parent of all JNI
             scanners, while staying out of java-udf's own jar. -->
        <dependency>
            <groupId>commons-lang</groupId>
            <artifactId>commons-lang</artifactId>
            <scope>runtime</scope>
        </dependency>
        <!-- Runtime-only: the BE trino-connector scanner's oshi (pulled via trino-main) initializes
             com.sun.jna.platform.linux.Udev, which calls com.sun.jna.Native.load(String, Class) (jna >= 5.0).
             JNI scanners load through JniScannerClassLoader, a parent-first URLClassLoader whose parent is the
             BE system classpath; that classpath (bin/start_be.sh) places preload-extensions ahead of
             lib/hadoop_hdfs, whose bundled jna is < 5.0. jna used to reach this shared preload classpath
             transitively via fe-common -> trino-main -> oshi, so the modern jna shadowed the hadoop copy; after
             ca840c9dbea made fe-common trino-free it was evicted, and the scanner's own bundled jna cannot win
             (parent-first resolves Native from the hadoop jar first) -> NoSuchMethodError. Restore it here (the
             shared parent of all JNI scanners), mirroring the commons-lang / fastutil-core restorations. Pin
             both core and platform to 5.13.0 (what oshi 6.4.5 pulls and what fe-core pins inline). -->
        <dependency>
            <groupId>net.java.dev.jna</groupId>
            <artifactId>jna</artifactId>
            <version>5.13.0</version>
            <scope>runtime</scope>
        </dependency>
        <dependency>
            <groupId>net.java.dev.jna</groupId>
            <artifactId>jna-platform</artifactId>
            <version>5.13.0</version>
            <scope>runtime</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.arrow</groupId>
            <artifactId>arrow-memory-unsafe</artifactId>
            <version>${arrow.version}</version>
        </dependency>
        <dependency>
            <groupId>org.apache.arrow</groupId>
            <artifactId>arrow-vector</artifactId>
            <version>${arrow.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>com.fasterxml.jackson.core</groupId>
                    <artifactId>jackson-databind</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <!--
          NOTE: the paimon OSS/S3 FileIO plugins (paimon-s3 / paimon-jindo) are intentionally
          NOT here. They register org.apache.paimon.fs.FileIOLoader service providers
          (S3Loader / JindoLoader), but the FileIOLoader interface itself lives in paimon-common,
          which is bundled in paimon-scanner, NOT on this shared/preload classpath. Because each
          JniScannerClassLoader delegates parent-first to the JVM app classloader (where this
          preload jar sits), a provider defined here could not resolve the child-only FileIOLoader
          interface, so ServiceLoader discovery in paimon-scanner's FileIO.get() failed with
          NoClassDefFoundError: FileIOLoader. The plugins are therefore co-located with the
          FileIOLoader interface in paimon-scanner (its only BE consumer) so the whole FileIO SPI
          lives in one classloader. Do not move them back here without also putting paimon-common
          on this classpath.
        -->
        <!-- For Avro and Hudi Scanner PreLoad -->
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <!-- Must be provided, we use hadoop_libs in BE's 3rd party instead -->
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-annotations</artifactId>
            <version>${hadoop.version}</version>
            <scope>provided</scope>
        </dependency>
        <dependency>
            <groupId>org.antlr</groupId>
            <artifactId>antlr4-runtime</artifactId>
            <version>${antlr4.version}</version>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-core</artifactId>
        </dependency>
        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
        </dependency>
        <dependency>
            <!-- version of spark's jackson module is error -->
            <groupId>com.fasterxml.jackson.module</groupId>
            <artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
            <version>${jackson.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>com.google.guava</groupId>
                    <artifactId>guava</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
        <!-- For JDBC Scanner PreLoad-->
        <dependency>
            <groupId>com.zaxxer</groupId>
            <artifactId>HikariCP</artifactId>
        </dependency>
        <!-- For BE CosN Access: hadoop-cos lives in the `cos` profile below;
             disabled via -Ddisable.cos=true (build.sh exclude-cos-dependencies flag). -->
        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
        </dependency>

        <!-- these dependencies are for iceberg-aws -->
        <dependency>
            <groupId>software.amazon.awssdk</groupId>
            <artifactId>s3</artifactId>
        </dependency>
        <dependency>
            <groupId>software.amazon.awssdk</groupId>
            <artifactId>sts</artifactId>
        </dependency>
        <dependency>
            <groupId>software.amazon.awssdk</groupId>
            <artifactId>kms</artifactId>
        </dependency>
        <dependency>
            <groupId>software.amazon.awssdk</groupId>
            <artifactId>glue</artifactId>
        </dependency>
        <dependency>
            <groupId>software.amazon.awssdk</groupId>
            <artifactId>dynamodb</artifactId>
        </dependency>
        <dependency>
            <groupId>software.amazon.awssdk</groupId>
            <artifactId>s3tables</artifactId>
        </dependency>
        <dependency>
            <groupId>software.amazon.awssdk</groupId>
            <artifactId>s3-transfer-manager</artifactId>
        </dependency>
        <dependency>
            <groupId>software.amazon.s3tables</groupId>
            <artifactId>s3-tables-catalog-for-iceberg</artifactId>
            <version>${s3tables.catalog.version}</version>
        </dependency>
        <!-- AWS SDK Core -->
        <dependency>
            <groupId>software.amazon.awssdk</groupId>
            <artifactId>sdk-core</artifactId>
        </dependency>

        <!-- fastutil-core: this jar transitively bundles fe-foundation's ConcurrentLong2LongHashMap /
             ConcurrentLong2ObjectHashMap, which extend fastutil abstract maps. fe-foundation marks
             fastutil-core optional (to stay out of the FE SPI plugin zips), so it is not propagated
             here. BE's ScannerLoader eagerly loads every org.apache.doris.* class at startup, so the
             fastutil superclasses must be bundled or the BE fails JNI init and cannot start. -->
        <dependency>
            <groupId>it.unimi.dsi</groupId>
            <artifactId>fastutil-core</artifactId>
        </dependency>
    </dependencies>

    <!-- Tencent COS support. Active by default; disabled by -Ddisable.cos=true
         (build.sh exclude-cos-dependencies flag), which drops hadoop-cos so nothing from
         com.qcloud.cos is resolved or bundled into preload-extensions. -->
    <profiles>
        <profile>
            <id>cos</id>
            <activation>
                <property>
                    <name>!disable.cos</name>
                </property>
            </activation>
            <dependencies>
                <!-- For BE CosN Access -->
                <dependency>
                    <groupId>com.qcloud.cos</groupId>
                    <artifactId>hadoop-cos</artifactId>
                    <version>${tencentcos.version}</version>
                    <exclusions>
                        <exclusion>
                            <groupId>org.json</groupId>
                            <artifactId>json</artifactId>
                        </exclusion>
                    </exclusions>
                </dependency>
            </dependencies>
        </profile>
    </profiles>

    <build>
        <finalName>preload-extensions</finalName>
        <directory>${project.basedir}/target/</directory>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-assembly-plugin</artifactId>
                <executions>
                    <execution>
                        <id>make-assembly-deps</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                        <configuration>
                            <descriptors>
                            <descriptor>src/main/resources/package-deps.xml</descriptor>
                            </descriptors>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>