feat(loader): bundle native library inside the JAR (#77)
diff --git a/core/pom.xml b/core/pom.xml
index 5eddb3b..524282d 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -34,6 +34,12 @@
<name>Apache DataFusion Java</name>
+ <properties>
+ <!-- Which cargo build profile (debug/release) the bundled native
+ library is copied from. Override with -Ddatafusion.native.profile=release. -->
+ <datafusion.native.profile>debug</datafusion.native.profile>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.junit.jupiter</groupId>
@@ -77,10 +83,35 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
- <argLine>-Djava.library.path=${maven.multiModuleProjectDirectory}/native/target/debug --add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
+ <argLine>--add-opens=java.base/java.nio=ALL-UNNAMED</argLine>
</configuration>
</plugin>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>copy-native-lib</id>
+ <phase>process-classes</phase>
+ <goals><goal>run</goal></goals>
+ <configuration>
+ <target>
+ <property name="datafusion.native.lib.source"
+ value="${maven.multiModuleProjectDirectory}/native/target/${datafusion.native.profile}/${datafusion.lib.filename}"/>
+ <fail message="Native library not found at ${datafusion.native.lib.source}. Run 'cd native && cargo build' (or 'make') before building the JAR.">
+ <condition><not><available file="${datafusion.native.lib.source}"/></not></condition>
+ </fail>
+ <mkdir dir="${project.build.outputDirectory}/org/apache/datafusion/${datafusion.lib.os}/${datafusion.lib.arch}"/>
+ <copy file="${datafusion.native.lib.source}"
+ tofile="${project.build.outputDirectory}/org/apache/datafusion/${datafusion.lib.os}/${datafusion.lib.arch}/${datafusion.lib.filename}"
+ preservelastmodified="true"
+ verbose="true"/>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
<groupId>com.googlecode.maven-download-plugin</groupId>
<artifactId>download-maven-plugin</artifactId>
<version>1.9.0</version>
@@ -149,4 +180,85 @@
</plugin>
</plugins>
</build>
+
+ <profiles>
+ <!--
+ One of these profiles activates per host so the antrun copy
+ knows the OS/arch directory and library filename to use.
+ Resource layout follows the bundling convention:
+
+ org/apache/datafusion/linux/amd64/libdatafusion_jni.so
+ org/apache/datafusion/linux/aarch64/libdatafusion_jni.so
+ org/apache/datafusion/darwin/x86_64/libdatafusion_jni.dylib
+ org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib
+ org/apache/datafusion/windows/amd64/datafusion_jni.dll
+ -->
+ <profile>
+ <id>native-linux-amd64</id>
+ <activation>
+ <os><family>unix</family><name>linux</name><arch>amd64</arch></os>
+ </activation>
+ <properties>
+ <datafusion.lib.os>linux</datafusion.lib.os>
+ <datafusion.lib.arch>amd64</datafusion.lib.arch>
+ <datafusion.lib.filename>libdatafusion_jni.so</datafusion.lib.filename>
+ </properties>
+ </profile>
+ <profile>
+ <id>native-linux-aarch64</id>
+ <activation>
+ <os><family>unix</family><name>linux</name><arch>aarch64</arch></os>
+ </activation>
+ <properties>
+ <datafusion.lib.os>linux</datafusion.lib.os>
+ <datafusion.lib.arch>aarch64</datafusion.lib.arch>
+ <datafusion.lib.filename>libdatafusion_jni.so</datafusion.lib.filename>
+ </properties>
+ </profile>
+ <profile>
+ <id>native-mac-x86_64</id>
+ <activation>
+ <os><family>mac</family><arch>x86_64</arch></os>
+ </activation>
+ <properties>
+ <datafusion.lib.os>darwin</datafusion.lib.os>
+ <datafusion.lib.arch>x86_64</datafusion.lib.arch>
+ <datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
+ </properties>
+ </profile>
+ <profile>
+ <!-- Some JVMs report os.arch=amd64 even on macOS x86_64. -->
+ <id>native-mac-amd64</id>
+ <activation>
+ <os><family>mac</family><arch>amd64</arch></os>
+ </activation>
+ <properties>
+ <datafusion.lib.os>darwin</datafusion.lib.os>
+ <datafusion.lib.arch>x86_64</datafusion.lib.arch>
+ <datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
+ </properties>
+ </profile>
+ <profile>
+ <id>native-mac-aarch64</id>
+ <activation>
+ <os><family>mac</family><arch>aarch64</arch></os>
+ </activation>
+ <properties>
+ <datafusion.lib.os>darwin</datafusion.lib.os>
+ <datafusion.lib.arch>aarch64</datafusion.lib.arch>
+ <datafusion.lib.filename>libdatafusion_jni.dylib</datafusion.lib.filename>
+ </properties>
+ </profile>
+ <profile>
+ <id>native-windows-amd64</id>
+ <activation>
+ <os><family>windows</family><arch>amd64</arch></os>
+ </activation>
+ <properties>
+ <datafusion.lib.os>windows</datafusion.lib.os>
+ <datafusion.lib.arch>amd64</datafusion.lib.arch>
+ <datafusion.lib.filename>datafusion_jni.dll</datafusion.lib.filename>
+ </properties>
+ </profile>
+ </profiles>
</project>
diff --git a/core/src/main/java/org/apache/datafusion/NativeLibraryLoader.java b/core/src/main/java/org/apache/datafusion/NativeLibraryLoader.java
index 2c74341..426bd7d 100644
--- a/core/src/main/java/org/apache/datafusion/NativeLibraryLoader.java
+++ b/core/src/main/java/org/apache/datafusion/NativeLibraryLoader.java
@@ -19,16 +19,96 @@
package org.apache.datafusion;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+/**
+ * Loads the {@code datafusion_jni} native library on demand.
+ *
+ * <p>The loader first tries {@link System#loadLibrary(String)} so that operators can override the
+ * bundled library by placing a build on {@code java.library.path} (for example via {@code
+ * -Djava.library.path=...} or {@code LD_LIBRARY_PATH}). If that fails the loader extracts the
+ * platform-specific library from the JAR resource tree and loads it via {@link
+ * System#load(String)}.
+ *
+ * <p>The bundled libraries live at the conventional path {@code
+ * org/apache/datafusion/<os>/<arch>/<libfile>}. Each JVM extracts into its own
+ * private temporary directory created with {@link Files#createTempDirectory(String,
+ * java.nio.file.attribute.FileAttribute[])}, which is mode 0700 on POSIX and user-private under the
+ * default Windows {@code TMP}. Keeping the directory private prevents another user on the host from
+ * planting a same-named library for the JVM to pick up.
+ */
public final class NativeLibraryLoader {
- private static final String LIBRARY_NAME = "datafusion_jni";
- private static boolean loaded = false;
+
+ private static final String TMP_DIR_PREFIX = "datafusion-java-";
+
+ private static volatile boolean loaded;
private NativeLibraryLoader() {}
public static synchronized void loadLibrary() {
- if (!loaded) {
- System.loadLibrary(LIBRARY_NAME);
- loaded = true;
+ if (loaded) {
+ return;
}
+ if (tryLoadFromLibraryPath()) {
+ loaded = true;
+ return;
+ }
+ loadFromClasspath();
+ loaded = true;
+ }
+
+ private static boolean tryLoadFromLibraryPath() {
+ try {
+ System.loadLibrary(Platform.LIBRARY_NAME);
+ return true;
+ } catch (UnsatisfiedLinkError ignored) {
+ return false;
+ }
+ }
+
+ private static void loadFromClasspath() {
+ Platform platform = Platform.current();
+ String resource = platform.resourcePath();
+ try (InputStream check = NativeLibraryLoader.class.getResourceAsStream(resource)) {
+ if (check == null) {
+ throw new UnsatisfiedLinkError(
+ "No bundled datafusion_jni library for "
+ + platform
+ + " (expected classpath:"
+ + resource
+ + ")."
+ + " Build the native crate and add it to java.library.path,"
+ + " or depend on a JAR built for this platform.");
+ }
+ } catch (IOException e) {
+ throw linkError("Failed to probe " + resource, e);
+ }
+
+ try {
+ Path extracted = extractToPrivateTempDir(resource, platform.libFileName());
+ System.load(extracted.toAbsolutePath().toString());
+ } catch (IOException e) {
+ throw linkError("Failed to extract " + resource, e);
+ }
+ }
+
+ private static Path extractToPrivateTempDir(String resource, String fileName) throws IOException {
+ Path tmpDir = Files.createTempDirectory(TMP_DIR_PREFIX);
+ tmpDir.toFile().deleteOnExit();
+ Path target = tmpDir.resolve(fileName);
+ target.toFile().deleteOnExit();
+ try (InputStream in = NativeLibraryLoader.class.getResourceAsStream(resource)) {
+ Files.copy(in, target);
+ }
+ return target;
+ }
+
+ private static UnsatisfiedLinkError linkError(String message, Throwable cause) {
+ UnsatisfiedLinkError err = new UnsatisfiedLinkError(message + ": " + cause.getMessage());
+ err.initCause(cause);
+ return err;
}
}
diff --git a/core/src/main/java/org/apache/datafusion/Platform.java b/core/src/main/java/org/apache/datafusion/Platform.java
new file mode 100644
index 0000000..9c3f0a5
--- /dev/null
+++ b/core/src/main/java/org/apache/datafusion/Platform.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datafusion;
+
+import java.util.Locale;
+
+/**
+ * Identifies a target OS/arch pair and the path at which the bundled native library is published
+ * inside the JAR.
+ *
+ * <p>Resource layout follows the convention from the {@code datafusion-java} packaging design:
+ *
+ * <pre>
+ * org/apache/datafusion/linux/amd64/libdatafusion_jni.so
+ * org/apache/datafusion/linux/aarch64/libdatafusion_jni.so
+ * org/apache/datafusion/darwin/x86_64/libdatafusion_jni.dylib
+ * org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib
+ * org/apache/datafusion/windows/amd64/datafusion_jni.dll
+ * </pre>
+ *
+ * <p>Package-private; consumed only by {@link NativeLibraryLoader}.
+ */
+final class Platform {
+
+ static final String LIBRARY_NAME = "datafusion_jni";
+ static final String RESOURCE_PREFIX = "org/apache/datafusion";
+
+ enum Os {
+ LINUX("linux", "lib", "so"),
+ DARWIN("darwin", "lib", "dylib"),
+ WINDOWS("windows", "", "dll");
+
+ final String dirName;
+ final String libPrefix;
+ final String libSuffix;
+
+ Os(String dirName, String libPrefix, String libSuffix) {
+ this.dirName = dirName;
+ this.libPrefix = libPrefix;
+ this.libSuffix = libSuffix;
+ }
+ }
+
+ final Os os;
+ final String arch;
+
+ private Platform(Os os, String arch) {
+ this.os = os;
+ this.arch = arch;
+ }
+
+ static Platform current() {
+ return of(System.getProperty("os.name"), System.getProperty("os.arch"));
+ }
+
+ static Platform of(String osName, String osArch) {
+ Os os = detectOs(osName);
+ String arch = detectArch(os, osArch);
+ return new Platform(os, arch);
+ }
+
+ static Os detectOs(String osName) {
+ if (osName == null) {
+ throw new UnsupportedOperationException("os.name is not set");
+ }
+ String n = osName.toLowerCase(Locale.ROOT);
+ if (n.startsWith("linux")) {
+ return Os.LINUX;
+ }
+ if (n.startsWith("mac") || n.contains("darwin")) {
+ return Os.DARWIN;
+ }
+ if (n.startsWith("windows")) {
+ return Os.WINDOWS;
+ }
+ throw new UnsupportedOperationException("Unsupported OS for datafusion_jni: " + osName);
+ }
+
+ /**
+ * Returns the architecture segment used in the resource path for {@code os}.
+ *
+ * <p>Linux and Windows use {@code amd64} (Java's preferred name for x86_64), while macOS uses
+ * {@code x86_64}; all use {@code aarch64} for ARM64.
+ */
+ static String detectArch(Os os, String osArch) {
+ if (osArch == null) {
+ throw new UnsupportedOperationException("os.arch is not set");
+ }
+ String n = osArch.toLowerCase(Locale.ROOT);
+ boolean isX64 = n.equals("amd64") || n.equals("x86_64") || n.equals("x64");
+ boolean isArm64 = n.equals("aarch64") || n.equals("arm64");
+ if (isX64) {
+ return os == Os.DARWIN ? "x86_64" : "amd64";
+ }
+ if (isArm64) {
+ return "aarch64";
+ }
+ throw new UnsupportedOperationException(
+ "Unsupported CPU architecture for datafusion_jni: " + osArch);
+ }
+
+ String libFileName() {
+ return os.libPrefix + LIBRARY_NAME + "." + os.libSuffix;
+ }
+
+ /**
+ * Absolute classpath resource path (with leading slash) of the bundled native library for this
+ * platform.
+ */
+ String resourcePath() {
+ return "/" + RESOURCE_PREFIX + "/" + os.dirName + "/" + arch + "/" + libFileName();
+ }
+
+ @Override
+ public String toString() {
+ return os.dirName + "/" + arch;
+ }
+}
diff --git a/core/src/test/java/org/apache/datafusion/PlatformTest.java b/core/src/test/java/org/apache/datafusion/PlatformTest.java
new file mode 100644
index 0000000..21f0ee3
--- /dev/null
+++ b/core/src/test/java/org/apache/datafusion/PlatformTest.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datafusion;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import org.junit.jupiter.api.Test;
+
+class PlatformTest {
+
+ @Test
+ void detectsLinuxFromOsName() {
+ assertEquals(Platform.Os.LINUX, Platform.detectOs("Linux"));
+ assertEquals(Platform.Os.LINUX, Platform.detectOs("linux"));
+ }
+
+ @Test
+ void detectsDarwinFromOsName() {
+ assertEquals(Platform.Os.DARWIN, Platform.detectOs("Mac OS X"));
+ assertEquals(Platform.Os.DARWIN, Platform.detectOs("macOS"));
+ assertEquals(Platform.Os.DARWIN, Platform.detectOs("Darwin"));
+ }
+
+ @Test
+ void detectsWindowsFromOsName() {
+ assertEquals(Platform.Os.WINDOWS, Platform.detectOs("Windows 10"));
+ assertEquals(Platform.Os.WINDOWS, Platform.detectOs("Windows Server 2019"));
+ }
+
+ @Test
+ void rejectsUnknownOs() {
+ assertThrows(UnsupportedOperationException.class, () -> Platform.detectOs("Solaris"));
+ }
+
+ @Test
+ void rejectsNullOs() {
+ assertThrows(UnsupportedOperationException.class, () -> Platform.detectOs(null));
+ }
+
+ @Test
+ void usesAmd64OnLinuxForX86_64Aliases() {
+ assertEquals("amd64", Platform.detectArch(Platform.Os.LINUX, "amd64"));
+ assertEquals("amd64", Platform.detectArch(Platform.Os.LINUX, "x86_64"));
+ assertEquals("amd64", Platform.detectArch(Platform.Os.LINUX, "x64"));
+ }
+
+ @Test
+ void usesX86_64OnDarwinForX86_64Aliases() {
+ assertEquals("x86_64", Platform.detectArch(Platform.Os.DARWIN, "amd64"));
+ assertEquals("x86_64", Platform.detectArch(Platform.Os.DARWIN, "x86_64"));
+ }
+
+ @Test
+ void usesAmd64OnWindowsForX86_64Aliases() {
+ assertEquals("amd64", Platform.detectArch(Platform.Os.WINDOWS, "amd64"));
+ assertEquals("amd64", Platform.detectArch(Platform.Os.WINDOWS, "x86_64"));
+ assertEquals("amd64", Platform.detectArch(Platform.Os.WINDOWS, "x64"));
+ }
+
+ @Test
+ void usesAarch64ForArm64Aliases() {
+ assertEquals("aarch64", Platform.detectArch(Platform.Os.LINUX, "aarch64"));
+ assertEquals("aarch64", Platform.detectArch(Platform.Os.LINUX, "arm64"));
+ assertEquals("aarch64", Platform.detectArch(Platform.Os.DARWIN, "aarch64"));
+ assertEquals("aarch64", Platform.detectArch(Platform.Os.DARWIN, "arm64"));
+ }
+
+ @Test
+ void rejectsUnknownArch() {
+ assertThrows(
+ UnsupportedOperationException.class,
+ () -> Platform.detectArch(Platform.Os.LINUX, "ppc64le"));
+ }
+
+ @Test
+ void libFileNameUsesPlatformConventions() {
+ assertEquals("libdatafusion_jni.so", Platform.of("Linux", "amd64").libFileName());
+ assertEquals("libdatafusion_jni.so", Platform.of("Linux", "aarch64").libFileName());
+ assertEquals("libdatafusion_jni.dylib", Platform.of("Mac OS X", "x86_64").libFileName());
+ assertEquals("libdatafusion_jni.dylib", Platform.of("Mac OS X", "aarch64").libFileName());
+ assertEquals("datafusion_jni.dll", Platform.of("Windows 11", "amd64").libFileName());
+ }
+
+ @Test
+ void resourcePathMatchesSpec() {
+ assertEquals(
+ "/org/apache/datafusion/linux/amd64/libdatafusion_jni.so",
+ Platform.of("Linux", "amd64").resourcePath());
+ assertEquals(
+ "/org/apache/datafusion/linux/aarch64/libdatafusion_jni.so",
+ Platform.of("Linux", "aarch64").resourcePath());
+ assertEquals(
+ "/org/apache/datafusion/darwin/x86_64/libdatafusion_jni.dylib",
+ Platform.of("Mac OS X", "x86_64").resourcePath());
+ assertEquals(
+ "/org/apache/datafusion/darwin/aarch64/libdatafusion_jni.dylib",
+ Platform.of("Mac OS X", "aarch64").resourcePath());
+ assertEquals(
+ "/org/apache/datafusion/windows/amd64/datafusion_jni.dll",
+ Platform.of("Windows 11", "amd64").resourcePath());
+ }
+}
diff --git a/examples/pom.xml b/examples/pom.xml
index 97a6b40..96c9ad5 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -75,7 +75,6 @@
<configuration>
<executable>java</executable>
<arguments>
- <argument>-Djava.library.path=${maven.multiModuleProjectDirectory}/native/target/debug</argument>
<argument>--add-opens=java.base/java.nio=ALL-UNNAMED</argument>
<argument>-classpath</argument>
<classpath/>
