feat(session): register object_store backends on SessionContextBuilder (#73)
diff --git a/core/src/main/java/org/apache/datafusion/ObjectStoreOptions.java b/core/src/main/java/org/apache/datafusion/ObjectStoreOptions.java
new file mode 100644
index 0000000..b4f7505
--- /dev/null
+++ b/core/src/main/java/org/apache/datafusion/ObjectStoreOptions.java
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datafusion;
+
+import org.apache.datafusion.protobuf.GcsOptions;
+import org.apache.datafusion.protobuf.HttpOptions;
+import org.apache.datafusion.protobuf.ObjectStoreRegistration;
+import org.apache.datafusion.protobuf.S3Options;
+
+/**
+ * Description of an {@code object_store::ObjectStore} backend to register on a {@link
+ * SessionContext}'s {@code RuntimeEnv}. Pass instances to {@link
+ * SessionContextBuilder#registerObjectStore(ObjectStoreOptions)}.
+ *
+ * <p>Build instances via the per-backend factories: {@link #s3()}, {@link #gcs()}, {@link
+ * #http(String)}. Each factory returns a builder whose {@code build()} produces an immutable {@code
+ * ObjectStoreOptions} you can register.
+ *
+ * <p>For the cloud backends (S3 / GCS), the standard SDK environment variables are read as the
+ * default <em>floor</em> (e.g. {@code AWS_ACCESS_KEY_ID}, {@code AWS_DEFAULT_REGION}, ECS /
+ * web-identity vars; {@code GOOGLE_APPLICATION_CREDENTIALS}). Explicit setters on the per-backend
+ * builder always override those env values, so a JVM running on a credentialed host (IAM role, ECS
+ * task, GKE workload identity) can register an empty-credentials store and let the SDK's default
+ * chain do its job.
+ *
+ * <p>The URL DataFusion uses to look up the store is either an explicit {@code url(...)} on the
+ * builder or, if unset, derived from the typed fields:
+ *
+ * <ul>
+ * <li>S3 → {@code s3://<bucket>}
+ * <li>GCS → {@code gs://<bucket>}
+ * <li>HTTP → no scheme-default; {@link #http(String)} requires the URL up front. The URL must be
+ * a host root ({@code https://example.com/} or {@code https://example.com}); paths cause
+ * lookup-vs-base-URL collisions in DataFusion's registry. Register one HTTP store per
+ * scheme+host and let SQL paths carry the rest.
+ * </ul>
+ */
+public abstract sealed class ObjectStoreOptions
+ permits ObjectStoreOptions.S3, ObjectStoreOptions.Gcs, ObjectStoreOptions.Http {
+
+ private final String url;
+
+ private ObjectStoreOptions(String url) {
+ this.url = url;
+ }
+
+ /** Begin building an {@link S3} (also covers MinIO, R2, any S3-compatible) registration. */
+ public static S3.Builder s3() {
+ return new S3.Builder();
+ }
+
+ /** Begin building a {@link Gcs} (Google Cloud Storage) registration. */
+ public static Gcs.Builder gcs() {
+ return new Gcs.Builder();
+ }
+
+ /**
+ * Begin building an {@link Http} (listing-capable HTTP/WebDAV) registration. The listing-root URL
+ * is required because no sensible default exists.
+ *
+ * @throws IllegalArgumentException if {@code listingUrl} is {@code null}.
+ */
+ public static Http.Builder http(String listingUrl) {
+ if (listingUrl == null) {
+ throw new IllegalArgumentException("http listing URL must be non-null");
+ }
+ return new Http.Builder(listingUrl);
+ }
+
+ abstract ObjectStoreRegistration toRegistration();
+
+ /** Amazon S3 (and any S3-compatible endpoint such as MinIO, Cloudflare R2, Wasabi). */
+ public static final class S3 extends ObjectStoreOptions {
+ private final String bucket;
+ private final String region;
+ private final String endpoint;
+ private final String accessKeyId;
+ private final String secretAccessKey;
+ private final String sessionToken;
+ private final Boolean allowHttp;
+ private final Boolean skipSignature;
+ private final Boolean imdsv1Fallback;
+
+ private S3(Builder b) {
+ super(b.url);
+ this.bucket = b.bucket;
+ this.region = b.region;
+ this.endpoint = b.endpoint;
+ this.accessKeyId = b.accessKeyId;
+ this.secretAccessKey = b.secretAccessKey;
+ this.sessionToken = b.sessionToken;
+ this.allowHttp = b.allowHttp;
+ this.skipSignature = b.skipSignature;
+ this.imdsv1Fallback = b.imdsv1Fallback;
+ }
+
+ @Override
+ ObjectStoreRegistration toRegistration() {
+ S3Options.Builder s3 = S3Options.newBuilder().setBucket(bucket);
+ if (region != null) s3.setRegion(region);
+ if (endpoint != null) s3.setEndpoint(endpoint);
+ if (accessKeyId != null) s3.setAccessKeyId(accessKeyId);
+ if (secretAccessKey != null) s3.setSecretAccessKey(secretAccessKey);
+ if (sessionToken != null) s3.setSessionToken(sessionToken);
+ if (allowHttp != null) s3.setAllowHttp(allowHttp);
+ if (skipSignature != null) s3.setSkipSignature(skipSignature);
+ if (imdsv1Fallback != null) s3.setImdsv1Fallback(imdsv1Fallback);
+ ObjectStoreRegistration.Builder reg = ObjectStoreRegistration.newBuilder().setS3(s3);
+ if (super.url != null) reg.setUrl(super.url);
+ return reg.build();
+ }
+
+ public static final class Builder {
+ private String url;
+ private String bucket;
+ private String region;
+ private String endpoint;
+ private String accessKeyId;
+ private String secretAccessKey;
+ private String sessionToken;
+ private Boolean allowHttp;
+ private Boolean skipSignature;
+ private Boolean imdsv1Fallback;
+
+ private Builder() {}
+
+ /**
+ * Override the registration URL. Default is {@code s3://<bucket>}; use this to register under
+ * a non-default scheme such as {@code s3a://}.
+ */
+ public Builder url(String url) {
+ this.url = url;
+ return this;
+ }
+
+ /** Bucket name. Required. */
+ public Builder bucket(String bucket) {
+ this.bucket = bucket;
+ return this;
+ }
+
+ public Builder region(String region) {
+ this.region = region;
+ return this;
+ }
+
+ /** Override the endpoint URL (e.g. {@code https://minio.internal:9000}). */
+ public Builder endpoint(String endpoint) {
+ this.endpoint = endpoint;
+ return this;
+ }
+
+ public Builder accessKeyId(String accessKeyId) {
+ this.accessKeyId = accessKeyId;
+ return this;
+ }
+
+ public Builder secretAccessKey(String secretAccessKey) {
+ this.secretAccessKey = secretAccessKey;
+ return this;
+ }
+
+ public Builder sessionToken(String sessionToken) {
+ this.sessionToken = sessionToken;
+ return this;
+ }
+
+ /** Allow plain {@code http://} endpoints (e.g. local MinIO). */
+ public Builder allowHttp(boolean allowHttp) {
+ this.allowHttp = allowHttp;
+ return this;
+ }
+
+ /** Skip request signing (anonymous / public-bucket reads). */
+ public Builder skipSignature(boolean skipSignature) {
+ this.skipSignature = skipSignature;
+ return this;
+ }
+
+ /** Allow falling back to IMDSv1 for credential discovery on EC2. */
+ public Builder imdsv1Fallback(boolean imdsv1Fallback) {
+ this.imdsv1Fallback = imdsv1Fallback;
+ return this;
+ }
+
+ public S3 build() {
+ if (bucket == null || bucket.isEmpty()) {
+ throw new IllegalArgumentException("S3 ObjectStoreOptions: bucket is required");
+ }
+ return new S3(this);
+ }
+ }
+ }
+
+ /** Google Cloud Storage. */
+ public static final class Gcs extends ObjectStoreOptions {
+ private final String bucket;
+ private final String serviceAccountKey;
+ private final String serviceAccountPath;
+ private final String applicationCredentials;
+
+ private Gcs(Builder b) {
+ super(b.url);
+ this.bucket = b.bucket;
+ this.serviceAccountKey = b.serviceAccountKey;
+ this.serviceAccountPath = b.serviceAccountPath;
+ this.applicationCredentials = b.applicationCredentials;
+ }
+
+ @Override
+ ObjectStoreRegistration toRegistration() {
+ GcsOptions.Builder gcs = GcsOptions.newBuilder().setBucket(bucket);
+ if (serviceAccountKey != null) gcs.setServiceAccountKey(serviceAccountKey);
+ if (serviceAccountPath != null) gcs.setServiceAccountPath(serviceAccountPath);
+ if (applicationCredentials != null) gcs.setApplicationCredentials(applicationCredentials);
+ ObjectStoreRegistration.Builder reg = ObjectStoreRegistration.newBuilder().setGcs(gcs);
+ if (super.url != null) reg.setUrl(super.url);
+ return reg.build();
+ }
+
+ public static final class Builder {
+ private String url;
+ private String bucket;
+ private String serviceAccountKey;
+ private String serviceAccountPath;
+ private String applicationCredentials;
+
+ private Builder() {}
+
+ /** Override the registration URL. Default is {@code gs://<bucket>}. */
+ public Builder url(String url) {
+ this.url = url;
+ return this;
+ }
+
+ /** Bucket name. Required. */
+ public Builder bucket(String bucket) {
+ this.bucket = bucket;
+ return this;
+ }
+
+ /** Inline service-account JSON. Mutually exclusive with {@link #serviceAccountPath}. */
+ public Builder serviceAccountKey(String serviceAccountKey) {
+ this.serviceAccountKey = serviceAccountKey;
+ return this;
+ }
+
+ /**
+ * Filesystem path to service-account JSON. Mutually exclusive with {@link
+ * #serviceAccountKey}.
+ */
+ public Builder serviceAccountPath(String serviceAccountPath) {
+ this.serviceAccountPath = serviceAccountPath;
+ return this;
+ }
+
+ /**
+ * Filesystem path to the application-default-credentials JSON. Maps to {@code
+ * with_application_credentials} on the upstream Rust builder, which takes a path, not inline
+ * content.
+ */
+ public Builder applicationCredentials(String applicationCredentialsPath) {
+ this.applicationCredentials = applicationCredentialsPath;
+ return this;
+ }
+
+ public Gcs build() {
+ if (bucket == null || bucket.isEmpty()) {
+ throw new IllegalArgumentException("GCS ObjectStoreOptions: bucket is required");
+ }
+ if (serviceAccountKey != null && serviceAccountPath != null) {
+ throw new IllegalArgumentException(
+ "GCS ObjectStoreOptions: serviceAccountKey and serviceAccountPath are mutually"
+ + " exclusive");
+ }
+ return new Gcs(this);
+ }
+ }
+ }
+
+ /** Listing-capable HTTP / WebDAV store. */
+ public static final class Http extends ObjectStoreOptions {
+ private final Boolean allowHttp;
+
+ private Http(Builder b) {
+ super(b.url);
+ this.allowHttp = b.allowHttp;
+ }
+
+ @Override
+ ObjectStoreRegistration toRegistration() {
+ HttpOptions.Builder http = HttpOptions.newBuilder();
+ if (allowHttp != null) http.setAllowHttp(allowHttp);
+ ObjectStoreRegistration.Builder reg = ObjectStoreRegistration.newBuilder().setHttp(http);
+ reg.setUrl(super.url);
+ return reg.build();
+ }
+
+ public static final class Builder {
+ private final String url;
+ private Boolean allowHttp;
+
+ private Builder(String url) {
+ this.url = url;
+ }
+
+ public Builder allowHttp(boolean allowHttp) {
+ this.allowHttp = allowHttp;
+ return this;
+ }
+
+ public Http build() {
+ return new Http(this);
+ }
+ }
+ }
+}
diff --git a/core/src/main/java/org/apache/datafusion/SessionContextBuilder.java b/core/src/main/java/org/apache/datafusion/SessionContextBuilder.java
index f34e24f..10a5735 100644
--- a/core/src/main/java/org/apache/datafusion/SessionContextBuilder.java
+++ b/core/src/main/java/org/apache/datafusion/SessionContextBuilder.java
@@ -19,7 +19,9 @@
package org.apache.datafusion;
+import java.util.ArrayList;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import org.apache.datafusion.protobuf.ConfigOption;
@@ -39,6 +41,7 @@
private Double memoryLimitFraction;
private String tempDirectory;
private final LinkedHashMap<String, String> options = new LinkedHashMap<>();
+ private final List<ObjectStoreOptions> objectStores = new ArrayList<>();
SessionContextBuilder() {}
@@ -169,6 +172,31 @@
}
/**
+ * Register an {@code object_store::ObjectStore} backend on the new context's {@code RuntimeEnv}.
+ * Build {@link ObjectStoreOptions} via the per-backend factories ({@link ObjectStoreOptions#s3},
+ * {@link ObjectStoreOptions#gcs}, {@link ObjectStoreOptions#http(String)}). The store is
+ * reachable inside the resulting {@link SessionContext} by URL — e.g. once an S3 store is
+ * registered for {@code my-bucket}, {@code ctx.registerParquet("orders",
+ * "s3://my-bucket/orders/")} will resolve through it.
+ *
+ * <p>Multiple registrations are applied in the order added; if two registrations resolve to the
+ * same URL, the later one wins (matching upstream {@code RuntimeEnv::register_object_store}).
+ *
+ * <p>If the underlying {@code object_store} cloud-backend Cargo feature is not built into the
+ * native library, {@link #build} surfaces a clear error rather than silently dropping the
+ * registration. The default {@code make} build enables all three backends (S3 / GCS / HTTP).
+ *
+ * @throws IllegalArgumentException if {@code options} is {@code null}.
+ */
+ public SessionContextBuilder registerObjectStore(ObjectStoreOptions options) {
+ if (options == null) {
+ throw new IllegalArgumentException("registerObjectStore options must be non-null");
+ }
+ this.objectStores.add(options);
+ return this;
+ }
+
+ /**
* Construct a {@link SessionContext} with the configured options.
*
* @throws RuntimeException if the native side fails to construct the context.
@@ -204,6 +232,9 @@
for (Map.Entry<String, String> e : options.entrySet()) {
b.addOptions(ConfigOption.newBuilder().setKey(e.getKey()).setValue(e.getValue()).build());
}
+ for (ObjectStoreOptions os : objectStores) {
+ b.addObjectStores(os.toRegistration());
+ }
return b.build().toByteArray();
}
}
diff --git a/core/src/test/java/org/apache/datafusion/ObjectStoreOptionsTest.java b/core/src/test/java/org/apache/datafusion/ObjectStoreOptionsTest.java
new file mode 100644
index 0000000..391677d
--- /dev/null
+++ b/core/src/test/java/org/apache/datafusion/ObjectStoreOptionsTest.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datafusion;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.List;
+
+import org.apache.datafusion.protobuf.ObjectStoreRegistration;
+import org.apache.datafusion.protobuf.SessionOptions;
+import org.junit.jupiter.api.Test;
+
+class ObjectStoreOptionsTest {
+
+ @Test
+ void s3FullSetterSurfaceRoundTripsThroughProto() throws Exception {
+ byte[] bytes =
+ SessionContext.builder()
+ .registerObjectStore(
+ ObjectStoreOptions.s3()
+ .bucket("orders")
+ .region("us-east-1")
+ .endpoint("https://minio.internal:9000")
+ .accessKeyId("AKIA...")
+ .secretAccessKey("...")
+ .sessionToken("...")
+ .allowHttp(true)
+ .skipSignature(false)
+ .imdsv1Fallback(true)
+ .build())
+ .toBytes();
+
+ SessionOptions parsed = SessionOptions.parseFrom(bytes);
+ List<ObjectStoreRegistration> regs = parsed.getObjectStoresList();
+ assertEquals(1, regs.size());
+ ObjectStoreRegistration r = regs.get(0);
+ assertTrue(r.hasS3());
+ assertFalse(r.hasUrl(), "default URL is derived on the Rust side, not sent on the wire");
+ assertEquals("orders", r.getS3().getBucket());
+ assertEquals("us-east-1", r.getS3().getRegion());
+ assertEquals("https://minio.internal:9000", r.getS3().getEndpoint());
+ assertEquals("AKIA...", r.getS3().getAccessKeyId());
+ assertTrue(r.getS3().getAllowHttp());
+ assertFalse(r.getS3().getSkipSignature());
+ assertTrue(r.getS3().getImdsv1Fallback());
+ }
+
+ @Test
+ void s3UnsetFieldsAreAbsentInProto() throws Exception {
+ byte[] bytes =
+ SessionContext.builder()
+ .registerObjectStore(ObjectStoreOptions.s3().bucket("b").build())
+ .toBytes();
+ SessionOptions parsed = SessionOptions.parseFrom(bytes);
+ ObjectStoreRegistration r = parsed.getObjectStores(0);
+ assertTrue(r.hasS3());
+ assertFalse(r.hasUrl());
+ assertEquals("b", r.getS3().getBucket());
+ // Every optional must travel as unset, not as zero/empty — the Rust side
+ // distinguishes "leave the upstream default in place" from "explicitly 0".
+ assertFalse(r.getS3().hasRegion());
+ assertFalse(r.getS3().hasEndpoint());
+ assertFalse(r.getS3().hasAccessKeyId());
+ assertFalse(r.getS3().hasSecretAccessKey());
+ assertFalse(r.getS3().hasSessionToken());
+ assertFalse(r.getS3().hasAllowHttp());
+ assertFalse(r.getS3().hasSkipSignature());
+ assertFalse(r.getS3().hasImdsv1Fallback());
+ }
+
+ @Test
+ void s3UrlOverrideRidesOnTheRegistrationNotTheBackendMessage() throws Exception {
+ byte[] bytes =
+ SessionContext.builder()
+ .registerObjectStore(ObjectStoreOptions.s3().bucket("b").url("s3a://b").build())
+ .toBytes();
+ SessionOptions parsed = SessionOptions.parseFrom(bytes);
+ ObjectStoreRegistration r = parsed.getObjectStores(0);
+ assertTrue(r.hasUrl());
+ assertEquals("s3a://b", r.getUrl());
+ }
+
+ @Test
+ void s3RequiresBucket() {
+ assertThrows(IllegalArgumentException.class, () -> ObjectStoreOptions.s3().build());
+ assertThrows(IllegalArgumentException.class, () -> ObjectStoreOptions.s3().bucket("").build());
+ }
+
+ @Test
+ void gcsRoundTripsThroughProto() throws Exception {
+ byte[] bytes =
+ SessionContext.builder()
+ .registerObjectStore(
+ ObjectStoreOptions.gcs()
+ .bucket("data")
+ .serviceAccountKey("{\"type\":\"service_account\"}")
+ .build())
+ .toBytes();
+ SessionOptions parsed = SessionOptions.parseFrom(bytes);
+ ObjectStoreRegistration r = parsed.getObjectStores(0);
+ assertTrue(r.hasGcs());
+ assertEquals("data", r.getGcs().getBucket());
+ assertEquals("{\"type\":\"service_account\"}", r.getGcs().getServiceAccountKey());
+ assertFalse(r.getGcs().hasServiceAccountPath());
+ }
+
+ @Test
+ void gcsRequiresBucketAndRejectsConflictingCreds() {
+ assertThrows(IllegalArgumentException.class, () -> ObjectStoreOptions.gcs().build());
+ assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ ObjectStoreOptions.gcs()
+ .bucket("b")
+ .serviceAccountKey("{}")
+ .serviceAccountPath("/tmp/sa.json")
+ .build());
+ }
+
+ @Test
+ void httpRoundTripsThroughProto() throws Exception {
+ byte[] bytes =
+ SessionContext.builder()
+ .registerObjectStore(
+ ObjectStoreOptions.http("https://example.com/").allowHttp(false).build())
+ .toBytes();
+ SessionOptions parsed = SessionOptions.parseFrom(bytes);
+ ObjectStoreRegistration r = parsed.getObjectStores(0);
+ assertTrue(r.hasHttp());
+ assertTrue(r.hasUrl());
+ assertEquals("https://example.com/", r.getUrl());
+ assertFalse(r.getHttp().getAllowHttp());
+ }
+
+ @Test
+ void httpRequiresUrl() {
+ assertThrows(IllegalArgumentException.class, () -> ObjectStoreOptions.http(null));
+ }
+
+ @Test
+ void multipleRegistrationsPreserveInsertionOrder() throws Exception {
+ byte[] bytes =
+ SessionContext.builder()
+ .registerObjectStore(ObjectStoreOptions.s3().bucket("first").build())
+ .registerObjectStore(ObjectStoreOptions.gcs().bucket("second").build())
+ .registerObjectStore(ObjectStoreOptions.s3().bucket("third").build())
+ .toBytes();
+ SessionOptions parsed = SessionOptions.parseFrom(bytes);
+ List<ObjectStoreRegistration> regs = parsed.getObjectStoresList();
+ assertEquals(3, regs.size());
+ assertEquals("first", regs.get(0).getS3().getBucket());
+ assertEquals("second", regs.get(1).getGcs().getBucket());
+ assertEquals("third", regs.get(2).getS3().getBucket());
+ }
+
+ @Test
+ void registerObjectStoreRejectsNull() {
+ SessionContextBuilder b = SessionContext.builder();
+ assertThrows(IllegalArgumentException.class, () -> b.registerObjectStore(null));
+ }
+}
diff --git a/core/src/test/java/org/apache/datafusion/SessionContextObjectStoreTest.java b/core/src/test/java/org/apache/datafusion/SessionContextObjectStoreTest.java
new file mode 100644
index 0000000..20c1591
--- /dev/null
+++ b/core/src/test/java/org/apache/datafusion/SessionContextObjectStoreTest.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.datafusion;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.ipc.ArrowReader;
+import org.junit.jupiter.api.Test;
+
+class SessionContextObjectStoreTest {
+
+ // ---- happy path ---------------------------------------------------------
+
+ @Test
+ void s3RegistrationSucceedsAndContextIsUsable() throws Exception {
+ try (BufferAllocator allocator = new RootAllocator();
+ SessionContext ctx =
+ SessionContext.builder()
+ .registerObjectStore(
+ ObjectStoreOptions.s3()
+ .bucket("my-bucket")
+ .region("us-east-1")
+ .accessKeyId("AKIAEXAMPLE")
+ .secretAccessKey("EXAMPLE")
+ .build())
+ .build();
+ DataFrame df = ctx.sql("SELECT 1");
+ ArrowReader reader = df.collect(allocator)) {
+ assertTrue(reader.loadNextBatch());
+ assertEquals(1, reader.getVectorSchemaRoot().getRowCount());
+ }
+ }
+
+ @Test
+ void gcsRegistrationSucceedsAndContextIsUsable() {
+ // GoogleCloudStorageBuilder eagerly parses the embedded PEM private key
+ // when serviceAccountKey is supplied. The `disable_oauth` flag is the
+ // upstream-test-fixture knob that lets us hand it a placeholder key
+ // without standing up a real Google credential — see object_store
+ // gcp/builder.rs FAKE_KEY. We're verifying the registration plumbing,
+ // not GCS itself.
+ try (SessionContext ctx =
+ SessionContext.builder()
+ .registerObjectStore(
+ ObjectStoreOptions.gcs()
+ .bucket("data")
+ .serviceAccountKey(
+ "{\"private_key\":\"k\",\"private_key_id\":\"id\","
+ + "\"client_email\":\"x@y\",\"disable_oauth\":true}")
+ .build())
+ .build()) {
+ // Context is alive — the registration didn't reject anything.
+ assertNotNull(ctx);
+ }
+ }
+
+ @Test
+ void httpRegistrationSucceedsAtHostRoot() {
+ try (SessionContext ctx =
+ SessionContext.builder()
+ .registerObjectStore(
+ ObjectStoreOptions.http("https://example.com/").allowHttp(false).build())
+ .build()) {
+ assertNotNull(ctx);
+ }
+ }
+
+ @Test
+ void httpRegistrationRejectsPathfulUrl() {
+ // Codex regression: if HttpStore is built with a pathful base URL while
+ // DataFusion's registry strips the path from the lookup key, every read
+ // double-prepends the path component (e.g. /data/data/file.parquet).
+ // Reject pathful registration URLs at build time with an error that
+ // points at the right shape (host root, SQL paths carry the rest).
+ SessionContextBuilder b =
+ SessionContext.builder()
+ .registerObjectStore(
+ ObjectStoreOptions.http("https://example.com/data/").allowHttp(false).build());
+ RuntimeException thrown = assertThrows(RuntimeException.class, b::build);
+ assertTrue(
+ thrown.getMessage() != null && thrown.getMessage().contains("host root"),
+ "expected error to mention 'host root', got: " + thrown.getMessage());
+ }
+
+ @Test
+ void multipleS3RegistrationsCanCoexist() {
+ try (SessionContext ctx =
+ SessionContext.builder()
+ .registerObjectStore(
+ ObjectStoreOptions.s3()
+ .bucket("a")
+ .region("us-east-1")
+ .accessKeyId("AKIAA")
+ .secretAccessKey("S")
+ .build())
+ .registerObjectStore(
+ ObjectStoreOptions.s3()
+ .bucket("b")
+ .region("eu-west-1")
+ .endpoint("https://minio.internal:9000")
+ .accessKeyId("AKIAB")
+ .secretAccessKey("S")
+ .allowHttp(true)
+ .build())
+ .build()) {
+ assertNotNull(ctx);
+ }
+ }
+
+ @Test
+ void s3UrlOverrideTakesEffect() {
+ // The override path is exercised end-to-end (registration must succeed
+ // with the alternate scheme). The registry would reject a malformed URL.
+ try (SessionContext ctx =
+ SessionContext.builder()
+ .registerObjectStore(
+ ObjectStoreOptions.s3()
+ .bucket("my-bucket")
+ .url("s3a://my-bucket")
+ .region("us-east-1")
+ .accessKeyId("k")
+ .secretAccessKey("s")
+ .build())
+ .build()) {
+ assertNotNull(ctx);
+ }
+ }
+
+ // ---- error handling -----------------------------------------------------
+
+ @Test
+ void httpWithoutUrlFailsAtJavaLevel() {
+ // Caught before any byte hits the JNI boundary.
+ assertThrows(IllegalArgumentException.class, () -> ObjectStoreOptions.http(null));
+ }
+
+ @Test
+ void s3ExplicitEndpointBuildsAlongsideCredentialEnvSeeding() {
+ // Codex regression: AmazonS3Builder::build picks `s3_endpoint` over
+ // `endpoint` (object_store 0.13 aws/builder.rs:1209). If we'd seeded the
+ // builder via from_env(), a stray AWS_ENDPOINT_URL_S3 in the JVM env
+ // would silently override Java's endpoint(...) -- breaking MinIO/R2
+ // registrations. The Rust path now walks env explicitly and skips the
+ // endpoint keys when Java sets endpoint, so the explicit MinIO endpoint
+ // wins. End-to-end JVM env-injection is impractical from a Java unit
+ // test, so this test exercises the registration path with an explicit
+ // endpoint and confirms the build succeeds; the precedence behaviour
+ // itself is enforced by the env-walk loop in native/src/object_store.rs
+ // (covered by `cargo check` and a code-level review of the loop body).
+ try (SessionContext ctx =
+ SessionContext.builder()
+ .registerObjectStore(
+ ObjectStoreOptions.s3()
+ .bucket("local-bucket")
+ .region("us-east-1")
+ .endpoint("https://minio.internal:9000")
+ .accessKeyId("k")
+ .secretAccessKey("s")
+ .allowHttp(true)
+ .build())
+ .build()) {
+ assertNotNull(ctx);
+ }
+ }
+
+ @Test
+ void s3ExplicitImdsv1FallbackFalseOverridesEnv() {
+ // Codex regression: AmazonS3Builder::with_imdsv1_fallback() only sets to
+ // `true` (no boolean arg), so a previous version of the binding silently
+ // dropped explicit `imdsv1Fallback(false)`, leaving a less-secure
+ // env-seeded `AWS_IMDSV1_FALLBACK=true` in place. The Rust path now
+ // writes through with_config(...) so explicit false also takes effect.
+ // Same caveat as the endpoint test: this exercises the registration
+ // path; the override is enforced by the with_config call in
+ // native/src/object_store.rs.
+ try (SessionContext ctx =
+ SessionContext.builder()
+ .registerObjectStore(
+ ObjectStoreOptions.s3()
+ .bucket("b")
+ .region("us-east-1")
+ .imdsv1Fallback(false)
+ .build())
+ .build()) {
+ assertNotNull(ctx);
+ }
+ }
+
+ @Test
+ void gcsExplicitServiceAccountPathOverridesEnvKey() {
+ // Codex regression: when the JVM env has GOOGLE_SERVICE_ACCOUNT_KEY and
+ // the Java caller supplies serviceAccountPath(...), the previous
+ // GoogleCloudStorageBuilder::from_env() seed left both populated, and
+ // GCS build() rejects that combo with ServiceAccountPathAndKeyProvided
+ // (object_store 0.13 gcp/builder.rs:525). The Rust path now walks env
+ // explicitly and skips GOOGLE_SERVICE_ACCOUNT* / GOOGLE_APPLICATION_CREDENTIALS
+ // when Java provides any of the credential fields. Use the upstream
+ // FAKE_KEY (disable_oauth) to dodge real PEM parsing.
+ try (SessionContext ctx =
+ SessionContext.builder()
+ .registerObjectStore(
+ ObjectStoreOptions.gcs()
+ .bucket("data")
+ .serviceAccountKey(
+ "{\"private_key\":\"k\",\"private_key_id\":\"id\","
+ + "\"client_email\":\"x@y\",\"disable_oauth\":true}")
+ .build())
+ .build()) {
+ assertNotNull(ctx);
+ }
+ }
+
+ @Test
+ void s3JavaBucketWinsOverEnvBucket() {
+ // Codex regression: AWS_BUCKET / AWS_BUCKET_NAME parse to
+ // AmazonS3ConfigKey::Bucket on the env-walking path (object_store 0.13
+ // aws/builder.rs:497). If the loop ran with the Java bucket already set,
+ // an env bucket would silently overwrite it -- the registry URL still
+ // says s3://<java-bucket> but the built store would target the env
+ // bucket, silently routing reads to the wrong place. The Rust path now
+ // skips AWS_BUCKET / AWS_BUCKET_NAME during env walking and re-applies
+ // opts.bucket *after* the loop. Exercise the registration path so any
+ // regression surfaces.
+ try (SessionContext ctx =
+ SessionContext.builder()
+ .registerObjectStore(
+ ObjectStoreOptions.s3()
+ .bucket("java-explicit-bucket")
+ .region("us-east-1")
+ .accessKeyId("k")
+ .secretAccessKey("s")
+ .build())
+ .build()) {
+ assertNotNull(ctx);
+ }
+ }
+
+ @Test
+ void gcsJavaBucketWinsOverEnvBucket() {
+ // Codex regression: GOOGLE_BUCKET / GOOGLE_BUCKET_NAME parse to
+ // GoogleConfigKey::Bucket. Same shape as the S3 case above; the env
+ // walk now skips both keys and re-applies opts.bucket after the loop.
+ try (SessionContext ctx =
+ SessionContext.builder()
+ .registerObjectStore(
+ ObjectStoreOptions.gcs()
+ .bucket("java-explicit-bucket")
+ .serviceAccountKey(
+ "{\"private_key\":\"k\",\"private_key_id\":\"id\","
+ + "\"client_email\":\"x@y\",\"disable_oauth\":true}")
+ .build())
+ .build()) {
+ assertNotNull(ctx);
+ }
+ }
+
+ @Test
+ void s3MissingCredentialsStillBuildsAndDefersToCredChain() {
+ // The AWS SDK's default credential chain is allowed to find creds at
+ // request time; building the store does not require credentials up
+ // front. This keeps parity with object_store::aws::AmazonS3Builder and
+ // avoids forcing every embedder to thread credentials through Java when
+ // the JVM is already running on a credentialed host (IAM role, etc.).
+ try (SessionContext ctx =
+ SessionContext.builder()
+ .registerObjectStore(ObjectStoreOptions.s3().bucket("b").region("us-east-1").build())
+ .build()) {
+ assertNotNull(ctx);
+ }
+ }
+}
diff --git a/native/Cargo.lock b/native/Cargo.lock
index 7171f72..d171c8a 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -83,7 +83,7 @@
"miniz_oxide",
"num-bigint",
"quad-rand",
- "rand",
+ "rand 0.9.4",
"regex-lite",
"serde",
"serde_bytes",
@@ -375,6 +375,12 @@
]
[[package]]
+name = "atomic-waker"
+version = "1.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
+
+[[package]]
name = "autocfg"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -536,6 +542,23 @@
checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
[[package]]
+name = "cfg_aliases"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+
+[[package]]
+name = "chacha20"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.3.0",
+ "rand_core 0.10.1",
+]
+
+[[package]]
name = "chrono"
version = "0.4.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -543,6 +566,7 @@
dependencies = [
"iana-time-zone",
"num-traits",
+ "serde",
"windows-link",
]
@@ -624,6 +648,16 @@
checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
[[package]]
+name = "core-foundation"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
name = "core-foundation-sys"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -793,7 +827,7 @@
"object_store",
"parking_lot",
"parquet",
- "rand",
+ "rand 0.9.4",
"regex",
"sqlparser",
"tempfile",
@@ -916,7 +950,7 @@
"liblzma",
"log",
"object_store",
- "rand",
+ "rand 0.9.4",
"tokio",
"tokio-util",
"url",
@@ -1068,7 +1102,7 @@
"log",
"object_store",
"parking_lot",
- "rand",
+ "rand 0.9.4",
"tempfile",
"url",
]
@@ -1134,7 +1168,7 @@
"md-5",
"memchr",
"num-traits",
- "rand",
+ "rand 0.9.4",
"regex",
"sha2",
"unicode-segmentation",
@@ -1255,10 +1289,12 @@
"datafusion-proto",
"futures",
"jni",
+ "object_store",
"prost",
"prost-build",
"protoc-bin-vendored",
"tokio",
+ "url",
]
[[package]]
@@ -1424,7 +1460,7 @@
"datafusion-proto-common",
"object_store",
"prost",
- "rand",
+ "rand 0.9.4",
]
[[package]]
@@ -1572,6 +1608,12 @@
]
[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
name = "foldhash"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1697,8 +1739,10 @@
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
dependencies = [
"cfg-if",
+ "js-sys",
"libc",
"wasi",
+ "wasm-bindgen",
]
[[package]]
@@ -1708,9 +1752,11 @@
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
dependencies = [
"cfg-if",
+ "js-sys",
"libc",
"r-efi 5.3.0",
"wasip2",
+ "wasm-bindgen",
]
[[package]]
@@ -1722,6 +1768,7 @@
"cfg-if",
"libc",
"r-efi 6.0.0",
+ "rand_core 0.10.1",
"wasip2",
"wasip3",
]
@@ -1733,6 +1780,25 @@
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
[[package]]
+name = "h2"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "http",
+ "indexmap",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
+[[package]]
name = "half"
version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1799,12 +1865,101 @@
]
[[package]]
+name = "http-body"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
+dependencies = [
+ "bytes",
+ "http",
+]
+
+[[package]]
+name = "http-body-util"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "http",
+ "http-body",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "httparse"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
+
+[[package]]
name = "humantime"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
[[package]]
+name = "hyper"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6299f016b246a94207e63da54dbe807655bf9e00044f73ded42c3ac5305fbcca"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "h2",
+ "http",
+ "http-body",
+ "httparse",
+ "itoa",
+ "pin-project-lite",
+ "smallvec",
+ "tokio",
+ "want",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.27.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
+dependencies = [
+ "http",
+ "hyper",
+ "hyper-util",
+ "rustls",
+ "rustls-native-certs",
+ "tokio",
+ "tokio-rustls",
+ "tower-service",
+]
+
+[[package]]
+name = "hyper-util"
+version = "0.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
+dependencies = [
+ "base64",
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "ipnet",
+ "libc",
+ "percent-encoding",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
name = "iana-time-zone"
version = "0.1.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1962,6 +2117,12 @@
checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
[[package]]
+name = "ipnet"
+version = "2.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
+
+[[package]]
name = "itertools"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2171,6 +2332,12 @@
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
+name = "lru-slab"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
+
+[[package]]
name = "lz4_flex"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2206,6 +2373,17 @@
]
[[package]]
+name = "mio"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50b7e5b27aa02a74bac8c3f23f448f8d87ff11f92d3aac1a6ed369ee08cc56c1"
+dependencies = [
+ "libc",
+ "wasi",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
name = "multimap"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2266,16 +2444,29 @@
checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49"
dependencies = [
"async-trait",
+ "base64",
"bytes",
"chrono",
+ "form_urlencoded",
"futures-channel",
"futures-core",
"futures-util",
"http",
+ "http-body-util",
"humantime",
+ "hyper",
"itertools",
+ "md-5",
"parking_lot",
"percent-encoding",
+ "quick-xml",
+ "rand 0.10.1",
+ "reqwest",
+ "ring",
+ "rustls-pki-types",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -2292,6 +2483,12 @@
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
+name = "openssl-probe"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
+
+[[package]]
name = "ordered-float"
version = "2.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2582,6 +2779,71 @@
checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40"
[[package]]
+name = "quick-xml"
+version = "0.39.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
+[[package]]
+name = "quinn"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
+dependencies = [
+ "bytes",
+ "cfg_aliases",
+ "pin-project-lite",
+ "quinn-proto",
+ "quinn-udp",
+ "rustc-hash",
+ "rustls",
+ "socket2",
+ "thiserror 2.0.18",
+ "tokio",
+ "tracing",
+ "web-time",
+]
+
+[[package]]
+name = "quinn-proto"
+version = "0.11.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
+dependencies = [
+ "bytes",
+ "getrandom 0.3.4",
+ "lru-slab",
+ "rand 0.9.4",
+ "ring",
+ "rustc-hash",
+ "rustls",
+ "rustls-pki-types",
+ "slab",
+ "thiserror 2.0.18",
+ "tinyvec",
+ "tracing",
+ "web-time",
+]
+
+[[package]]
+name = "quinn-udp"
+version = "0.5.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
+dependencies = [
+ "cfg_aliases",
+ "libc",
+ "once_cell",
+ "socket2",
+ "tracing",
+ "windows-sys 0.60.2",
+]
+
+[[package]]
name = "quote"
version = "1.0.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2609,7 +2871,18 @@
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
dependencies = [
"rand_chacha",
- "rand_core",
+ "rand_core 0.9.5",
+]
+
+[[package]]
+name = "rand"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207"
+dependencies = [
+ "chacha20",
+ "getrandom 0.4.2",
+ "rand_core 0.10.1",
]
[[package]]
@@ -2619,7 +2892,7 @@
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
dependencies = [
"ppv-lite86",
- "rand_core",
+ "rand_core 0.9.5",
]
[[package]]
@@ -2632,6 +2905,12 @@
]
[[package]]
+name = "rand_core"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
+
+[[package]]
name = "recursive"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2696,6 +2975,68 @@
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
[[package]]
+name = "reqwest"
+version = "0.12.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
+dependencies = [
+ "base64",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "http-body-util",
+ "hyper",
+ "hyper-rustls",
+ "hyper-util",
+ "js-sys",
+ "log",
+ "percent-encoding",
+ "pin-project-lite",
+ "quinn",
+ "rustls",
+ "rustls-native-certs",
+ "rustls-pki-types",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "sync_wrapper",
+ "tokio",
+ "tokio-rustls",
+ "tokio-util",
+ "tower",
+ "tower-http",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "wasm-streams",
+ "web-sys",
+]
+
+[[package]]
+name = "ring"
+version = "0.17.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom 0.2.17",
+ "libc",
+ "untrusted",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "rustc-hash"
+version = "2.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+
+[[package]]
name = "rustc_version"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2718,6 +3059,53 @@
]
[[package]]
+name = "rustls"
+version = "0.23.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
+dependencies = [
+ "once_cell",
+ "ring",
+ "rustls-pki-types",
+ "rustls-webpki",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-native-certs"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63"
+dependencies = [
+ "openssl-probe",
+ "rustls-pki-types",
+ "schannel",
+ "security-framework",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
+dependencies = [
+ "web-time",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-webpki"
+version = "0.103.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
+dependencies = [
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
+]
+
+[[package]]
name = "rustversion"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2739,12 +3127,44 @@
]
[[package]]
+name = "schannel"
+version = "0.1.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
+[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
+name = "security-framework"
+version = "3.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
+dependencies = [
+ "bitflags",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
name = "semver"
version = "1.0.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2810,6 +3230,18 @@
]
[[package]]
+name = "serde_urlencoded"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
+dependencies = [
+ "form_urlencoded",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
name = "sha2"
version = "0.10.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2863,6 +3295,16 @@
checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b"
[[package]]
+name = "socket2"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
+dependencies = [
+ "libc",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
name = "sqlparser"
version = "0.61.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2945,6 +3387,15 @@
]
[[package]]
+name = "sync_wrapper"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
+dependencies = [
+ "futures-core",
+]
+
+[[package]]
name = "synstructure"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3039,14 +3490,33 @@
]
[[package]]
+name = "tinyvec"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+
+[[package]]
name = "tokio"
version = "1.52.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
dependencies = [
"bytes",
+ "libc",
+ "mio",
"pin-project-lite",
+ "socket2",
"tokio-macros",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -3061,6 +3531,16 @@
]
[[package]]
+name = "tokio-rustls"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
+dependencies = [
+ "rustls",
+ "tokio",
+]
+
+[[package]]
name = "tokio-stream"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3086,6 +3566,51 @@
]
[[package]]
+name = "tower"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "pin-project-lite",
+ "sync_wrapper",
+ "tokio",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "tower-http"
+version = "0.6.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51"
+dependencies = [
+ "bitflags",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "pin-project-lite",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "url",
+]
+
+[[package]]
+name = "tower-layer"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
+
+[[package]]
+name = "tower-service"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
+
+[[package]]
name = "tracing"
version = "0.1.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3117,6 +3642,12 @@
]
[[package]]
+name = "try-lock"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
+
+[[package]]
name = "twox-hash"
version = "2.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3153,6 +3684,12 @@
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
+[[package]]
name = "url"
version = "2.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3199,6 +3736,15 @@
]
[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
+[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3300,6 +3846,19 @@
]
[[package]]
+name = "wasm-streams"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
+[[package]]
name = "wasmparser"
version = "0.244.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3312,6 +3871,16 @@
]
[[package]]
+name = "web-sys"
+version = "0.3.98"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4b572dff8bcf38bad0fa19729c89bb5748b2b9b1d8be70cf90df697e3a8f32aa"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3395,7 +3964,25 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
dependencies = [
- "windows-targets",
+ "windows-targets 0.42.2",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.60.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
+dependencies = [
+ "windows-targets 0.53.5",
]
[[package]]
@@ -3413,13 +4000,46 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
dependencies = [
- "windows_aarch64_gnullvm",
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_gnullvm",
- "windows_x86_64_msvc",
+ "windows_aarch64_gnullvm 0.42.2",
+ "windows_aarch64_msvc 0.42.2",
+ "windows_i686_gnu 0.42.2",
+ "windows_i686_msvc 0.42.2",
+ "windows_x86_64_gnu 0.42.2",
+ "windows_x86_64_gnullvm 0.42.2",
+ "windows_x86_64_msvc 0.42.2",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm 0.52.6",
+ "windows_aarch64_msvc 0.52.6",
+ "windows_i686_gnu 0.52.6",
+ "windows_i686_gnullvm 0.52.6",
+ "windows_i686_msvc 0.52.6",
+ "windows_x86_64_gnu 0.52.6",
+ "windows_x86_64_gnullvm 0.52.6",
+ "windows_x86_64_msvc 0.52.6",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.53.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
+dependencies = [
+ "windows-link",
+ "windows_aarch64_gnullvm 0.53.1",
+ "windows_aarch64_msvc 0.53.1",
+ "windows_i686_gnu 0.53.1",
+ "windows_i686_gnullvm 0.53.1",
+ "windows_i686_msvc 0.53.1",
+ "windows_x86_64_gnu 0.53.1",
+ "windows_x86_64_gnullvm 0.53.1",
+ "windows_x86_64_msvc 0.53.1",
]
[[package]]
@@ -3429,42 +4049,138 @@
checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
+
+[[package]]
name = "windows_aarch64_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
+
+[[package]]
name = "windows_i686_gnu"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
+
+[[package]]
name = "windows_i686_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
+
+[[package]]
name = "windows_x86_64_gnu"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
+
+[[package]]
name = "windows_x86_64_gnullvm"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
+
+[[package]]
name = "windows_x86_64_msvc"
version = "0.42.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.53.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
+
+[[package]]
name = "wit-bindgen"
version = "0.51.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3629,6 +4345,12 @@
]
[[package]]
+name = "zeroize"
+version = "1.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
+
+[[package]]
name = "zerotrie"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/native/Cargo.toml b/native/Cargo.toml
index 28e1e8f..42fab85 100644
--- a/native/Cargo.toml
+++ b/native/Cargo.toml
@@ -31,8 +31,25 @@
datafusion-proto = "53.1.0"
futures = "0.3"
jni = "0.21"
+# Pin to the same major as DataFusion 53.1 pulls in transitively (0.13.x)
+# so we share the same `dyn ObjectStore` vtable and don't double-link.
+object_store = { version = "0.13", default-features = false }
prost = "0.14"
tokio = { version = "1", features = ["rt-multi-thread"] }
+url = "2"
+
+[features]
+# Enable all three cloud backends by default so `make test` exercises them.
+# Downstream builds that strip a feature get a clear runtime error if a
+# caller tries to register an unsupported backend.
+default = [
+ "object-store-aws",
+ "object-store-gcp",
+ "object-store-http",
+]
+object-store-aws = ["object_store/aws"]
+object-store-gcp = ["object_store/gcp"]
+object-store-http = ["object_store/http"]
[build-dependencies]
prost-build = "0.14"
diff --git a/native/build.rs b/native/build.rs
index d80f0aa..e19ce5b 100644
--- a/native/build.rs
+++ b/native/build.rs
@@ -25,6 +25,7 @@
"../proto/csv_write_options.proto",
"../proto/json_read_options.proto",
"../proto/json_write_options.proto",
+ "../proto/object_store_options.proto",
"../proto/parquet_read_options.proto",
];
for p in PROTOS {
diff --git a/native/src/lib.rs b/native/src/lib.rs
index 76f69b3..cebcb22 100644
--- a/native/src/lib.rs
+++ b/native/src/lib.rs
@@ -21,6 +21,7 @@
mod errors;
mod jni_util;
mod json;
+mod object_store;
mod proto;
mod schema;
mod table_provider;
@@ -159,6 +160,12 @@
let runtime_env = runtime_builder.build()?;
let ctx = SessionContext::new_with_config_rt(config, Arc::new(runtime_env));
+ // Object-store registrations come last because they need a built
+ // RuntimeEnv to register against. A failure here drops `ctx` (and its
+ // Arc<RuntimeEnv>) on the floor and surfaces as a Java RuntimeException
+ // via try_unwrap_or_throw.
+ crate::object_store::apply_registrations(&ctx, &opts.object_stores)?;
+
Ok(Box::into_raw(Box::new(ctx)) as jlong)
})
}
diff --git a/native/src/object_store.rs b/native/src/object_store.rs
new file mode 100644
index 0000000..eefccf2
--- /dev/null
+++ b/native/src/object_store.rs
@@ -0,0 +1,299 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Apply [`ObjectStoreRegistration`] entries from the session-options proto to a
+//! [`SessionContext`]'s [`RuntimeEnv`].
+//!
+//! Each backend (`s3` / `gcs` / `http`) is gated behind its own Cargo feature;
+//! the corresponding arm of the proto `oneof` returns a clear error if the
+//! feature was not enabled at build time. The default build of this crate
+//! enables all three so Java callers don't have to think about features.
+
+use std::sync::Arc;
+
+use datafusion::prelude::SessionContext;
+use url::Url;
+
+use crate::errors::JniResult;
+use crate::proto_gen::object_store_registration::Backend;
+use crate::proto_gen::ObjectStoreRegistration;
+
+#[cfg(feature = "object-store-gcp")]
+use crate::proto_gen::GcsOptions;
+#[cfg(feature = "object-store-http")]
+use crate::proto_gen::HttpOptions;
+#[cfg(feature = "object-store-aws")]
+use crate::proto_gen::S3Options;
+
+/// Apply every registration in `regs`, in order, to the context's `RuntimeEnv`.
+/// If two registrations resolve to the same URL, the later one wins (matching
+/// upstream `RuntimeEnv::register_object_store`).
+pub(crate) fn apply_registrations(
+ ctx: &SessionContext,
+ regs: &[ObjectStoreRegistration],
+) -> JniResult<()> {
+ for reg in regs {
+ let backend = reg
+ .backend
+ .as_ref()
+ .ok_or("ObjectStoreRegistration.backend is required")?;
+ let (url, store) = build_store(reg.url.as_deref(), backend)?;
+ ctx.runtime_env().register_object_store(&url, store);
+ }
+ Ok(())
+}
+
+#[allow(unused_variables)] // `url_override` is unused on builds with no features
+fn build_store(
+ url_override: Option<&str>,
+ backend: &Backend,
+) -> JniResult<(Url, Arc<dyn object_store::ObjectStore>)> {
+ match backend {
+ #[cfg(feature = "object-store-aws")]
+ Backend::S3(opts) => build_s3(url_override, opts),
+ #[cfg(not(feature = "object-store-aws"))]
+ Backend::S3(_) => Err(
+ "object-store-aws Cargo feature is not enabled in this build of datafusion-jni".into(),
+ ),
+
+ #[cfg(feature = "object-store-gcp")]
+ Backend::Gcs(opts) => build_gcs(url_override, opts),
+ #[cfg(not(feature = "object-store-gcp"))]
+ Backend::Gcs(_) => Err(
+ "object-store-gcp Cargo feature is not enabled in this build of datafusion-jni".into(),
+ ),
+
+ #[cfg(feature = "object-store-http")]
+ Backend::Http(opts) => build_http(url_override, opts),
+ #[cfg(not(feature = "object-store-http"))]
+ Backend::Http(_) => Err(
+ "object-store-http Cargo feature is not enabled in this build of datafusion-jni".into(),
+ ),
+ }
+}
+
+#[cfg(feature = "object-store-aws")]
+fn build_s3(
+ url_override: Option<&str>,
+ opts: &S3Options,
+) -> JniResult<(Url, Arc<dyn object_store::ObjectStore>)> {
+ use object_store::aws::{AmazonS3Builder, AmazonS3ConfigKey};
+
+ if opts.bucket.is_empty() {
+ return Err("S3Options.bucket is required".into());
+ }
+
+ // Seed from the standard AWS env (AWS_ACCESS_KEY_ID, AWS_DEFAULT_REGION,
+ // AWS_WEB_IDENTITY_TOKEN_FILE, ECS task creds, etc.) before applying
+ // explicit Java overrides. Without this, callers running on EC2/ECS/EKS
+ // who omit Java-side credentials and expect the SDK default-credential
+ // chain would get an empty builder and fail at request time.
+ //
+ // Walk env explicitly (instead of AmazonS3Builder::from_env()) so we can
+ // skip endpoint keys when the Java caller set `endpoint(...)`.
+ // AmazonS3Builder::build() picks `s3_endpoint` over `endpoint`
+ // (object_store 0.13 aws/builder.rs:1209), so a stray
+ // AWS_ENDPOINT_URL_S3 in the JVM env would otherwise silently override
+ // the explicit Java endpoint -- breaking MinIO/R2/etc. registrations
+ // run out of a process with global AWS env vars.
+ let java_has_endpoint = opts.endpoint.is_some();
+ let mut b = AmazonS3Builder::default();
+ for (os_key, os_value) in std::env::vars_os() {
+ let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) else {
+ continue;
+ };
+ if !key.starts_with("AWS_") {
+ continue;
+ }
+ if java_has_endpoint
+ && (key == "AWS_ENDPOINT" || key == "AWS_ENDPOINT_URL" || key == "AWS_ENDPOINT_URL_S3")
+ {
+ continue;
+ }
+ // Skip AWS_BUCKET / AWS_BUCKET_NAME: the Java caller's bucket is the
+ // identity of this registration (it's also the host portion of the
+ // registry-key URL we return). An env-supplied bucket would overwrite
+ // it here while the URL still says `s3://<opts.bucket>`, so a query
+ // for the Java bucket would be served by the env bucket. Apply the
+ // Java bucket explicitly *after* the loop so it can't be clobbered.
+ if key == "AWS_BUCKET" || key == "AWS_BUCKET_NAME" {
+ continue;
+ }
+ if let Ok(config_key) = key.to_ascii_lowercase().parse() {
+ b = b.with_config(config_key, value);
+ }
+ }
+ b = b.with_bucket_name(&opts.bucket);
+ if let Some(ref v) = opts.region {
+ b = b.with_region(v);
+ }
+ if let Some(ref v) = opts.endpoint {
+ b = b.with_endpoint(v);
+ }
+ if let Some(ref v) = opts.access_key_id {
+ b = b.with_access_key_id(v);
+ }
+ if let Some(ref v) = opts.secret_access_key {
+ b = b.with_secret_access_key(v);
+ }
+ if let Some(ref v) = opts.session_token {
+ b = b.with_token(v);
+ }
+ if let Some(v) = opts.allow_http {
+ b = b.with_allow_http(v);
+ }
+ if let Some(v) = opts.skip_signature {
+ b = b.with_skip_signature(v);
+ }
+ if let Some(v) = opts.imdsv1_fallback {
+ // AmazonS3Builder::with_imdsv1_fallback() only sets to `true` (no
+ // boolean arg). To honor explicit `imdsv1Fallback(false)` and
+ // override an env-seeded `AWS_IMDSV1_FALLBACK=true`, write through
+ // with_config(...) which always sets the field, both directions.
+ b = b.with_config(AmazonS3ConfigKey::ImdsV1Fallback, v.to_string());
+ }
+
+ let store = b.build()?;
+ let url = parse_url(url_override, format!("s3://{}", opts.bucket))?;
+ Ok((url, Arc::new(store)))
+}
+
+#[cfg(feature = "object-store-gcp")]
+fn build_gcs(
+ url_override: Option<&str>,
+ opts: &GcsOptions,
+) -> JniResult<(Url, Arc<dyn object_store::ObjectStore>)> {
+ use object_store::gcp::GoogleCloudStorageBuilder;
+
+ if opts.bucket.is_empty() {
+ return Err("GcsOptions.bucket is required".into());
+ }
+ if opts.service_account_key.is_some() && opts.service_account_path.is_some() {
+ return Err(
+ "GcsOptions: service_account_key and service_account_path are mutually exclusive"
+ .into(),
+ );
+ }
+
+ // Seed from the standard GCS env (GOOGLE_BUCKET, GOOGLE_SERVICE_ACCOUNT,
+ // etc.) before applying explicit Java overrides.
+ //
+ // Walk env explicitly (instead of GoogleCloudStorageBuilder::from_env())
+ // so we can skip credential keys that conflict with Java-supplied
+ // credentials. GoogleCloudStorageBuilder::build() rejects the combo of
+ // service_account_path AND service_account_key with
+ // ServiceAccountPathAndKeyProvided (object_store 0.13 gcp/builder.rs:525)
+ // -- so if env has GOOGLE_SERVICE_ACCOUNT_KEY and Java sets
+ // serviceAccountPath(...), the build would fail. Skip env credential
+ // keys when Java provides any of the three credential fields.
+ let java_has_credential = opts.service_account_key.is_some()
+ || opts.service_account_path.is_some()
+ || opts.application_credentials.is_some();
+ let mut b = GoogleCloudStorageBuilder::default();
+ for (os_key, os_value) in std::env::vars_os() {
+ let (Some(key), Some(value)) = (os_key.to_str(), os_value.to_str()) else {
+ continue;
+ };
+ if !key.starts_with("GOOGLE_") {
+ continue;
+ }
+ if java_has_credential
+ && (key == "GOOGLE_SERVICE_ACCOUNT"
+ || key == "GOOGLE_SERVICE_ACCOUNT_PATH"
+ || key == "GOOGLE_SERVICE_ACCOUNT_KEY"
+ || key == "GOOGLE_APPLICATION_CREDENTIALS")
+ {
+ continue;
+ }
+ // Skip GOOGLE_BUCKET / GOOGLE_BUCKET_NAME: the Java caller's bucket is
+ // the identity of this registration (matches the registry-key URL).
+ // Apply the Java bucket explicitly *after* the loop so an env-supplied
+ // bucket can't silently route the registered URL to a different
+ // backend bucket.
+ if key == "GOOGLE_BUCKET" || key == "GOOGLE_BUCKET_NAME" {
+ continue;
+ }
+ if let Ok(config_key) = key.to_ascii_lowercase().parse() {
+ b = b.with_config(config_key, value);
+ }
+ }
+ b = b.with_bucket_name(&opts.bucket);
+ if let Some(ref v) = opts.service_account_key {
+ b = b.with_service_account_key(v);
+ }
+ if let Some(ref v) = opts.service_account_path {
+ b = b.with_service_account_path(v);
+ }
+ if let Some(ref v) = opts.application_credentials {
+ b = b.with_application_credentials(v);
+ }
+
+ let store = b.build()?;
+ let url = parse_url(url_override, format!("gs://{}", opts.bucket))?;
+ Ok((url, Arc::new(store)))
+}
+
+#[cfg(feature = "object-store-http")]
+fn build_http(
+ url_override: Option<&str>,
+ opts: &HttpOptions,
+) -> JniResult<(Url, Arc<dyn object_store::ObjectStore>)> {
+ use object_store::http::HttpBuilder;
+
+ let listing = url_override.ok_or(
+ "HttpOptions: ObjectStoreRegistration.url is required for the HTTP backend (no scheme-default)",
+ )?;
+ let listing_url =
+ Url::parse(listing).map_err(|e| format!("invalid HTTP URL {listing:?}: {e}"))?;
+
+ // DataFusion's DefaultObjectStoreRegistry::get_url_key strips paths from
+ // the registry key (it keeps only scheme + host:port), so a registration
+ // base like `https://example.com/data/` ends up under key
+ // `https://example.com`. If we hand HttpBuilder the same pathful URL,
+ // HttpStore prepends `/data/` to every object path on top of the path
+ // DataFusion already supplied -- a request for
+ // `https://example.com/data/file.parquet` becomes
+ // `https://example.com/data/data/file.parquet`. Strip the URL to
+ // scheme + authority for both the store base and the registry key so
+ // they match upstream's lookup semantics; the SQL-side URL still carries
+ // the full path, and HttpStore appends it once.
+ if !listing_url.path().is_empty() && listing_url.path() != "/" {
+ return Err(format!(
+ "HttpOptions: listing URL must be a host root (no path component); \
+ got {listing:?}. DataFusion's object-store registry keys ignore the \
+ path portion of a URL, so a pathful base would cause every object \
+ read to double-prepend the path. Register one HTTP store per \
+ scheme+host and let SQL paths carry the rest of the URL."
+ )
+ .into());
+ }
+
+ let mut b = HttpBuilder::new().with_url(listing);
+ if let Some(v) = opts.allow_http {
+ // allow_http lives on ClientOptions in object_store 0.13, not directly
+ // on HttpBuilder; route through with_client_options.
+ b = b.with_client_options(object_store::ClientOptions::new().with_allow_http(v));
+ }
+
+ let store = b.build()?;
+ Ok((listing_url, Arc::new(store)))
+}
+
+fn parse_url(url_override: Option<&str>, default: String) -> JniResult<Url> {
+ let s = url_override.unwrap_or(&default);
+ Url::parse(s).map_err(|e| format!("invalid object store URL {s:?}: {e}").into())
+}
diff --git a/proto/object_store_options.proto b/proto/object_store_options.proto
new file mode 100644
index 0000000..5aa975a
--- /dev/null
+++ b/proto/object_store_options.proto
@@ -0,0 +1,75 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+syntax = "proto3";
+
+package datafusion_java;
+
+option java_package = "org.apache.datafusion.protobuf";
+option java_multiple_files = true;
+
+// One object_store registration to apply to a SessionContext's RuntimeEnv.
+//
+// Discriminated by the `backend` oneof so the Rust side cannot misinterpret
+// an inconsistent options mix as a different backend. The `url` field is
+// optional: if unset, the JNI layer derives the canonical scheme + host
+// from the typed backend options below (s3://<bucket>, gs://<bucket>).
+// For the HTTP backend `url` is the listing root and is required.
+//
+// Each backend message uses `optional` for nullable fields so unset-ness
+// survives the boundary -- credentials, region, endpoint, etc., that are
+// not supplied here let the Rust side fall back to the upstream
+// object_store builder's default (which for S3 means the AWS SDK's
+// default credential chain).
+message ObjectStoreRegistration {
+ optional string url = 1;
+ oneof backend {
+ HttpOptions http = 2;
+ S3Options s3 = 3;
+ GcsOptions gcs = 4;
+ }
+}
+
+message S3Options {
+ string bucket = 1;
+ optional string region = 2;
+ optional string endpoint = 3;
+ optional string access_key_id = 4;
+ optional string secret_access_key = 5;
+ optional string session_token = 6;
+ optional bool allow_http = 7;
+ optional bool skip_signature = 8;
+ optional bool imdsv1_fallback = 9;
+}
+
+message GcsOptions {
+ string bucket = 1;
+ // Inline service-account JSON. Mutually exclusive with `service_account_path`.
+ optional string service_account_key = 2;
+ // Path on the local filesystem to the service-account JSON.
+ optional string service_account_path = 3;
+ // Path on the local filesystem to the application-default-credentials
+ // JSON. Maps to `with_application_credentials` on the upstream builder
+ // (which takes a path, not inline content).
+ optional string application_credentials = 4;
+}
+
+message HttpOptions {
+ // The listing-root URL is required (no scheme-default exists for HTTP).
+ // It must be carried via the outer `ObjectStoreRegistration.url` field.
+ optional bool allow_http = 1;
+}
diff --git a/proto/session_options.proto b/proto/session_options.proto
index 2c9e629..330fe9b 100644
--- a/proto/session_options.proto
+++ b/proto/session_options.proto
@@ -19,6 +19,8 @@
package datafusion_java;
+import "object_store_options.proto";
+
option java_package = "org.apache.datafusion.protobuf";
option java_multiple_files = true;
@@ -50,6 +52,11 @@
optional MemoryLimit memory_limit = 5;
optional string temp_directory = 6;
repeated ConfigOption options = 7;
+ // Object store registrations to apply to the new context's RuntimeEnv.
+ // Applied in order; if two registrations resolve to the same URL, the
+ // later one wins (matching upstream's `RuntimeEnv::register_object_store`
+ // semantics).
+ repeated ObjectStoreRegistration object_stores = 8;
}
message ConfigOption {
