[test](catalog)add obs and gcs test (#55925)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
index 9183f2e..b3d99a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
@@ -98,10 +98,10 @@
return new ParamRules()
.require(glueAccessKey,
- "glue.access_key or aws.glue.access-key or client.credentials-provider.glue.access_key")
+ "glue.access_key is required")
.require(glueSecretKey,
- "glue.secret_key or aws.glue.secret-key or client.credentials-provider.glue.secret_key")
- .require(glueEndpoint, "glue.endpoint or aws.endpoint or aws.glue.endpoint is required");
+ "glue.secret_key is required")
+ .require(glueEndpoint, "glue.endpoint is required");
}
private void checkAndInit() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreProperties.java
deleted file mode 100644
index 8dcee8e..0000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreProperties.java
+++ /dev/null
@@ -1,76 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.property.metastore;
-
-import org.apache.doris.datasource.property.ConnectorProperty;
-
-import com.google.common.collect.Maps;
-import lombok.Getter;
-
-import java.util.Map;
-
-public class AWSGlueMetaStoreProperties extends MetastoreProperties {
-
- private AWSGlueMetaStoreBaseProperties baseProperties;
-
- @ConnectorProperty(names = {"glue.catalog_id"},
- description = "The catalog id of the AWS Glue.",
- supported = false)
- private String glueCatalogId = "";
-
- public AWSGlueMetaStoreProperties(Map<String, String> origProps) {
- super(Type.HMS, origProps);
- }
-
-
- @Override
- public void initNormalizeAndCheckProps() {
- super.initNormalizeAndCheckProps();
- baseProperties = AWSGlueMetaStoreBaseProperties.of(origProps);
- }
-
-
- public AWSCatalogMetastoreClientCredentials getAWSCatalogMetastoreClientCredentials() {
- return new AWSCatalogMetastoreClientCredentials(baseProperties.glueEndpoint, baseProperties.glueAccessKey,
- baseProperties.glueSecretKey);
- }
-
- public void toIcebergGlueCatalogProperties(Map<String, String> catalogProps) {
- // See AwsClientProperties.java for property keys
- catalogProps.put("client.credentials-provider",
- "com.amazonaws.glue.catalog.credentials.ConfigurationAWSCredentialsProvider2x");
- catalogProps.put("client.credentials-provider.glue.access_key", baseProperties.glueAccessKey);
- catalogProps.put("client.credentials-provider.glue.secret_key", baseProperties.glueSecretKey);
- catalogProps.put("client.region", baseProperties.glueRegion);
- }
-
- @Getter
- public static class AWSCatalogMetastoreClientCredentials {
- private Map<String, String> credentials = Maps.newHashMap();
-
- // Used for AWSCatalogMetastoreClient
- // See AWSGlueClientFactory in AWSCatalogMetastoreClient.java
- public AWSCatalogMetastoreClientCredentials(String endpoint, String ak, String sk) {
- credentials.put("aws.catalog.credentials.provider.factory.class",
- "com.amazonaws.glue.catalog.credentials.ConfigurationAWSCredentialsProviderFactory");
- credentials.put("aws.glue.access-key", ak);
- credentials.put("aws.glue.secret-key", sk);
- credentials.put("aws.glue.endpoint", endpoint);
- }
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/DataProcProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/DataProcProperties.java
deleted file mode 100644
index 2007fd5..0000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/DataProcProperties.java
+++ /dev/null
@@ -1,37 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.property.metastore;
-
-import org.apache.doris.datasource.property.ConnectorProperty;
-
-import java.util.Map;
-
-public class DataProcProperties extends MetastoreProperties {
- @ConnectorProperty(names = {"hive.metastore.uri"},
- description = "The uri of the hive metastore in DataProc")
- private String hiveMetastoreUri = "";
-
- public DataProcProperties(Map<String, String> origProps) {
- super(Type.DATAPROC, origProps);
- }
-
- @Override
- protected void checkRequiredProperties() {
-
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/FileMetastoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/FileMetastoreProperties.java
deleted file mode 100644
index e4c4800..0000000
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/FileMetastoreProperties.java
+++ /dev/null
@@ -1,46 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.property.metastore;
-
-import org.apache.doris.common.UserException;
-import org.apache.doris.datasource.property.storage.StorageProperties;
-
-import java.util.List;
-import java.util.Map;
-
-public class FileMetastoreProperties extends MetastoreProperties {
-
- private List<StorageProperties> storageProperties;
-
- public FileMetastoreProperties(Map<String, String> origProps) {
- super(Type.FILE_SYSTEM, origProps);
- try {
- storageProperties = StorageProperties.createAll(origProps);
- } catch (UserException e) {
- throw new RuntimeException(e);
- }
- }
-
- public List<StorageProperties> getStorageProperties() {
- return storageProperties;
- }
-
- @Override
- protected void checkRequiredProperties() {
- }
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStoreProperties.java
index 7dd97b0..9323d78 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStoreProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStoreProperties.java
@@ -24,7 +24,6 @@
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.catalog.Catalog;
import org.apache.iceberg.hadoop.HadoopCatalog;
@@ -76,8 +75,10 @@
if (storagePropertiesList.size() == 1 && storagePropertiesList.get(0) instanceof HdfsProperties) {
HdfsProperties hdfsProps = (HdfsProperties) storagePropertiesList.get(0);
if (hdfsProps.isKerberos()) {
- props.put(CatalogProperties.FILE_IO_IMPL,
- "org.apache.doris.datasource.iceberg.fileio.DelegateFileIO");
+ // NOTE: Custom FileIO implementation (KerberizedHadoopFileIO) is commented out by default.
+ // Using FileIO for Kerberos authentication may cause serialization issues when accessing
+ // Iceberg system tables (e.g., history, snapshots, manifests).
+ //props.put(CatalogProperties.FILE_IO_IMPL,"org.apache.doris.datasource.iceberg.fileio.DelegateFileIO");
this.executionAuthenticator = new HadoopExecutionAuthenticator(hdfsProps.getHadoopAuthenticator());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
index 33a0c77..7952f8d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
@@ -28,7 +28,6 @@
import org.apache.iceberg.aws.s3.S3FileIOProperties;
import org.apache.iceberg.catalog.Catalog;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -74,10 +73,6 @@
return catalog;
}
- private Map<String, String> prepareBaseCatalogProps() {
- return new HashMap<>(origProps);
- }
-
private void appendS3Props(Map<String, String> props) {
props.put(S3FileIOProperties.ACCESS_KEY_ID, s3Properties.getAccessKey());
props.put(S3FileIOProperties.SECRET_ACCESS_KEY, s3Properties.getSecretKey());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
index db9ec67..fe04cd1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
@@ -20,12 +20,10 @@
import org.apache.doris.common.security.authentication.HadoopExecutionAuthenticator;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
import org.apache.doris.datasource.property.ConnectorProperty;
-import org.apache.doris.datasource.property.storage.HdfsProperties;
import org.apache.doris.datasource.property.storage.StorageProperties;
-import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.catalog.Catalog;
import org.apache.iceberg.hive.HiveCatalog;
@@ -74,20 +72,24 @@
for (Map.Entry<String, String> entry : sp.getHadoopStorageConfig()) {
catalogProps.put(entry.getKey(), entry.getValue());
}
- if (sp instanceof HdfsProperties) {
+ // NOTE: Custom FileIO implementation (KerberizedHadoopFileIO) is commented out by default.
+ // Using FileIO for Kerberos authentication may cause serialization issues when accessing
+ // Iceberg system tables (e.g., history, snapshots, manifests).
+ /*if (sp instanceof HdfsProperties) {
HdfsProperties hdfsProps = (HdfsProperties) sp;
if (hdfsProps.isKerberos()) {
catalogProps.put(CatalogProperties.FILE_IO_IMPL,
"org.apache.doris.datasource.iceberg.fileio.DelegateFileIO");
}
- }
+ }*/
});
+ buildCatalogProperties(catalogProps);
try {
this.executionAuthenticator.execute(() -> hiveCatalog.initialize(catalogName, catalogProps));
return hiveCatalog;
} catch (Exception e) {
throw new RuntimeException("Failed to initialize HiveCatalog for Iceberg. "
- + "CatalogName=" + catalogName + ", warehouse=" + warehouse, e);
+ + "CatalogName=" + catalogName + ", msg :" + ExceptionUtils.getRootCauseMessage(e), e);
}
}
@@ -108,17 +110,10 @@
/**
* Constructs HiveCatalog's property map.
*/
- private Map<String, String> buildCatalogProperties() {
+ private void buildCatalogProperties(Map<String, String> catalogProps) {
Map<String, String> props = new HashMap<>();
- props.put(HiveCatalog.LIST_ALL_TABLES, String.valueOf(listAllTables));
-
- if (StringUtils.isNotBlank(warehouse)) {
- props.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
- }
-
+ catalogProps.put(HiveCatalog.LIST_ALL_TABLES, String.valueOf(listAllTables));
props.put("uri", hmsBaseProperties.getHiveMetastoreUri());
- props.putAll(origProps); // Keep at end to allow override, but risky if overlaps exist
- return props;
}
private void checkInitialized() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
index 3694ab7..0b10162 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
@@ -22,6 +22,7 @@
import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.iceberg.catalog.Catalog;
import software.amazon.s3tables.iceberg.S3TablesCatalog;
@@ -60,7 +61,8 @@
return catalog;
} catch (Exception e) {
throw new RuntimeException("Failed to initialize S3TablesCatalog for Iceberg. "
- + "CatalogName=" + catalogName + ", region=" + s3Properties.getRegion(), e);
+ + "CatalogName=" + catalogName + ", region=" + s3Properties.getRegion()
+ + ", msg: " + ExceptionUtils.getRootCauseMessage(e), e);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java
index a7afa94..2f67d10 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java
@@ -22,6 +22,7 @@
import org.apache.doris.datasource.property.ConnectorProperty;
import org.apache.doris.datasource.property.storage.StorageProperties;
+import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
@@ -55,7 +56,7 @@
@Override
public String getPaimonCatalogType() {
- return PaimonExternalCatalog.PAIMON_DLF;
+ return PaimonExternalCatalog.PAIMON_HMS;
}
protected PaimonHMSMetaStoreProperties(Map<String, String> props) {
@@ -95,7 +96,8 @@
try {
return executionAuthenticator.execute(() -> CatalogFactory.createCatalog(catalogContext));
} catch (Exception e) {
- throw new RuntimeException("Failed to create Paimon catalog with HMS metastore", e);
+ throw new RuntimeException("Failed to create Paimon catalog with HMS metastore, msg: "
+ + ExceptionUtils.getRootCause(e), e);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
index 9baaba4..af4c332 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
@@ -100,7 +100,6 @@
* Flag indicating whether to use path-style URLs for the object storage system.
* This value is optional and can be configured by the user.
*/
- @Setter
@Getter
@ConnectorProperty(names = {"cos.use_path_style", "use_path_style", "s3.path-style-access"}, required = false,
description = "Whether to use path style URL for the storage.")
@@ -108,7 +107,6 @@
@ConnectorProperty(names = {"cos.force_parsing_by_standard_uri", "force_parsing_by_standard_uri"}, required = false,
description = "Whether to use path style URL for the storage.")
- @Setter
@Getter
protected String forceParsingByStandardUrl = "false";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java
index 719de30..dffd18d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/GCSProperties.java
@@ -124,7 +124,6 @@
* Flag indicating whether to use path-style URLs for the object storage system.
* This value is optional and can be configured by the user.
*/
- @Setter
@Getter
@ConnectorProperty(names = {"gs.use_path_style", "use_path_style", "s3.path-style-access"}, required = false,
description = "Whether to use path style URL for the storage.")
@@ -132,7 +131,6 @@
@ConnectorProperty(names = {"gs.force_parsing_by_standard_uri", "force_parsing_by_standard_uri"}, required = false,
description = "Whether to use path style URL for the storage.")
- @Setter
@Getter
protected String forceParsingByStandardUrl = "false";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index 44edef7..6db19bd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -290,6 +290,7 @@
}).build();
}
// For anonymous access (no credentials required)
+ //fixme: should return AwsCredentialsProviderChain
if (StringUtils.isBlank(accessKey) && StringUtils.isBlank(secretKey)) {
return AnonymousCredentialsProvider.create();
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBasePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBasePropertiesTest.java
new file mode 100644
index 0000000..945c862
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBasePropertiesTest.java
@@ -0,0 +1,111 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class AWSGlueMetaStoreBasePropertiesTest {
+ private Map<String, String> baseValidProps() {
+ Map<String, String> props = new HashMap<>();
+ props.put("glue.access_key", "ak");
+ props.put("glue.secret_key", "sk");
+ props.put("glue.endpoint", "glue.us-east-1.amazonaws.com");
+ return props;
+ }
+
+ @Test
+ void testValidPropertiesWithRegionFromEndpoint() {
+ Map<String, String> props = baseValidProps();
+ // no region set -> should be extracted from endpoint
+ AWSGlueMetaStoreBaseProperties glueProps = AWSGlueMetaStoreBaseProperties.of(props);
+ Assertions.assertEquals("ak", glueProps.glueAccessKey);
+ Assertions.assertEquals("sk", glueProps.glueSecretKey);
+ Assertions.assertEquals("us-east-1", glueProps.glueRegion);
+ }
+
+ @Test
+ void testValidPropertiesWithExplicitRegion() {
+ Map<String, String> props = baseValidProps();
+ props.put("glue.region", "ap-southeast-1");
+ AWSGlueMetaStoreBaseProperties glueProps = AWSGlueMetaStoreBaseProperties.of(props);
+ Assertions.assertEquals("ap-southeast-1", glueProps.glueRegion);
+ }
+
+ @Test
+ void testMissingAccessKeyThrows() {
+ Map<String, String> props = baseValidProps();
+ props.remove("glue.access_key");
+ IllegalArgumentException ex = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> AWSGlueMetaStoreBaseProperties.of(props)
+ );
+ Assertions.assertTrue(ex.getMessage().contains("glue.access_key"));
+ }
+
+ @Test
+ void testMissingSecretKeyThrows() {
+ Map<String, String> props = baseValidProps();
+ props.remove("glue.secret_key");
+
+ IllegalArgumentException ex = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> AWSGlueMetaStoreBaseProperties.of(props)
+ );
+ Assertions.assertTrue(ex.getMessage().contains("glue.secret_key is required"));
+ }
+
+ @Test
+ void testMissingEndpointThrows() {
+ Map<String, String> props = baseValidProps();
+ props.remove("glue.endpoint");
+
+ IllegalArgumentException ex = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> AWSGlueMetaStoreBaseProperties.of(props)
+ );
+ Assertions.assertTrue(ex.getMessage().contains("glue.endpoint is required"));
+ }
+
+ @Test
+ void testInvalidEndpointThrows() {
+ Map<String, String> props = baseValidProps();
+ props.put("glue.endpoint", "http://invalid-endpoint.com");
+
+ IllegalArgumentException ex = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> AWSGlueMetaStoreBaseProperties.of(props)
+ );
+ Assertions.assertTrue(ex.getMessage().contains("Invalid AWS Glue endpoint"));
+ }
+
+ @Test
+ void testExtractRegionFailsWhenPatternMatchesButNoRegion() {
+ Map<String, String> props = baseValidProps();
+ props.put("glue.endpoint", "glue..amazonaws.com"); // malformed
+
+ IllegalArgumentException ex = Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> AWSGlueMetaStoreBaseProperties.of(props)
+ );
+ Assertions.assertTrue(ex.getMessage().contains("Invalid AWS Glue endpoint"));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStorePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStorePropertiesTest.java
deleted file mode 100644
index 28db855..0000000
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStorePropertiesTest.java
+++ /dev/null
@@ -1,174 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.property.metastore;
-
-import org.apache.doris.common.UserException;
-
-import com.google.common.collect.Maps;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.Disabled;
-import org.junit.jupiter.api.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-@Disabled("wait metastore integration")
-public class AWSGlueMetaStorePropertiesTest {
- private static Map<String, String> baseProps = new HashMap<>();
-
- @BeforeAll
- public static void setUp() {
- baseProps.put("iceberg.catalog.type", "glue");
- baseProps.put("type", "iceberg");
- }
-
- @Test
- public void testBasicProperties() throws UserException {
- Map<String, String> props = baseProps;
- props.put("glue.access_key", "test_access_key");
- props.put("glue.secret_key", "test_secret_key");
-
- props.put("glue.endpoint", "https://glue.ap-northeast-1.amazonaws.com");
-
- AWSGlueMetaStoreProperties glueProperties = (AWSGlueMetaStoreProperties) MetastoreProperties.create(props);
-
- Map<String, String> catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("com.amazonaws.glue.catalog.credentials.ConfigurationAWSCredentialsProvider2x", catalogProps
- .get("client.credentials-provider"));
- Assertions.assertEquals(props.get("glue.access_key"), catalogProps
- .get("client.credentials-provider.glue.access_key"));
- Assertions.assertEquals(props.get("glue.secret_key"), catalogProps
- .get("client.credentials-provider.glue.secret_key"));
- Assertions.assertEquals("ap-northeast-1", catalogProps
- .get("client.region"));
- AWSGlueMetaStoreProperties.AWSCatalogMetastoreClientCredentials awsCatalogMetastoreClientCredentials = glueProperties.getAWSCatalogMetastoreClientCredentials();
- Map<String, String> credentials = awsCatalogMetastoreClientCredentials.getCredentials();
- Assertions.assertEquals("test_access_key", credentials.get("aws.glue.access-key"));
- Assertions.assertEquals("test_secret_key", credentials.get("aws.glue.secret-key"));
- Assertions.assertEquals("https://glue.ap-northeast-1.amazonaws.com", credentials.get("aws.glue.endpoint"));
- //Test glue.endpoint
- props = new HashMap<>();
- props.put("type", "hms");
- props.put("hive.metastore.type", "glue");
- props.put("glue.endpoint", "https://glue.ap-northeast-1.amazonaws.com");
- props.put("aws.glue.secret-key", "test_secret_key");
- props.put("aws.glue.access-key", "test_access_key");
- glueProperties = (AWSGlueMetaStoreProperties) MetastoreProperties.create(props);
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("ap-northeast-1", catalogProps.get("client.region"));
- Assertions.assertEquals("test_access_key", catalogProps.get("client.credentials-provider.glue.access_key"));
- Assertions.assertEquals("test_secret_key", catalogProps.get("client.credentials-provider.glue.secret_key"));
- props = new HashMap<>();
- props.put("type", "hms");
- props.put("hive.metastore.type", "glue");
- props.put("glue.endpoint", "https://glue.ap-northeast-1.amazonaws.com");
- props.put("aws.glue.secret-key", "test_secret_key");
- props.put("glue.access_key", "test_glue_access_key");
- glueProperties = (AWSGlueMetaStoreProperties) MetastoreProperties.create(props);
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("ap-northeast-1", catalogProps.get("client.region"));
- Assertions.assertEquals("test_secret_key", catalogProps.get("client.credentials-provider.glue.secret_key"));
- Assertions.assertEquals("test_glue_access_key", catalogProps.get("client.credentials-provider.glue.access_key"));
- }
-
- @Test
- public void testMissingRequiredProperties() {
- Map<String, String> props = Maps.newHashMap();
- Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
- MetastoreProperties.create(props);
- });
- props.put("glue.access_key", "test_access_key");
- Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
- MetastoreProperties.create(props);
- });
- props.put("glue.secret_key", "test_secret_key");
- Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
- MetastoreProperties.create(props);
- });
- }
-
- @Test
- public void testEmptyRequiredProperty() {
- Map<String, String> props = Maps.newHashMap();
- props.put("glue.access_key", " ");
- props.put("glue.secret_key", "test_secret_key");
- props.put("glue.endpoint", "https://glue.ap-northeast-1.amazonaws.com");
-
- Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
- MetastoreProperties.create(props);
- }, "AWS Glue properties(glue.access_key, glue.secret_key, glue.endpoint) are not set correctly.");
- props.put("glue.access_key", "");
- Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
- MetastoreProperties.create(props);
- }, "AWS Glue properties(glue.access_key, glue.secret_key, glue.endpoint) are not set correctly.");
- props.put("glue.access_key", "test_access_key");
- props.put("glue.secret_key", " ");
- Assertions.assertThrowsExactly(IllegalArgumentException.class, () -> {
- MetastoreProperties.create(props);
- }, "AWS Glue properties(glue.access_key, glue.secret_key, glue.endpoint) are not set correctly.");
- }
-
- @Test
- public void testEndpointParams() throws UserException {
- Map<String, String> props = Maps.newHashMap();
- props.put("glue.access_key", "a");
- props.put("glue.secret_key", "test_secret_key");
- props.put("glue.endpoint", "https://glue.us-west-2.amazonaws.com");
- AWSGlueMetaStoreProperties glueProperties = (AWSGlueMetaStoreProperties) MetastoreProperties.create(props);
- Map<String, String> catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("us-west-2", catalogProps.get("client.region"));
- props.put("glue.endpoint", "https://glue-fips.us-west-2.api.aws");
- glueProperties = (AWSGlueMetaStoreProperties) MetastoreProperties.create(props);
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("us-west-2", catalogProps.get("client.region"));
- props.put("glue.endpoint", "https://glue-fips.us-west-2.amazonaws.com");
- glueProperties = (AWSGlueMetaStoreProperties) MetastoreProperties.create(props);
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("us-west-2", catalogProps.get("client.region"));
- props.put("glue.endpoint", "https://glue.us-west-2.api.aws");
- glueProperties = (AWSGlueMetaStoreProperties) MetastoreProperties.create(props);
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("us-west-2", catalogProps.get("client.region"));
- props.put("glue.endpoint", "https://glue.us-west-2.amazonaws.com");
- glueProperties = (AWSGlueMetaStoreProperties) MetastoreProperties.create(props);
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("us-west-2", catalogProps.get("client.region"));
-
- props.put("glue.endpoint", "glue.us-west-2.amazonaws.com");
- glueProperties = (AWSGlueMetaStoreProperties) MetastoreProperties.create(props);
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("us-west-2", catalogProps.get("client.region"));
- catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
- Assertions.assertEquals("us-west-2", catalogProps.get("client.region"));
- props.put("glue.endpoint", "https://glue.us-west-2.amaaws.com");
- Assertions.assertThrows(IllegalArgumentException.class, () -> MetastoreProperties.create(props), "AWS Glue properties (glue.endpoint) are not set correctly: https://glue.us-west-2.amaaws.com");
-
-
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractIcebergPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractIcebergPropertiesTest.java
new file mode 100644
index 0000000..275180c
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractIcebergPropertiesTest.java
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.catalog.Catalog;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class AbstractIcebergPropertiesTest {
+
+ private static class TestIcebergProperties extends AbstractIcebergProperties {
+ private final Catalog catalogToReturn;
+ private Map<String, String> capturedCatalogProps;
+
+ TestIcebergProperties(Map<String, String> props, Catalog catalogToReturn) {
+ super(props);
+ this.catalogToReturn = catalogToReturn;
+ }
+
+ @Override
+ public String getIcebergCatalogType() {
+ return "test";
+ }
+
+ @Override
+ protected Catalog initCatalog(String catalogName,
+ Map<String, String> catalogProps,
+ List<StorageProperties> storagePropertiesList) {
+ // Capture the catalogProps for verification
+ this.capturedCatalogProps = new HashMap<>(catalogProps);
+ return catalogToReturn;
+ }
+
+ Map<String, String> getCapturedCatalogProps() {
+ return capturedCatalogProps;
+ }
+ }
+
+ @Test
+ void testInitializeCatalogWithWarehouse() {
+ Catalog mockCatalog = Mockito.mock(Catalog.class);
+ Mockito.when(mockCatalog.name()).thenReturn("mocked-catalog");
+ Map<String, String> props = new HashMap<>();
+ props.put("k1", "v1");
+ TestIcebergProperties properties = new TestIcebergProperties(props, mockCatalog);
+ properties.warehouse = "s3://bucket/warehouse";
+ Catalog result = properties.initializeCatalog("testCatalog", Collections.emptyList());
+ Assertions.assertNotNull(result);
+ Assertions.assertEquals("mocked-catalog", result.name());
+ // Verify that warehouse is included in catalogProps
+ Assertions.assertTrue(properties.getCapturedCatalogProps()
+ .containsKey(CatalogProperties.WAREHOUSE_LOCATION));
+ Assertions.assertEquals("s3://bucket/warehouse",
+ properties.getCapturedCatalogProps().get(CatalogProperties.WAREHOUSE_LOCATION));
+ Assertions.assertNotNull(properties.getExecutionAuthenticator());
+ }
+
+ @Test
+ void testInitializeCatalogWithoutWarehouse() {
+ Catalog mockCatalog = Mockito.mock(Catalog.class);
+ Mockito.when(mockCatalog.name()).thenReturn("no-warehouse");
+ TestIcebergProperties properties = new TestIcebergProperties(new HashMap<>(), mockCatalog);
+ properties.warehouse = null;
+ Catalog result = properties.initializeCatalog("testCatalog", Collections.emptyList());
+ Assertions.assertNotNull(result);
+ Assertions.assertEquals("no-warehouse", result.name());
+ // Verify that warehouse key is not present
+ Assertions.assertFalse(properties.getCapturedCatalogProps()
+ .containsKey(CatalogProperties.WAREHOUSE_LOCATION));
+ }
+
+ @Test
+ void testInitializeCatalogThrowsWhenNull() {
+ AbstractIcebergProperties properties = new AbstractIcebergProperties(new HashMap<>()) {
+ @Override
+ public String getIcebergCatalogType() {
+ return "test";
+ }
+
+ @Override
+ protected Catalog initCatalog(String catalogName,
+ Map<String, String> catalogProps,
+ List<StorageProperties> storagePropertiesList) {
+ return null; // Simulate a failure case
+ }
+ };
+
+ IllegalStateException ex = Assertions.assertThrows(
+ IllegalStateException.class,
+ () -> properties.initializeCatalog("testCatalog", Collections.emptyList())
+ );
+ Assertions.assertEquals("Catalog must not be null after initialization.", ex.getMessage());
+ }
+
+ @Test
+ void testExecutionAuthenticatorNotNull() {
+ Catalog mockCatalog = Mockito.mock(Catalog.class);
+ TestIcebergProperties properties = new TestIcebergProperties(new HashMap<>(), mockCatalog);
+ Assertions.assertNotNull(properties.executionAuthenticator);
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/GlueCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/GlueCatalogTest.java
index 66f54d7..5352907 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/GlueCatalogTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/GlueCatalogTest.java
@@ -35,7 +35,7 @@
public class GlueCatalogTest {
private GlueCatalog glueCatalog;
- private AWSGlueMetaStoreProperties glueProperties;
+ private AWSGlueMetaStoreBaseProperties glueProperties;
private static final Namespace queryNameSpace = Namespace.of("test"); // Replace with your namespace
private static final String AWS_ACCESS_KEY_ID = "YOUR_ACCESS_KEY_ID"; // Replace with actual access key
private static final String AWS_SECRET_ACCESS_KEY = "YOUR_SECRET_ACCESS_KEY"; // Replace with actual secret key
@@ -57,11 +57,12 @@
// Initialize AWSGlueProperties
- glueProperties = (AWSGlueMetaStoreProperties) AWSGlueMetaStoreProperties.create(props);
+ glueProperties = (AWSGlueMetaStoreBaseProperties) AWSGlueMetaStoreBaseProperties.of(props);
// Convert to catalog properties
Map<String, String> catalogProps = new HashMap<>();
- glueProperties.toIcebergGlueCatalogProperties(catalogProps);
+ catalogProps.put("catalog-name", "ck");
+
// Initialize Glue Catalog
glueCatalog.initialize("ck", catalogProps);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSAliyunDLFMetaStorePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSAliyunDLFMetaStorePropertiesTest.java
new file mode 100644
index 0000000..b93886e
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSAliyunDLFMetaStorePropertiesTest.java
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.common.UserException;
+
+import com.aliyun.datalake.metastore.common.DataLakeConfig;
+import com.google.common.collect.Maps;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+public class HMSAliyunDLFMetaStorePropertiesTest {
+
+ @Test
+ public void testCreate() throws UserException {
+ Map<String, String> props = Maps.newHashMap();
+ props.put("type", "hms");
+ props.put("hive.metastore.type", "dlf");
+ props.put("dlf.endpoint", "dlf.cn-shanghai.aliyuncs.com");
+ props.put("dlf.region", "cn-shanghai");
+ props.put("dlf.access_key", "xxx");
+ props.put("dlf.secret_key", "xxx");
+ props.put("dlf.catalog_id", "5789");
+ HMSAliyunDLFMetaStoreProperties hmsAliyunDLFMetaStoreProperties =
+ (HMSAliyunDLFMetaStoreProperties) MetastoreProperties.create(props);
+ Assertions.assertEquals("xxx", hmsAliyunDLFMetaStoreProperties.getHiveConf().get(DataLakeConfig.CATALOG_ACCESS_KEY_ID));
+ Assertions.assertEquals("xxx", hmsAliyunDLFMetaStoreProperties.getHiveConf().get(DataLakeConfig.CATALOG_ACCESS_KEY_SECRET));
+ Assertions.assertEquals("5789", hmsAliyunDLFMetaStoreProperties.getHiveConf().get(DataLakeConfig.CATALOG_ID));
+ Assertions.assertEquals("cn-shanghai", hmsAliyunDLFMetaStoreProperties.getHiveConf().get(DataLakeConfig.CATALOG_REGION_ID));
+ Assertions.assertEquals("dlf.cn-shanghai.aliyuncs.com", hmsAliyunDLFMetaStoreProperties.getHiveConf().get(DataLakeConfig.CATALOG_ENDPOINT));
+ Assertions.assertEquals("dlf", hmsAliyunDLFMetaStoreProperties.getHiveConf().get("hive.metastore.type"));
+ Assertions.assertEquals("hms", hmsAliyunDLFMetaStoreProperties.getHiveConf().get("type"));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStorePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStorePropertiesTest.java
index a1c0f7a..6ac93e4 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStorePropertiesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStorePropertiesTest.java
@@ -74,7 +74,7 @@
incompleteConfig.remove("aws.glue.secret-key");
HMSGlueMetaStoreProperties props = new HMSGlueMetaStoreProperties(incompleteConfig);
Exception exception = Assertions.assertThrows(IllegalArgumentException.class, props::initNormalizeAndCheckProps);
- Assertions.assertTrue(exception.getMessage().contains("glue.secret_key or aws.glue.secret-key"));
+ Assertions.assertTrue(exception.getMessage().contains("glue.secret_key is required"));
}
@Test
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergAliyunDLFMetaStorePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergAliyunDLFMetaStorePropertiesTest.java
new file mode 100644
index 0000000..5c77396
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergAliyunDLFMetaStorePropertiesTest.java
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.datasource.iceberg.dlf.DLFCatalog;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+import org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
+
+import org.apache.iceberg.catalog.Catalog;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+public class IcebergAliyunDLFMetaStorePropertiesTest {
+ @Test
+ void testGetIcebergCatalogType() {
+ Map<String, String> props = new HashMap<>();
+ props.put("dlf.access_key", "ak");
+ props.put("dlf.secret_key", "sk");
+ props.put("dlf.endpoint", "custom-endpoint");
+ props.put("dlf.catalog.uid", "123");
+
+ IcebergAliyunDLFMetaStoreProperties properties =
+ new IcebergAliyunDLFMetaStoreProperties(props);
+
+ Assertions.assertEquals("dlf", properties.getIcebergCatalogType());
+ }
+
+ @Test
+ void testInitCatalog() {
+ Map<String, String> props = new HashMap<>();
+ props.put("dlf.access_key", "ak");
+ props.put("dlf.secret_key", "sk");
+ props.put("dlf.endpoint", "dlf-vpc.cn-beijing.aliyuncs.com");
+ props.put("dlf.region", "cn-hz");
+ props.put("dlf.catalog.uid", "uid-123");
+ props.put("dlf.catalog.id", "id-456");
+ props.put("dlf.proxy.mode", "DLF_ONLY");
+
+ IcebergAliyunDLFMetaStoreProperties properties =
+ new IcebergAliyunDLFMetaStoreProperties(props);
+ // Replace DLFCatalog with a mock
+ Catalog catalog = properties.initCatalog("test_catalog", props,
+ Collections.singletonList(StorageProperties.createPrimary(props)));
+ Assertions.assertEquals(DLFCatalog.class, catalog.getClass());
+ }
+
+ @Test
+ void testAliyunDLFBasePropertiesSuccessWithPublicEndpoint() {
+ Map<String, String> props = new HashMap<>();
+ props.put("dlf.access_key", "ak");
+ props.put("dlf.secret_key", "sk");
+ props.put("dlf.region", "cn-shanghai");
+ props.put("dlf.access.public", "true");
+ props.put("dlf.uid", "uid-001");
+
+ AliyunDLFBaseProperties base = AliyunDLFBaseProperties.of(props);
+
+ Assertions.assertEquals("ak", base.dlfAccessKey);
+ Assertions.assertEquals("sk", base.dlfSecretKey);
+ Assertions.assertEquals("dlf.cn-shanghai.aliyuncs.com", base.dlfEndpoint);
+ Assertions.assertEquals("uid-001", base.dlfCatalogId); // defaulted to uid
+ }
+
+ @Test
+ void testAliyunDLFBasePropertiesSuccessWithVpcEndpoint() {
+ Map<String, String> props = new HashMap<>();
+ props.put("dlf.access_key", "ak");
+ props.put("dlf.secret_key", "sk");
+ props.put("dlf.region", "cn-hangzhou");
+ props.put("dlf.access.public", "false");
+ props.put("dlf.uid", "uid-002");
+ AliyunDLFBaseProperties base = AliyunDLFBaseProperties.of(props);
+ Assertions.assertEquals("dlf-vpc.cn-hangzhou.aliyuncs.com", base.dlfEndpoint);
+ Assertions.assertEquals("uid-002", base.dlfCatalogId);
+ }
+
+ @Test
+ void testAliyunDLFBasePropertiesThrowsWhenEndpointMissing() {
+ Map<String, String> props = new HashMap<>();
+ props.put("dlf.access_key", "ak");
+ props.put("dlf.secret_key", "sk");
+ // No endpoint and no region
+
+ Assertions.assertThrows(StoragePropertiesException.class,
+ () -> AliyunDLFBaseProperties.of(props));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStorePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStorePropertiesTest.java
new file mode 100644
index 0000000..6aa33c2
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStorePropertiesTest.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.datasource.property.storage.HdfsProperties;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class IcebergFileSystemMetaStorePropertiesTest {
+
+ @Test
+ public void testKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(HdfsProperties.FS_HDFS_SUPPORT, "true");
+ props.put("fs.defaultFS", "hdfs://mycluster_test");
+ props.put("hadoop.security.authentication", "kerberos");
+ props.put("hadoop.kerberos.principal", "myprincipal");
+ props.put("hadoop.kerberos.keytab", "mykeytab");
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "hadoop");
+ props.put("warehouse", "hdfs://mycluster_test/ice");
+ IcebergFileSystemMetaStoreProperties icebergProps = (IcebergFileSystemMetaStoreProperties) MetastoreProperties.create(props);
+ List<StorageProperties> storagePropertiesList = Collections.singletonList(StorageProperties.createPrimary(props));
+ //We expect a Kerberos-related exception, but because the messages vary by environment, we’re only doing a simple check.
+ Assertions.assertThrows(RuntimeException.class, () -> icebergProps.initializeCatalog("iceberg", storagePropertiesList));
+ }
+
+ @Test
+ public void testNonKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(HdfsProperties.FS_HDFS_SUPPORT, "true");
+ props.put("fs.defaultFS", "file:///tmp");
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "hadoop");
+ props.put("warehouse", "file:///tmp");
+ IcebergFileSystemMetaStoreProperties icebergProps = (IcebergFileSystemMetaStoreProperties) MetastoreProperties.create(props);
+ Assertions.assertEquals("hadoop", icebergProps.getIcebergCatalogType());
+ List<StorageProperties> storagePropertiesList = Collections.singletonList(StorageProperties.createPrimary(props));
+ Assertions.assertDoesNotThrow(() -> icebergProps.initializeCatalog("iceberg", storagePropertiesList));
+ props.put("fs.defaultFS", "hdfs://mycluster" + System.currentTimeMillis());
+ props.put("warehouse", "hdfs://mycluster" + System.currentTimeMillis());
+ IcebergFileSystemMetaStoreProperties icebergPropsFailed = (IcebergFileSystemMetaStoreProperties) MetastoreProperties.create(props);
+ RuntimeException e = Assertions.assertThrows(RuntimeException.class, () -> icebergPropsFailed.initializeCatalog("iceberg", storagePropertiesList));
+ Assertions.assertTrue(e.getMessage().contains("UnknownHostException:"));
+ }
+
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStorePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStorePropertiesTest.java
new file mode 100644
index 0000000..2eb5ca3
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStorePropertiesTest.java
@@ -0,0 +1,66 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.aws.glue.GlueCatalog;
+import org.apache.iceberg.catalog.Catalog;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+public class IcebergGlueMetaStorePropertiesTest {
+
+ @Test
+ public void glueTest() throws UserException {
+ Map<String, String> baseProps = ImmutableMap.of(
+ "type", "iceberg",
+ "iceberg.catalog.type", "glue",
+ "glue.region", "us-west-2",
+ "glue.access_key", "AK",
+ "glue.secret_key", "SK",
+ "glue.endpoint", "https://glue.us-west-2.amazonaws.com",
+ "warehouse", "s3://my-bucket/warehouse");
+ IcebergGlueMetaStoreProperties properties = (IcebergGlueMetaStoreProperties) MetastoreProperties.create(baseProps);
+ Assertions.assertEquals("glue", properties.getIcebergCatalogType());
+ Catalog catalog = properties.initializeCatalog("iceberg_catalog", StorageProperties.createAll(baseProps));
+ Assertions.assertEquals(GlueCatalog.class, catalog.getClass());
+ }
+
+ @Test
+ public void glueAndS3Test() throws UserException {
+ Map<String, String> baseProps = ImmutableMap.of(
+ "type", "iceberg",
+ "iceberg.catalog.type", "glue",
+ "glue.region", "us-west-2",
+ "glue.access_key", "AK",
+ "glue.secret_key", "SK",
+ "glue.endpoint", "https://glue.us-west-2.amazonaws.com",
+ "warehouse", "s3://my-bucket/warehouse",
+ "s3.region", "us-west-2",
+ "s3.endpoint", "https://s3.us-west-2.amazonaws.com"
+ );
+ IcebergGlueMetaStoreProperties properties = (IcebergGlueMetaStoreProperties) MetastoreProperties.create(baseProps);
+ Catalog catalog = properties.initializeCatalog("iceberg_catalog", StorageProperties.createAll(baseProps));
+ Assertions.assertEquals(GlueCatalog.class, catalog.getClass());
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStorePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStorePropertiesTest.java
new file mode 100644
index 0000000..63d3a0b
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStorePropertiesTest.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.common.security.authentication.HadoopExecutionAuthenticator;
+import org.apache.doris.datasource.property.storage.HdfsProperties;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class IcebergHMSMetaStorePropertiesTest {
+ @Test
+ public void testKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(HdfsProperties.FS_HDFS_SUPPORT, "true");
+ props.put("fs.defaultFS", "hdfs://mycluster_test");
+ props.put("hadoop.security.authentication", "kerberos");
+ props.put("hadoop.kerberos.principal", "myprincipal");
+ props.put("hadoop.kerberos.keytab", "mykeytab");
+ props.put("type", "iceberg");
+ props.put("hive.metastore.uris", "thrift://localhost:12345");
+ props.put("iceberg.catalog.type", "hms");
+ props.put("warehouse", "hdfs://mycluster_test/ice");
+ IcebergHMSMetaStoreProperties icebergProps = (IcebergHMSMetaStoreProperties) MetastoreProperties.create(props);
+ List<StorageProperties> storagePropertiesList = Collections.singletonList(StorageProperties.createPrimary(props));
+ //We expect a Kerberos-related exception, but because the messages vary by environment, we’re only doing a simple check.
+ Assertions.assertThrows(RuntimeException.class,
+ () -> icebergProps.initializeCatalog("iceberg", storagePropertiesList));
+ }
+
+ @Test
+ public void testNonKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(HdfsProperties.FS_HDFS_SUPPORT, "true");
+ props.put("fs.defaultFS", "file:///tmp");
+ props.put("type", "iceberg");
+ props.put("iceberg.catalog.type", "hms");
+ props.put("hive.metastore.uris", "thrift://localhost:9083");
+ props.put("warehouse", "file:///tmp");
+ IcebergHMSMetaStoreProperties paimonProps = (IcebergHMSMetaStoreProperties) MetastoreProperties.create(props);
+ Assertions.assertEquals("hms", paimonProps.getIcebergCatalogType());
+
+ List<StorageProperties> storagePropertiesList = Collections.singletonList(StorageProperties.createPrimary(props));
+ Assertions.assertDoesNotThrow(() -> paimonProps.initializeCatalog("iceberg", storagePropertiesList));
+ Assertions.assertEquals(HadoopExecutionAuthenticator.class, paimonProps.getExecutionAuthenticator().getClass());
+ }
+
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStorePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStorePropertiesTest.java
new file mode 100644
index 0000000..81d1f00
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStorePropertiesTest.java
@@ -0,0 +1,59 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.catalog.Catalog;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import software.amazon.s3tables.iceberg.S3TablesCatalog;
+
+import java.util.Map;
+
+public class IcebergS3TablesMetaStorePropertiesTest {
+
+ @Test
+ public void s3TablesTest() throws UserException {
+ Map<String, String> baseProps = ImmutableMap.of(
+ "type", "iceberg",
+ "iceberg.catalog.type", "s3tables",
+ "warehouse", "s3://my-bucket/warehouse");
+ Map<String, String> s3Props = ImmutableMap.of(
+ "s3.region", "us-west-2",
+ "s3.access_key", "AK",
+ "s3.secret_key", "SK",
+ "s3.endpoint", "https://s3.us-west-2.amazonaws.com");
+ IcebergS3TablesMetaStoreProperties failedProperties = (IcebergS3TablesMetaStoreProperties) MetastoreProperties.create(baseProps);
+ Assertions.assertEquals("s3tables", failedProperties.getIcebergCatalogType());
+ RuntimeException exception = Assertions.assertThrows(RuntimeException.class, () -> {
+ failedProperties.initializeCatalog("iceberg_catalog", StorageProperties.createAll(baseProps));
+ });
+ Assertions.assertTrue(exception.getMessage().contains("region must not be blank or empty."));
+ Map<String, String> allProps = ImmutableMap.<String, String>builder()
+ .putAll(baseProps)
+ .putAll(s3Props)
+ .build();
+ IcebergS3TablesMetaStoreProperties properties = (IcebergS3TablesMetaStoreProperties) MetastoreProperties.create(allProps);
+ Catalog catalog = properties.initializeCatalog("iceberg_catalog", StorageProperties.createAll(allProps));
+ Assertions.assertEquals(S3TablesCatalog.class, catalog.getClass());
+ }
+
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStorePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStorePropertiesTest.java
new file mode 100644
index 0000000..f2eade7
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonAliyunDLFMetaStorePropertiesTest.java
@@ -0,0 +1,139 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.apache.paimon.catalog.Catalog;
+import org.apache.paimon.catalog.CatalogContext;
+import org.apache.paimon.catalog.CatalogFactory;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class PaimonAliyunDLFMetaStorePropertiesTest {
+ private Map<String, String> createValidProps() {
+ Map<String, String> props = new HashMap<>();
+ props.put("type", "paimon");
+ props.put("paimon.catalog.type", "dlf");
+ props.put("dlf.access_key", "ak");
+ props.put("dlf.secret_key", "sk");
+ props.put("dlf.endpoint", "dlf.cn-hangzhou.aliyuncs.com");
+ props.put("dlf.region", "cn-hangzhou");
+ props.put("dlf.catalog.id", "catalogId");
+ props.put("dlf.uid", "uid");
+ props.put("warehouse", "oss://bucket/warehouse");
+ return props;
+ }
+
+ @Test
+ void testInitNormalizeAndCheckProps() {
+ Map<String, String> props = createValidProps();
+ PaimonAliyunDLFMetaStoreProperties dlfProps =
+ new PaimonAliyunDLFMetaStoreProperties(props);
+
+ dlfProps.initNormalizeAndCheckProps();
+
+ Assertions.assertEquals(
+ "dlf",
+ dlfProps.getPaimonCatalogType(),
+ "Catalog type should be PAIMON_DLF"
+ );
+ Assertions.assertEquals(
+ "hive",
+ dlfProps.getMetastoreType(),
+ "Metastore type should be hive"
+ );
+ }
+
+ @Test
+ void testInitializeCatalogWithValidOssProperties() throws UserException {
+ Map<String, String> props = createValidProps();
+ PaimonAliyunDLFMetaStoreProperties dlfProps =
+ new PaimonAliyunDLFMetaStoreProperties(props);
+ dlfProps.initNormalizeAndCheckProps();
+
+ // Prepare OSSProperties mock
+ Map<String, String> ossProps = new HashMap<>();
+ ossProps.put("oss.access_key", "ak");
+ ossProps.put("oss.secret_key", "sk");
+ ossProps.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com");
+
+
+ List<StorageProperties> storageProperties = StorageProperties.createAll(ossProps);
+
+ Catalog mockCatalog = Mockito.mock(Catalog.class);
+
+ try (MockedStatic<CatalogFactory> mocked = Mockito.mockStatic(CatalogFactory.class)) {
+ mocked.when(() -> CatalogFactory.createCatalog(Mockito.any(CatalogContext.class)))
+ .thenReturn(mockCatalog);
+
+ Catalog catalog = dlfProps.initializeCatalog("testCatalog", storageProperties);
+
+ Assertions.assertNotNull(catalog, "Catalog should not be null");
+ Assertions.assertEquals(mockCatalog, catalog, "Catalog should be the mocked one");
+
+ mocked.verify(() -> CatalogFactory.createCatalog(Mockito.any(CatalogContext.class)));
+ }
+ }
+
+ @Test
+ void testInitializeCatalogWithoutOssPropertiesThrows() {
+ Map<String, String> props = createValidProps();
+ PaimonAliyunDLFMetaStoreProperties dlfProps =
+ new PaimonAliyunDLFMetaStoreProperties(props);
+ dlfProps.initNormalizeAndCheckProps();
+
+ List<StorageProperties> storageProperties = new ArrayList<>(); // No OSS properties
+
+ IllegalStateException ex = Assertions.assertThrows(
+ IllegalStateException.class,
+ () -> dlfProps.initializeCatalog("testCatalog", storageProperties)
+ );
+
+ Assertions.assertTrue(ex.getMessage().contains("OSS storage properties"));
+ }
+
+ @Test
+ void testInitializeCatalogWithNonOssTypeThrows() {
+ Map<String, String> props = createValidProps();
+ PaimonAliyunDLFMetaStoreProperties dlfProps =
+ new PaimonAliyunDLFMetaStoreProperties(props);
+ dlfProps.initNormalizeAndCheckProps();
+
+ StorageProperties nonOssProps = Mockito.mock(StorageProperties.class);
+ Mockito.when(nonOssProps.getType()).thenReturn(StorageProperties.Type.HDFS);
+
+ List<StorageProperties> storageProperties = Collections.singletonList(nonOssProps);
+
+ IllegalStateException ex = Assertions.assertThrows(
+ IllegalStateException.class,
+ () -> dlfProps.initializeCatalog("testCatalog", storageProperties)
+ );
+
+ Assertions.assertTrue(ex.getMessage().contains("Paimon DLF metastore requires OSS storage properties."));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java
index 59766ef..8563300 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java
@@ -20,6 +20,7 @@
import org.apache.doris.datasource.property.storage.StorageProperties;
import org.apache.paimon.catalog.Catalog;
+import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@@ -70,12 +71,10 @@
System.out.println("Catalog initialization skipped: Missing valid S3 Access Key/Secret Key.");
return null;
}
-
AbstractPaimonProperties metaStoreProps =
(AbstractPaimonProperties) MetastoreProperties.create(params);
-
metaStoreProps.initNormalizeAndCheckProps();
-
+ Assertions.assertNotNull(metaStoreProps.getExecutionAuthenticator());
List<StorageProperties> storageProps = StorageProperties.createAll(params);
return metaStoreProps.initializeCatalog("paimon_catalog", storageProps);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStorePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStorePropertiesTest.java
new file mode 100644
index 0000000..fa52316
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStorePropertiesTest.java
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.common.security.authentication.HadoopExecutionAuthenticator;
+import org.apache.doris.datasource.property.storage.HdfsProperties;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.apache.paimon.catalog.FileSystemCatalogFactory;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class PaimonFileSystemMetaStorePropertiesTest {
+
+ @Test
+ public void testKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(HdfsProperties.FS_HDFS_SUPPORT, "true");
+ props.put("fs.defaultFS", "hdfs://mycluster_test");
+ props.put("hadoop.security.authentication", "kerberos");
+ props.put("hadoop.kerberos.principal", "myprincipal");
+ props.put("hadoop.kerberos.keytab", "mykeytab");
+ props.put("type", "paimon");
+ props.put("paimon.catalog.type", "filesystem");
+ props.put("warehouse", "hdfs://mycluster_test/paimon");
+ PaimonFileSystemMetaStoreProperties paimonProps = (PaimonFileSystemMetaStoreProperties) MetastoreProperties.create(props);
+ //We expect a Kerberos-related exception, but because the messages vary by environment, we’re only doing a simple check.
+ Assertions.assertThrows(RuntimeException.class, () -> paimonProps.initializeCatalog("paimon", StorageProperties.createAll(props))
+ );
+ }
+
+ @Test
+ public void testNonKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("fs.defaultFS", "file:///tmp");
+ props.put("type", "paimon");
+ props.put("paimon.catalog.type", "filesystem");
+ props.put("warehouse", "file:///tmp");
+ PaimonFileSystemMetaStoreProperties paimonProps = (PaimonFileSystemMetaStoreProperties) MetastoreProperties.create(props);
+ Assertions.assertEquals(FileSystemCatalogFactory.IDENTIFIER, paimonProps.getMetastoreType());
+ Assertions.assertEquals("filesystem", paimonProps.getPaimonCatalogType());
+ Assertions.assertDoesNotThrow(() -> paimonProps.initializeCatalog("paimon", StorageProperties.createAll(props)));
+ Assertions.assertEquals(HadoopExecutionAuthenticator.class, paimonProps.getExecutionAuthenticator().getClass());
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStorePropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStorePropertiesTest.java
new file mode 100644
index 0000000..ef382c2
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStorePropertiesTest.java
@@ -0,0 +1,65 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.datasource.property.storage.HdfsProperties;
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class PaimonHMSMetaStorePropertiesTest {
+ @Test
+ public void testKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(HdfsProperties.FS_HDFS_SUPPORT, "true");
+ props.put("fs.defaultFS", "hdfs://mycluster_test");
+ props.put("hadoop.security.authentication", "kerberos");
+ props.put("hadoop.kerberos.principal", "myprincipal");
+ props.put("hadoop.kerberos.keytab", "mykeytab");
+ props.put("type", "paimon");
+ props.put("hive.metastore.uris", "thrift://localhost:12345");
+ props.put("paimon.catalog.type", "hms");
+ props.put("warehouse", "hdfs://mycluster/paimon");
+ PaimonHMSMetaStoreProperties paimonProps = (PaimonHMSMetaStoreProperties) MetastoreProperties.create(props);
+ List<StorageProperties> storagePropertiesList = Collections.singletonList(StorageProperties.createPrimary(props));
+ //We expect a Kerberos-related exception, but because the messages vary by environment, we’re only doing a simple check.
+ Assertions.assertThrows(RuntimeException.class,
+ () -> paimonProps.initializeCatalog("paimon", storagePropertiesList));
+ }
+
+ @Test
+ public void testNonKerberosCatalog() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(HdfsProperties.FS_HDFS_SUPPORT, "true");
+ props.put("fs.defaultFS", "file:///tmp");
+ props.put("type", "paimon");
+ props.put("paimon.catalog.type", "hms");
+ props.put("hive.metastore.uris", "thrift://localhost:9083");
+ props.put("warehouse", "file:///tmp");
+ PaimonHMSMetaStoreProperties paimonProps = (PaimonHMSMetaStoreProperties) MetastoreProperties.create(props);
+ Assertions.assertEquals("hms", paimonProps.getPaimonCatalogType());
+ //should mock connection to hms
+ }
+
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
index 0cf30e4..bd3c091 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
@@ -23,6 +23,8 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import java.util.HashMap;
import java.util.Map;
@@ -161,4 +163,17 @@
origProps.remove("cos.access_key");
Assertions.assertDoesNotThrow(() -> StorageProperties.createPrimary(origProps));
}
+
+ @Test
+ public void testAwsCredentialsProvider() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("fs.cos.support", "true");
+ props.put("cos.endpoint", "cos.ap-beijing.myqcloud.com");
+ COSProperties obsStorageProperties = (COSProperties) StorageProperties.createPrimary(props);
+ Assertions.assertEquals(AnonymousCredentialsProvider.class, obsStorageProperties.getAwsCredentialsProvider().getClass());
+ props.put("cos.access_key", "myAccessKey");
+ props.put("cos.secret_key", "mySecretKey");
+ obsStorageProperties = (COSProperties) StorageProperties.createPrimary(props);
+ Assertions.assertEquals(StaticCredentialsProvider.class, obsStorageProperties.getAwsCredentialsProvider().getClass());
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/GCSPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/GCSPropertiesTest.java
index dd98613..c7167a8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/GCSPropertiesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/GCSPropertiesTest.java
@@ -20,6 +20,8 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import java.util.HashMap;
import java.util.Map;
@@ -96,4 +98,16 @@
Assertions.assertEquals("888", s3Props.get("AWS_CONNECTION_TIMEOUT_MS"));
Assertions.assertEquals("true", s3Props.get("use_path_style"));
}
+
+ @Test
+ public void testGCSAwsCredentialsProvider() throws Exception {
+ Map<String, String> gcsProps = new HashMap<>();
+ gcsProps.put("fs.gcs.support", "true");
+ GCSProperties gcsStorageProperties = (GCSProperties) StorageProperties.createPrimary(gcsProps);
+ Assertions.assertEquals(AnonymousCredentialsProvider.class, gcsStorageProperties.getAwsCredentialsProvider().getClass());
+ gcsProps.put("gs.access_key", "myAccessKey");
+ gcsProps.put("gs.secret_key", "mySecretKey");
+ gcsStorageProperties = (GCSProperties) StorageProperties.createPrimary(gcsProps);
+ Assertions.assertEquals(StaticCredentialsProvider.class, gcsStorageProperties.getAwsCredentialsProvider().getClass());
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
index 2c6f5ff..acaa21a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
@@ -22,6 +22,8 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import java.util.HashMap;
import java.util.Map;
@@ -138,6 +140,19 @@
}
@Test
+ public void testAwsCredentialsProvider() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put("fs.obs.support", "true");
+ props.put("obs.endpoint", "obs.cn-north-4.myhuaweicloud.com");
+ OBSProperties obsStorageProperties = (OBSProperties) StorageProperties.createPrimary(props);
+ Assertions.assertEquals(AnonymousCredentialsProvider.class, obsStorageProperties.getAwsCredentialsProvider().getClass());
+ props.put("obs.access_key", "myAccessKey");
+ props.put("obs.secret_key", "mySecretKey");
+ obsStorageProperties = (OBSProperties) StorageProperties.createPrimary(props);
+ Assertions.assertEquals(StaticCredentialsProvider.class, obsStorageProperties.getAwsCredentialsProvider().getClass());
+ }
+
+ @Test
public void testMissingSecretKey() {
origProps.put("obs.endpoint", "obs.cn-north-4.myhuaweicloud.com");
origProps.put("obs.access_key", "myOBSAccessKey");
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
index 31b56a2..f00fb02 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
@@ -22,6 +22,8 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import java.util.HashMap;
import java.util.Map;
@@ -203,4 +205,17 @@
Assertions.assertEquals(1, StorageProperties.createAll(origProps).size());
}
+ @Test
+ public void testAwsCredentialsProvider() throws Exception {
+ Map<String, String> ossProps = new HashMap<>();
+ ossProps.put("fs.oss.support", "true");
+ ossProps.put("oss.endpoint", "oss-cn-hangzhou.aliyuncs.com");
+ OSSProperties ossStorageProperties = (OSSProperties) StorageProperties.createPrimary(ossProps);
+ Assertions.assertEquals(AnonymousCredentialsProvider.class, ossStorageProperties.getAwsCredentialsProvider().getClass());
+ ossProps.put("oss.access_key", "myAccessKey");
+ ossProps.put("oss.secret_key", "mySecretKey");
+ ossStorageProperties = (OSSProperties) StorageProperties.createPrimary(ossProps);
+ Assertions.assertEquals(StaticCredentialsProvider.class, ossStorageProperties.getAwsCredentialsProvider().getClass());
+ }
+
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
index 4852eff..70ffa2b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
@@ -26,6 +26,7 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
@@ -410,4 +411,13 @@
Assertions.assertEquals("aaa", s3Properties.accessKey);
Assertions.assertEquals("bbb", s3Properties.secretKey);
}
+
+ @Test
+ public void testS3PropertiesAwsAnonymousCredentialsProvider() {
+ Map<String, String> props = Maps.newHashMap();
+ props.put("s3.endpoint", "s3.us-west-2.amazonaws.com");
+ S3Properties s3Properties = (S3Properties) StorageProperties.createPrimary(props);
+ AwsCredentialsProvider provider = s3Properties.getAwsCredentialsProvider();
+ Assertions.assertEquals(AnonymousCredentialsProvider.class, provider.getClass());
+ }
}
diff --git a/regression-test/data/external_table_p0/hive/test_hive_basic_type.out b/regression-test/data/external_table_p0/hive/test_hive_basic_type.out
index 5527116..a0d73c5 100644
--- a/regression-test/data/external_table_p0/hive/test_hive_basic_type.out
+++ b/regression-test/data/external_table_p0/hive/test_hive_basic_type.out
@@ -532,7 +532,7 @@
123.456
-- !2 --
-1 7706 1 155190 17.00 21168.23 0.04 0.02 N O 1996-03-13 1996-02-12 1996-03-22 DELIVER IN PERSON TRUCK egular courts above the cn beijing
+1 638 6 15635 32.00 49620.16 0.07 0.02 N O 1996-01-30 1996-02-07 1996-02-03 DELIVER IN PERSON MAIL arefully slyly ex cn beijing
-- !3 --
6374628540732951412 -77 -65 -70 -107 -215 65 0 -526 -1309 3750 8827 -19795 34647 57042 -1662 -138248 -890685 -228568 1633079 -2725524 6163040 -10491702 697237 74565050 127767368 93532213 -209675435 -32116110 -3624917040 -2927805617 15581947241 21893441661 24075494509 -116822110531 -59683724667 -146210393388 114424524398 1341560771667 -1638742564263 520137948334 -2927347587131 7415137351179 -7963937754617 52157548982266 140803519083304 -294675355729619 -868076759504942 181128508165910 -91753231238823 -3511241416682881 -11545256318348796 -1952917510863468 -5161099825338866 -59726090170689781 287170105829528178 607326725526282735 1253194074103207461 -162443950414676064 -2964036188567341159 2602201580810990248 5581917084094110764 111739292249520611 -315687754593838642 -2804420462762366976 -2078683524
@@ -798,7 +798,7 @@
123.456
-- !2 --
-1 7706 1 155190 17.00 21168.23 0.04 0.02 N O 1996-03-13 1996-02-12 1996-03-22 DELIVER IN PERSON TRUCK egular courts above the cn beijing
+1 638 6 15635 32.00 49620.16 0.07 0.02 N O 1996-01-30 1996-02-07 1996-02-03 DELIVER IN PERSON MAIL arefully slyly ex cn beijing
-- !3 --
6374628540732951412 -77 -65 -70 -107 -215 65 0 -526 -1309 3750 8827 -19795 34647 57042 -1662 -138248 -890685 -228568 1633079 -2725524 6163040 -10491702 697237 74565050 127767368 93532213 -209675435 -32116110 -3624917040 -2927805617 15581947241 21893441661 24075494509 -116822110531 -59683724667 -146210393388 114424524398 1341560771667 -1638742564263 520137948334 -2927347587131 7415137351179 -7963937754617 52157548982266 140803519083304 -294675355729619 -868076759504942 181128508165910 -91753231238823 -3511241416682881 -11545256318348796 -1952917510863468 -5161099825338866 -59726090170689781 287170105829528178 607326725526282735 1253194074103207461 -162443950414676064 -2964036188567341159 2602201580810990248 5581917084094110764 111739292249520611 -315687754593838642 -2804420462762366976 -2078683524
diff --git a/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy b/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
index 4c531d7..68f5b9a 100644
--- a/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
+++ b/regression-test/suites/external_table_p2/refactor_catalog_param/hive_on_hms_and_dlf.groovy
@@ -167,6 +167,40 @@
'cos.endpoint' = '${cos_endpoint}'
"""
+ /***************OBS*******************/
+ String obs_ak = context.config.otherConfigs.get("hwYunAk")
+ String obs_sk = context.config.otherConfigs.get("hwYunSk")
+ String obs_endpoint =context.config.otherConfigs.get("hwYunEndpoint")
+ String obs_region = context.config.otherConfigs.get("hwYunRegion")
+ String obs_region_param = """
+ 'obs.region' = '${obs_region}',
+ """
+
+ String obs_parent_path = context.config.otherConfigs.get("hwYunBucket")
+
+ String obs_storage_properties = """
+ 'obs.access_key' = '${obs_ak}',
+ 'obs.secret_key' = '${obs_sk}',
+ 'obs.endpoint' = '${obs_endpoint}'
+ """
+ /***************GCS*******************/
+ String gcs_ak = context.config.otherConfigs.get("GCSAk")
+ String gcs_sk = context.config.otherConfigs.get("GCSSk")
+ String gcs_endpoint = "storage.googleapis.com"
+
+ String gcs_parent_path = "selectdb-qa-datalake-test";
+ String gcs_storage_old_properties = """
+ 'gs.access_key' = '${gcs_ak}',
+ 'gs.secret_key' = '${gcs_sk}',
+ 'gs.endpoint' = '${gcs_endpoint}'
+ """
+ String gcs_storage_new_properties = """
+ 'fs.gcs.support' = 'true',
+ 'gs.access_key' = '${gcs_ak}',
+ 'gs.secret_key' = '${gcs_sk}'
+
+ """
+
/****************HDFS*******************/
//simple
String hdfs_parent_path = "hdfs://${externalEnvIp}:8320/user/hive/warehouse"
@@ -227,8 +261,23 @@
String hms_type_properties = """
"type"="hms",
"""
+ //OBS
+ String db_location = "obs://${obs_parent_path}/hive/hms/" + System.currentTimeMillis()
+ testQueryAndInsert(hms_properties + obs_storage_properties, "hive_hms_obs_test", db_location)
+ testQueryAndInsert(hms_properties + obs_region_param + obs_storage_properties, "hive_hms_obs_test_region", db_location)
+ testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + obs_storage_properties, "hive_hms_on_obs_kerberos_old", db_location)
+ testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop + obs_storage_properties, "hive_hms_on_obs_kerberos_new", db_location)
+ //GCS
+ if(context.config.otherConfigs.get("enableGCS")){
+ db_location = "gs://${gcs_parent_path}/hive/hms/" + System.currentTimeMillis()
+ testQueryAndInsert(hms_properties + gcs_storage_old_properties, "hive_hms_gcs_test", db_location)
+ testQueryAndInsert(hms_properties + gcs_storage_new_properties, "hive_hms_gcs_test_new", db_location)
+ testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + gcs_storage_old_properties, "hive_hms_on_gcs_kerberos_old", db_location)
+ testQueryAndInsert(hms_type_properties + hms_kerberos_new_prop + gcs_storage_new_properties, "hive_hms_on_gcs_kerberos_new", db_location)
+ }
+
//COS
- String db_location = "cosn://${cos_parent_path}/hive/hms/" + System.currentTimeMillis()
+ db_location = "cosn://${cos_parent_path}/hive/hms/" + System.currentTimeMillis()
testQueryAndInsert(hms_properties + cos_storage_properties, "hive_hms_cos_test", db_location)
testQueryAndInsert(hms_properties + cos_region_param + cos_storage_properties, "hive_hms_cos_test_region", db_location)
testQueryAndInsert(hms_type_properties + hms_kerberos_old_prop + cos_storage_properties, "hive_hms_on_cos_kerberos_old", db_location)
diff --git a/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
index 56e21da..38c28a2 100644
--- a/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
+++ b/regression-test/suites/external_table_p2/refactor_catalog_param/iceberg_on_hms_and_filesystem_and_dlf.groovy
@@ -159,7 +159,39 @@
'cos.secret_key' = '${cos_sk}',
'cos.endpoint' = '${cos_endpoint}'
"""
+ /***************OBS*******************/
+ String obs_ak = context.config.otherConfigs.get("hwYunAk")
+ String obs_sk = context.config.otherConfigs.get("hwYunSk")
+ String obs_endpoint =context.config.otherConfigs.get("hwYunEndpoint")
+ String obs_region = context.config.otherConfigs.get("hwYunRegion")
+ String obs_region_param = """
+ 'obs.region' = '${obs_region}',
+ """
+ String obs_parent_path = context.config.otherConfigs.get("hwYunBucket")
+
+ String obs_storage_properties = """
+ 'obs.access_key' = '${obs_ak}',
+ 'obs.secret_key' = '${obs_sk}',
+ 'obs.endpoint' = '${obs_endpoint}'
+ """
+ /***************GCS*******************/
+ String gcs_ak = context.config.otherConfigs.get("GCSAk")
+ String gcs_sk = context.config.otherConfigs.get("GCSSk")
+ String gcs_endpoint = "storage.googleapis.com"
+
+ String gcs_parent_path = "selectdb-qa-datalake-test";
+ String gcs_storage_old_properties = """
+ 'gs.access_key' = '${gcs_ak}',
+ 'gs.secret_key' = '${gcs_sk}',
+ 'gs.endpoint' = '${gcs_endpoint}'
+ """
+ String gcs_storage_new_properties = """
+ 'fs.gcs.support' = 'true',
+ 'gs.access_key' = '${gcs_ak}',
+ 'gs.secret_key' = '${gcs_sk}'
+
+ """
/****************HDFS*******************/
//simple
String hdfs_parent_path = "hdfs://${externalEnvIp}:8320/user/iceberg/warehouse"
@@ -230,7 +262,35 @@
//new kerberos
testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_new_prop + warehouse + oss_storage_properties, "iceberg_hms_on_oss_kerberos_new")
+ /*--------HMS on OBS-----------*/
+ warehouse = """
+ 'warehouse' = 'obs://${obs_parent_path}/iceberg-hms-obs-warehouse',
+ """
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop
+ + warehouse + obs_storage_properties, "iceberg_hms_on_obs")
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop+
+ warehouse + obs_region_param + obs_storage_properties, "iceberg_hms_on_obs")
+ //old kerberos
+ testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_old_prop + warehouse + obs_storage_properties, "iceberg_hms_on_obs_kerberos_old")
+ //new kerberos
+ testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_new_prop + warehouse + obs_storage_properties, "iceberg_hms_on_obs_kerberos_new")
+ /*--------HMS on GCS-----------*/
+ if(context.config.otherConfigs.get("enableGCS")){
+ warehouse = """
+ 'warehouse' = 'gs://${gcs_parent_path}/gcs/iceberg-hms-gcs-warehouse',
+ """
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop
+ + warehouse + gcs_storage_old_properties, "iceberg_hms_on_gcs_old")
+ testQueryAndInsert(iceberg_hms_type_prop + hms_prop+
+ warehouse + gcs_storage_new_properties, "iceberg_hms_on_gcs_new")
+
+ //new kerberos
+ testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_new_prop + warehouse + gcs_storage_new_properties, "iceberg_hms_on_gcs_kerberos_new")
+ //old kerberos
+ testQueryAndInsert(iceberg_hms_type_prop + hms_kerberos_old_prop + warehouse + gcs_storage_new_properties, "iceberg_hms_on_gcs_kerberos_old")
+ }
+
/*--------HMS on COS-----------*/
warehouse = """
'warehouse' = 'cosn://${cos_parent_path}/iceberg-hms-cos-warehouse',
@@ -285,6 +345,31 @@
'type'='iceberg',
'iceberg.catalog.type'='hadoop',
"""
+ /** OBS **/
+ warehouse = """
+ 'warehouse' = 'obs://${obs_parent_path}/iceberg-fs-obs-warehouse',
+ """
+ testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse + obs_storage_properties, "iceberg_fs_on_obs")
+ testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse + obs_region_param + obs_storage_properties, "iceberg_fs_on_obs_region")
+ //s3
+ warehouse = """
+ 'warehouse' = 's3://${obs_parent_path}/iceberg-fs-obs-warehouse',
+ """
+ testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse + obs_storage_properties, "iceberg_fs_on_obs_s3")
+ if(context.config.otherConfigs.get("enableGCS")){
+ /** GCS **/
+ warehouse = """
+ 'warehouse' = 'gs://${gcs_parent_path}/gcs/iceberg-fs-gcs-warehouse',
+ """
+ testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse + gcs_storage_old_properties, "iceberg_fs_on_gcs_old")
+ testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse + gcs_storage_new_properties, "iceberg_fs_on_gcs_new")
+ //s3
+ warehouse = """
+ 'warehouse' = 's3://${gcs_parent_path}/gcs/iceberg-fs-gcs-warehouse',
+ """
+ testQueryAndInsert(iceberg_file_system_catalog_properties + warehouse + gcs_storage_old_properties, "iceberg_fs_on_gcs_old")
+
+ }
/** COS **/
warehouse = """
'warehouse' = 'cos://${cos_parent_path}/iceberg-fs-cos-warehouse',
@@ -383,4 +468,4 @@
}
-}
\ No newline at end of file
+}
