[Improve] config init bug fixed. (#3688)
* [Improve] StreamPark Platform configuration files improvements
* [Improve] springboot testcase improve
* [Improve] yarnUtils bug fixed
* [Improve] minor improve
* [Improve] init config bug fixed
---------
Co-authored-by: benjobs <benjobx@gmail.com>
diff --git a/streampark-console/streampark-console-service/src/main/assembly/assembly.xml b/streampark-console/streampark-console-service/src/main/assembly/assembly.xml
index b8f82cc..b221f60 100644
--- a/streampark-console/streampark-console-service/src/main/assembly/assembly.xml
+++ b/streampark-console/streampark-console-service/src/main/assembly/assembly.xml
@@ -67,6 +67,12 @@
<fileMode>0755</fileMode>
</fileSet>
<fileSet>
+ <directory>${project.build.directory}/../src/main/assembly/conf</directory>
+ <outputDirectory>conf</outputDirectory>
+ <lineEnding>unix</lineEnding>
+ <fileMode>0755</fileMode>
+ </fileSet>
+ <fileSet>
<directory>${project.build.directory}/../src/main/assembly/logs</directory>
<outputDirectory>logs</outputDirectory>
<fileMode>0755</fileMode>
@@ -103,9 +109,7 @@
<lineEnding>unix</lineEnding>
<fileMode>0755</fileMode>
<includes>
- <include>*.yml</include>
- <include>*.properties</include>
- <include>logback*.xml</include>
+ <include>logback-spring.xml</include>
</includes>
</fileSet>
<fileSet>
diff --git a/streampark-console/streampark-console-service/src/main/assembly/conf/config.yaml b/streampark-console/streampark-console-service/src/main/assembly/conf/config.yaml
new file mode 100644
index 0000000..e979918
--- /dev/null
+++ b/streampark-console/streampark-console-service/src/main/assembly/conf/config.yaml
@@ -0,0 +1,96 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+logging:
+ level:
+ root: info
+
+server:
+ port: 10000
+ session:
+ # The user's login session has a validity period. If it exceeds this time, the user will be automatically logout
+ # unit: s|m|h|d, s: second, m:minute, h:hour, d: day
+ ttl: 2h # unit[s|m|h|d], e.g: 24h, 2d....
+ undertow: # see: https://github.com/undertow-io/undertow/blob/master/core/src/main/java/io/undertow/Undertow.java
+ buffer-size: 1024
+ direct-buffers: true
+ threads:
+ io: 16
+ worker: 256
+
+# system database, default h2, mysql|pgsql|h2
+datasource:
+ dialect: h2 #h2, mysql, pgsql
+ # if datasource.dialect is mysql or pgsql, you need to configure the following connection information
+ # mysql/postgresql connect user
+ username:
+ # mysql/postgresql connect password
+ password:
+ # mysql/postgresql connect jdbcURL
+ # mysql example: datasource.url: jdbc:mysql://localhost:3306/streampark?useUnicode=true&characterEncoding=UTF-8&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=GMT%2B8
+ # postgresql example: jdbc:postgresql://localhost:5432/streampark?stringtype=unspecified
+ url:
+
+streampark:
+ workspace:
+ # Local workspace, storage directory of clone projects and compiled projects,Do not set under $APP_HOME. Set it to a directory outside of $APP_HOME.
+ local: /tmp/streampark
+ # The root hdfs path of the jars, Same as yarn.provided.lib.dirs for flink on yarn-application and Same as --jars for spark on yarn
+ remote: hdfs:///streampark/
+ proxy:
+ # lark proxy address, default https://open.feishu.cn
+ lark-url:
+ # hadoop yarn proxy path, e.g: knox process address https://streampark.com:8443/proxy/yarn
+ yarn-url:
+ yarn:
+ # flink on yarn or spark on yarn, monitoring job status from yarn, it is necessary to set hadoop.http.authentication.type
+ http-auth: 'simple' # default simple, or kerberos
+ # flink on yarn or spark on yarn, HADOOP_USER_NAME
+ hadoop-user-name: hdfs
+
+# flink on yarn or spark on yarn, when the hadoop cluster enable kerberos authentication, it is necessary to set Kerberos authentication parameters.
+security:
+ kerberos:
+ login:
+ debug: false
+ enable: false
+ keytab:
+ krb5:
+ principal:
+ ttl: 2h # unit [s|m|h|d]
+
+# sign streampark with ldap.
+ldap:
+ base-dn: dc=streampark,dc=com # Login Account
+ enable: false # ldap enabled'
+ username: cn=Manager,dc=streampark,dc=com
+ password: streampark
+ urls: ldap://99.99.99.99:389 #AD server IP, default port 389
+ user:
+ email-attribute: mail
+ identity-attribute: uid
+
+sso:
+ enable: false
+ callbackUrl: http://localhost:10000/callback
+ # Put all parameters under `properties`
+ # Check supported sso config parameters for different authentication clients from the below link
+ # https://github.com/pac4j/pac4j/blob/master/documentation/docs/config-module.md
+ properties:
+ principalNameAttribute:
+ # Optional, change by authentication client
+ # Please replace and fill in your client config below when enabled SSO
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java
index 9d86b0c..ecba470 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/config/SpringProperties.java
@@ -17,6 +17,7 @@
package org.apache.streampark.console.base.config;
+import org.apache.streampark.common.conf.ConfigKeys;
import org.apache.streampark.common.util.PropertiesUtils;
import org.apache.streampark.common.util.SystemPropertyUtils;
import org.apache.streampark.console.base.util.WebUtils;
@@ -24,18 +25,13 @@
import org.apache.commons.lang3.StringUtils;
import com.google.common.collect.Maps;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.InputStream;
import java.util.Map;
import java.util.Properties;
public class SpringProperties {
- private static final Logger log = LoggerFactory.getLogger(SpringProperties.class);
-
public static Properties get() {
// 1) get spring config
Properties springConfig = getSpringConfig();
@@ -111,32 +107,24 @@
});
}
- private static boolean useOldConfig() {
- String appHome = WebUtils.getAppHome();
- if (appHome == null) {
- return false;
- }
- File file = new File(appHome + "/conf/application.yml");
- return file.exists();
- }
-
private static Properties getUserConfig() {
String appHome = WebUtils.getAppHome();
+ if (StringUtils.isBlank(appHome)) {
+ throw new ExceptionInInitializerError(
+ String.format(
+ "[StreamPark] The system initialization check failed. If started local for development and debugging,"
+ + " please ensure the -D%s parameter is clearly specified,"
+ + " more detail: https://streampark.apache.org/docs/user-guide/deployment",
+ ConfigKeys.KEY_APP_HOME()));
+ }
Properties properties = new Properties();
- if (appHome != null) {
- File file = new File(appHome + "/conf/config.yaml");
- if (file.exists() && file.isFile()) {
- Map<String, String> config = PropertiesUtils.fromYamlFileAsJava(file.getAbsolutePath());
- properties.putAll(config);
- return properties;
- }
- throw new ExceptionInInitializerError(file.getAbsolutePath() + " not found, please check.");
- } else {
- InputStream inputStream =
- SpringProperties.class.getClassLoader().getResourceAsStream("config.yaml");
- Map<String, String> config = PropertiesUtils.fromYamlFileAsJava(inputStream);
+ File file = new File(appHome, "conf/config.yaml");
+ if (file.exists() && file.isFile()) {
+ Map<String, String> config = PropertiesUtils.fromYamlFileAsJava(file.getAbsolutePath());
properties.putAll(config);
return properties;
+ } else {
+ throw new ExceptionInInitializerError(file.getAbsolutePath() + " not found, please check.");
}
}
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/EnvInitializer.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/EnvInitializer.java
index 89bb662..e0fd144 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/EnvInitializer.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/runner/EnvInitializer.java
@@ -79,8 +79,6 @@
@Override
public void run(ApplicationArguments args) throws Exception {
- checkAppHome();
-
// init InternalConfig
initConfig();
@@ -145,19 +143,6 @@
initialized.add(storageType);
}
- private static void checkAppHome() {
- final String appHome = WebUtils.getAppHome();
- if (StringUtils.isBlank(appHome)) {
- throw new ExceptionInInitializerError(
- String.format(
- "[StreamPark] Workspace path check failed,"
- + " The system initialization check failed. If started local for development and debugging,"
- + " please ensure the -D%s parameter is clearly specified,"
- + " more detail: https://streampark.apache.org/docs/user-guide/deployment",
- ConfigKeys.KEY_APP_HOME()));
- }
- }
-
private void prepareWorkspace(
StorageType storageType, FsOperator fsOperator, Workspace workspace) {
if (LFS == storageType) {
diff --git a/streampark-console/streampark-console-service/src/main/resources/kerberos.yml b/streampark-console/streampark-console-service/src/main/resources/kerberos.yml
deleted file mode 100644
index b819905..0000000
--- a/streampark-console/streampark-console-service/src/main/resources/kerberos.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-security:
- kerberos:
- ttl: 2h # unit [s|m|h|d]
- login:
- enable: false
- debug: false
- principal:
- krb5:
- keytab:
