[ISSUES-3623] Project management encrypts user warehouse passwords (#3630)
* [ISSUES-3623] Project management encrypts user warehouse passwords
* [ISSUES-3623] Project management encrypts user warehouse passwords
* [ISSUES-3623] checkstyle & spotless:apply
---------
Co-authored-by: jianjun.xu <jianjun.xu@ly.com>
diff --git a/streampark-console/streampark-console-service/src/main/assembly/script/data/mysql-data.sql b/streampark-console/streampark-console-service/src/main/assembly/script/data/mysql-data.sql
index b478103..8b232c9 100644
--- a/streampark-console/streampark-console-service/src/main/assembly/script/data/mysql-data.sql
+++ b/streampark-console/streampark-console-service/src/main/assembly/script/data/mysql-data.sql
@@ -40,7 +40,7 @@
-- ----------------------------
-- Records of t_flink_project
-- ----------------------------
-insert into `t_flink_project` values (100000, 100000, 'streampark-quickstart', 'https://github.com/apache/incubator-streampark-quickstart', 'release-2.0.0', null, null, null, null, null, 1, 1, null, 'streampark-quickstart', -1, now(), now());
+insert into `t_flink_project` values (100000, 100000, 'streampark-quickstart', 'https://github.com/apache/incubator-streampark-quickstart', 'release-2.0.0', null, null, null, null, null, null, 1, 1, null, 'streampark-quickstart', -1, now(), now());
-- ----------------------------
-- Records of t_flink_sql
@@ -87,10 +87,10 @@
insert into `t_menu` values (130300, 130000, 'resource.upload', '/resource/upload', 'resource/upload/View', null, null, '0', 1, 3, now(), now());
insert into `t_menu` values (130101, 130100, 'project view', null, null, 'project:view', null, '1', 1, null, now(), now());
-insert into `t_menu` values (130102, 130100, 'project add', '/project/add', 'project/Add', 'project:create', '', '0', 0, null, now(), now());
+insert into `t_menu` values (130102, 130100, 'project add', '/project/add', 'resource/project/Add', 'project:create', '', '0', 0, null, now(), now());
insert into `t_menu` values (130103, 130100, 'project build', null, null, 'project:build', null, '1', 1, null, now(), now());
insert into `t_menu` values (130104, 130100, 'project delete', null, null, 'project:delete', null, '1', 1, null, now(), now());
-insert into `t_menu` values (130105, 130100, 'project edit', '/project/edit', 'project/Edit', 'project:update', null, '0', 0, null, now(), now());
+insert into `t_menu` values (130105, 130100, 'project edit', '/project/edit', 'resource/project/Edit', 'project:update', null, '0', 0, null, now(), now());
insert into `t_menu` values (130201, 130200, 'variable view', NULL, NULL, 'variable:view', NULL, '1', 1, null, now(), now());
insert into `t_menu` values (130202, 130200, 'variable depend view', null, null, 'variable:depend_apps', null, '1', 1, NULL, now(), now());
diff --git a/streampark-console/streampark-console-service/src/main/assembly/script/data/pgsql-data.sql b/streampark-console/streampark-console-service/src/main/assembly/script/data/pgsql-data.sql
index d9105e5..ee914d3 100644
--- a/streampark-console/streampark-console-service/src/main/assembly/script/data/pgsql-data.sql
+++ b/streampark-console/streampark-console-service/src/main/assembly/script/data/pgsql-data.sql
@@ -35,7 +35,7 @@
-- ----------------------------
-- Records of t_flink_project
-- ----------------------------
-insert into "public"."t_flink_project" values (100000, 100000, 'streampark-quickstart', 'https://github.com/apache/incubator-streampark-quickstart', 'release-2.0.0', null, null, null, null, null, 1, 1, null, 'streampark-quickstart', -1, now(), now());
+insert into "public"."t_flink_project" values (100000, 100000, 'streampark-quickstart', 'https://github.com/apache/incubator-streampark-quickstart', 'release-2.0.0', null, null, null, null, null, null, 1, 1, null, 'streampark-quickstart', -1, now(), now());
-- ----------------------------
@@ -87,10 +87,10 @@
insert into "public"."t_menu" values (110603, 110600, 'delete', null, null, 'member:delete', null, '1', '1', null, now(), now());
insert into "public"."t_menu" values (110604, 110600, 'role view', null, null, 'role:view', null, '1', '1', null, now(), now());
insert into "public"."t_menu" values (110605, 110600, 'view', null, null, 'member:view', null, '1', '1', null, now(), now());
-insert into "public"."t_menu" values (120101, 120100, 'add', '/flink/project/add', 'flink/project/Add', 'project:create', '', '0', '0', null, now(), now());
+insert into "public"."t_menu" values (120101, 120100, 'add', '/flink/project/add', 'resource/project/Add', 'project:create', '', '0', '0', null, now(), now());
insert into "public"."t_menu" values (120102, 120100, 'build', null, null, 'project:build', null, '1', '1', null, now(), now());
insert into "public"."t_menu" values (120103, 120100, 'delete', null, null, 'project:delete', null, '1', '1', null, now(), now());
-insert into "public"."t_menu" values (120104, 120100, 'edit', '/flink/project/edit', 'flink/project/Edit', 'project:update', null, '0', '0', null, now(), now());
+insert into "public"."t_menu" values (120104, 120100, 'edit', '/flink/project/edit', 'resource/project/Edit', 'project:update', null, '0', '0', null, now(), now());
insert into "public"."t_menu" values (120105, 120100, 'view', null, null, 'project:view', null, '1', '1', null, now(), now());
insert into "public"."t_menu" values (120201, 120200, 'add', '/flink/app/add', 'flink/app/Add', 'app:create', '', '0', '0', null, now(), now());
insert into "public"."t_menu" values (120202, 120200, 'detail app', '/flink/app/detail', 'flink/app/Detail', 'app:detail', '', '0', '0', null, now(), now());
diff --git a/streampark-console/streampark-console-service/src/main/assembly/script/schema/mysql-schema.sql b/streampark-console/streampark-console-service/src/main/assembly/script/schema/mysql-schema.sql
index ac4e5f2..08fce99 100644
--- a/streampark-console/streampark-console-service/src/main/assembly/script/schema/mysql-schema.sql
+++ b/streampark-console/streampark-console-service/src/main/assembly/script/schema/mysql-schema.sql
@@ -189,8 +189,9 @@
`url` varchar(255) collate utf8mb4_general_ci default null,
`branches` varchar(64) collate utf8mb4_general_ci default null,
`user_name` varchar(64) collate utf8mb4_general_ci default null,
- `password` varchar(64) collate utf8mb4_general_ci default null,
+ `password` varchar(512) collate utf8mb4_general_ci default null,
`prvkey_path` varchar(128) collate utf8mb4_general_ci default null,
+ `salt` varchar(26) collate utf8mb4_general_ci default null,
`pom` varchar(255) collate utf8mb4_general_ci default null,
`build_args` varchar(255) default null,
`type` tinyint default null,
diff --git a/streampark-console/streampark-console-service/src/main/assembly/script/schema/pgsql-schema.sql b/streampark-console/streampark-console-service/src/main/assembly/script/schema/pgsql-schema.sql
index 3b4608b..c3eb9dc 100644
--- a/streampark-console/streampark-console-service/src/main/assembly/script/schema/pgsql-schema.sql
+++ b/streampark-console/streampark-console-service/src/main/assembly/script/schema/pgsql-schema.sql
@@ -434,8 +434,9 @@
"url" varchar(255) collate "pg_catalog"."default",
"branches" varchar(64) collate "pg_catalog"."default",
"user_name" varchar(64) collate "pg_catalog"."default",
- "password" varchar(64) collate "pg_catalog"."default",
+ "password" varchar(512) collate "pg_catalog"."default",
"prvkey_path" varchar(128) collate "pg_catalog"."default",
+ "salt" varchar(26) collate "pg_catalog"."default",
"pom" varchar(255) collate "pg_catalog"."default",
"build_args" varchar(255) collate "pg_catalog"."default",
"type" int2,
diff --git a/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/mysql/2.2.2.sql b/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/mysql/2.2.2.sql
index d4707a8..5c6652f 100644
--- a/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/mysql/2.2.2.sql
+++ b/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/mysql/2.2.2.sql
@@ -244,3 +244,7 @@
alter table `t_flink_log`
add column `user_id` bigint default null comment 'operator user id';
+
+alter table `t_flink_project`
+ add column `salt` varchar(26) collate utf8mb4_general_ci default null comment 'password salt',
+ modify column `password` varchar(512) collate utf8mb4_general_ci default null comment 'password';
diff --git a/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/pgsql/2.2.2.sql b/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/pgsql/2.2.2.sql
index c8d023b..bc69fb9 100644
--- a/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/pgsql/2.2.2.sql
+++ b/streampark-console/streampark-console-service/src/main/assembly/script/upgrade/pgsql/2.2.2.sql
@@ -22,3 +22,11 @@
add column "user_id" int8 collate "pg_catalog"."default";
comment on column "public"."t_flink_log"."user_id" is 'operator user id';
+
+alter table "public"."t_flink_project"
+ add column `salt` varchar(26) collate "pg_catalog"."default";
+
+comment on column "public"."t_flink_project"."salt" is 'password salt';
+
+alter table "public"."t_flink_project"
+ alter column `password` type varchar(512) collate "pg_catalog"."default";
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java
index 4fa5548..72dc1f1 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/base/util/GitUtils.java
@@ -84,9 +84,18 @@
private static void setCredentials(TransportCommand<?, ?> transportCommand, Project project) {
if (project.isHttpRepositoryUrl()) {
if (!StringUtils.isAllEmpty(project.getUserName(), project.getPassword())) {
- UsernamePasswordCredentialsProvider credentialsProvider =
- new UsernamePasswordCredentialsProvider(project.getUserName(), project.getPassword());
- transportCommand.setCredentialsProvider(credentialsProvider);
+ try {
+ String decrypt =
+ StringUtils.isNotBlank(project.getSalt())
+ ? EncryptUtils.decrypt(project.getPassword(), project.getSalt())
+ : project.getPassword();
+ UsernamePasswordCredentialsProvider credentialsProvider =
+ new UsernamePasswordCredentialsProvider(project.getUserName(), decrypt);
+ transportCommand.setCredentialsProvider(credentialsProvider);
+ } catch (Exception e) {
+ throw new IllegalStateException(
+ "[StreamPark] git setCredentials: project password decrypt failed", e);
+ }
}
} else if (project.isSshRepositoryUrl()) {
transportCommand.setTransportConfigCallback(
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
index bbbb2f9..191e8ae 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/controller/ProjectController.java
@@ -105,7 +105,7 @@
@Operation(summary = "List git project branches")
@PostMapping("branches")
public RestResponse branches(Project project) {
- List<String> branches = project.getAllBranches();
+ List<String> branches = projectService.getAllBranches(project);
return RestResponse.success().data(branches);
}
@@ -120,7 +120,7 @@
@Operation(summary = "Authenticate git project")
@PostMapping("gitcheck")
public RestResponse gitCheck(Project project) {
- GitAuthorizedErrorEnum error = project.gitCheck();
+ GitAuthorizedErrorEnum error = projectService.gitCheck(project);
return RestResponse.success().data(error.getType());
}
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
index 9f5bd6c..8efaa0e 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/entity/Project.java
@@ -77,6 +77,9 @@
@TableField(updateStrategy = FieldStrategy.IGNORED)
private String prvkeyPath;
+ /** No salt value is returned */
+ @JsonIgnore private String salt;
+
/** 1:git 2:svn */
private Integer repository;
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java
index ec9d096..6767b24 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/ProjectService.java
@@ -21,6 +21,7 @@
import org.apache.streampark.console.base.domain.RestResponse;
import org.apache.streampark.console.core.entity.Application;
import org.apache.streampark.console.core.entity.Project;
+import org.apache.streampark.console.core.enums.GitAuthorizedErrorEnum;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.service.IService;
@@ -145,4 +146,20 @@
* @return whether the corresponding project exists
*/
boolean exists(Project project);
+
+ /**
+ * Gets branch information under the project
+ *
+ * @param project Project
+ * @return branch information under the project
+ */
+ List<String> getAllBranches(Project project);
+
+ /**
+ * Check git
+ *
+ * @param project Project
+ * @return Check git
+ */
+ GitAuthorizedErrorEnum gitCheck(Project project);
}
diff --git a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
index 778b1a0..1bc0478 100644
--- a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
+++ b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/service/impl/ProjectServiceImpl.java
@@ -28,11 +28,16 @@
import org.apache.streampark.console.base.domain.RestRequest;
import org.apache.streampark.console.base.domain.RestResponse;
import org.apache.streampark.console.base.exception.ApiAlertException;
+import org.apache.streampark.console.base.exception.ApiDetailException;
import org.apache.streampark.console.base.mybatis.pager.MybatisPager;
+import org.apache.streampark.console.base.util.EncryptUtils;
import org.apache.streampark.console.base.util.GZipUtils;
+import org.apache.streampark.console.base.util.GitUtils;
+import org.apache.streampark.console.base.util.ShaHashUtils;
import org.apache.streampark.console.core.entity.Application;
import org.apache.streampark.console.core.entity.Project;
import org.apache.streampark.console.core.enums.BuildStateEnum;
+import org.apache.streampark.console.core.enums.GitAuthorizedErrorEnum;
import org.apache.streampark.console.core.enums.ReleaseStateEnum;
import org.apache.streampark.console.core.mapper.ProjectMapper;
import org.apache.streampark.console.core.service.ProjectService;
@@ -40,6 +45,7 @@
import org.apache.streampark.console.core.task.ProjectBuildTask;
import org.apache.streampark.console.core.watcher.FlinkAppHttpWatcher;
+import org.apache.commons.lang3.StringUtils;
import org.apache.flink.configuration.MemorySize;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
@@ -94,7 +100,17 @@
RestResponse response = RestResponse.success();
ApiAlertException.throwIfTrue(count > 0, "project name already exists, add project failed");
-
+ if (StringUtils.isNotBlank(project.getPassword())) {
+ String salt = ShaHashUtils.getRandomSalt();
+ try {
+ String encrypt = EncryptUtils.encrypt(project.getPassword(), salt);
+ project.setSalt(salt);
+ project.setPassword(encrypt);
+ } catch (Exception e) {
+ log.error("Project password decrypt failed", e);
+ throw new ApiAlertException("Project github/gitlab password decrypt failed");
+ }
+ }
Date date = new Date();
project.setCreateTime(date);
project.setModifyTime(date);
@@ -117,6 +133,26 @@
!project.getBuildState().equals(BuildStateEnum.BUILDING.get()),
"The project is being built, update project failed.");
updateInternal(projectParam, project);
+ if (project.isHttpRepositoryUrl()) {
+ if (StringUtils.isBlank(projectParam.getUserName())) {
+ project.setUserName(null);
+ project.setPassword(null);
+ project.setSalt(null);
+ } else {
+ project.setUserName(projectParam.getUserName());
+ if (!Objects.equals(projectParam.getPassword(), project.getPassword())) {
+ try {
+ String salt = ShaHashUtils.getRandomSalt();
+ String encrypt = EncryptUtils.encrypt(projectParam.getPassword(), salt);
+ project.setPassword(encrypt);
+ project.setSalt(salt);
+ } catch (Exception e) {
+ log.error("The project github/gitlab password encrypt failed");
+ throw new ApiAlertException(e);
+ }
+ }
+ }
+ }
if (project.isSshRepositoryUrl()) {
project.setUserName(null);
} else {
@@ -378,4 +414,36 @@
private String getBuildLogPath(Long projectId) {
return String.format("%s/%s/build.log", Workspace.PROJECT_BUILD_LOG_PATH(), projectId);
}
+
+ @Override
+ public List<String> getAllBranches(Project project) {
+ try {
+ return GitUtils.getBranchList(remakeProject(project));
+ } catch (Exception e) {
+ throw new ApiDetailException(e);
+ }
+ }
+
+ @Override
+ public GitAuthorizedErrorEnum gitCheck(Project project) {
+ try {
+ GitUtils.getBranchList(remakeProject(project));
+ return GitAuthorizedErrorEnum.SUCCESS;
+ } catch (Exception e) {
+ String err = e.getMessage();
+ if (err.contains("not authorized")) {
+ return GitAuthorizedErrorEnum.ERROR;
+ } else if (err.contains("Authentication is required")) {
+ return GitAuthorizedErrorEnum.REQUIRED;
+ }
+ return GitAuthorizedErrorEnum.UNKNOW;
+ }
+ }
+
+ private Project remakeProject(Project project) {
+ if (Objects.nonNull(project.getId())) {
+ return this.baseMapper.selectById(project.getId());
+ }
+ return project;
+ }
}
diff --git a/streampark-console/streampark-console-webapp/src/views/resource/project/useProject.tsx b/streampark-console/streampark-console-webapp/src/views/resource/project/useProject.tsx
index 5c04c71..67c3a15 100644
--- a/streampark-console/streampark-console-webapp/src/views/resource/project/useProject.tsx
+++ b/streampark-console/streampark-console-webapp/src/views/resource/project/useProject.tsx
@@ -244,6 +244,7 @@
userName: values.userName || null,
password: values.password || null,
prvkeyPath: values.prvkeyPath || null,
+ id: route?.query?.id || null,
});
if (res === 0) {
if (branchList.value.length === 0) {
@@ -282,8 +283,9 @@
const prvkeyPath = values.prvkeyPath || null;
const userNull = userName === null || userName === undefined || userName === '';
const passNull = password === null || password === undefined || password === '';
+ const id = route?.query?.id || null;
if ((userNull && passNull) || (!userNull && !passNull)) {
- const res = await fetchBranches({ url, userName, password, prvkeyPath });
+ const res = await fetchBranches({ url, userName, password, prvkeyPath, id });
if (res) branchList.value = res.map((i) => ({ label: i, value: i }));
}
}
